text:
using Test
using NeXLSpectrum
using Statistics
using LinearAlgebra

@testset "Filter Fitting" begin
    @testset "Filter" begin
        eds = simpleEDS(2048, 10.0, 0.0, 135.0)
        filt = buildfilter(eds)
        # Each row sums to zero
        @test all(
            isapprox(sum(NeXLSpectrum.filterdata(filt, row)), 0.0, atol = 1.0e-8) for
            row in size(filt)[1]
        )
        # Symmetric about the center line
        @test all(
            isapprox(
                sum(NeXLSpectrum.filterdata(filt, r)[1:r-1]),
                sum(NeXLSpectrum.filterdata(filt, r)[r+1:end]),
                atol = 1.0e-8,
            ) for r = 2:(size(filt)[1]-1)
        )
        # Positive in the center
        @test all(NeXLSpectrum.filterdata(filt, r)[r] ≥ 0.0 for r = 1:size(filt)[1])
        # Symmetric one row off
        @test all(
            NeXLSpectrum.filterdata(filt, r)[r-1] == NeXLSpectrum.filterdata(filt, r)[r+1]
            for r = 2:(size(filt)[1]-1)
        )
    end

    @testset "LLSQ_K412_1" begin
        path = joinpath(@__DIR__, "K412 spectra")
        unks = loadspectrum.(joinpath(path, "III-E K412[$i][4].msa") for i = 0:4)
        al2o3 = loadspectrum(joinpath(path, "Al2O3 std.msa"))
        caf2 = loadspectrum(joinpath(path, "CaF2 std.msa"))
        fe = loadspectrum(joinpath(path, "Fe std.msa"))
        mgo = loadspectrum(joinpath(path, "MgO std.msa"))
        sio2 = loadspectrum(joinpath(path, "SiO2 std.msa"))

        det = simpleEDS(4096, 10.0, 0.0, 132.0)
        ff = buildfilter(det)

        ok = tophatfilter(
            CharXRayLabel(sio2, 34:66, characteristic(n"O", ktransitions)),
            ff,
            1.0 / dose(sio2),
        )
        mgk = tophatfilter(
            CharXRayLabel(mgo, 110:142, characteristic(n"Mg", ktransitions)),
            ff,
            1.0 / dose(mgo),
        )
        alk = tophatfilter(
            CharXRayLabel(al2o3, 135:170, characteristic(n"Al", ktransitions)),
            ff,
            1.0 / dose(al2o3),
        )
        sik = tophatfilter(
            CharXRayLabel(sio2, 159:196, characteristic(n"Si", ktransitions)),
            ff,
            1.0 / dose(sio2),
        )
        cak = tophatfilter(
            CharXRayLabel(caf2, 345:422, characteristic(n"Ca", ktransitions)),
            ff,
            1.0 / dose(caf2),
        )
        fel = tophatfilter(
            CharXRayLabel(fe, 51:87, characteristic(n"Fe", ltransitions)),
            ff,
            1.0 / dose(fe),
        )
        feka = tophatfilter(
            CharXRayLabel(fe, 615:666, characteristic(n"Fe", kalpha)),
            ff,
            1.0 / dose(fe),
        )
        fekb = tophatfilter(
            CharXRayLabel(fe, 677:735, characteristic(n"Fe", kbeta)),
            ff,
            1.0 / dose(fe),
        )

        fds = [ok, mgk, alk, sik, cak, fel, feka, fekb]

        unk = tophatfilter(unks[1], ff, 1.0 / dose(unks[1]))

        #println("Performing the weighted fit takes:")
        ff = filterfit(unk, fds)
        #@btime filterfit(unk, fds)
        #println("Performing the full generalized fit takes:")
        #@btime filterfit(unk, fds, fitcontiguousp)
        #@btime filterfit(unk, fds, fitcontiguousw)

        ## The comparison is against the k-ratios from DTSA-II.
        # The results won't be identical because the filters and other assumptions are different.
        @test isapprox(NeXLUncertainties.value(ff, ok.label), 0.6529, atol = 0.0016)
        @test isapprox(NeXLUncertainties.value(ff, fekb.label), 0.0665, atol = 0.0002)
        @test isapprox(NeXLUncertainties.value(ff, mgk.label), 0.1473, atol = 0.0004)
        @test isapprox(NeXLUncertainties.value(ff, alk.label), 0.0668, atol = 0.0005)
        @test isapprox(NeXLUncertainties.value(ff, sik.label), 0.3506, atol = 0.0008)
        @test isapprox(NeXLUncertainties.value(ff, cak.label), 0.1921, atol = 0.0001)
        @test isapprox(NeXLUncertainties.value(ff, fel.label), 0.0418, atol = 0.0002)
        @test isapprox(NeXLUncertainties.value(ff, feka.label), 0.0669, atol = 0.0001)

        @test isapprox(σ(ff, ok.label), 0.00081, atol = 0.0001)
        @test isapprox(σ(ff, mgk.label), 0.00018, atol = 0.00005)
        @test isapprox(σ(ff, alk.label), 0.00012, atol = 0.00005)
        @test isapprox(σ(ff, sik.label), 0.00024, atol = 0.00005)
        @test isapprox(σ(ff, cak.label), 0.00022, atol = 0.00005)
        @test isapprox(σ(ff, fel.label), 0.00043, atol = 0.00006)
        @test isapprox(σ(ff, feka.label), 0.00019, atol = 0.00005)
        @test isapprox(σ(ff, fekb.label), 0.00078, atol = 0.0002)
    end

    @testset "LLSQ_K412_2" begin
        path = joinpath(@__DIR__, "K412 spectra")
        unks = loadspectrum.(joinpath(path, "III-E K412[$i][4].msa") for i = 0:4)
        al2o3 = loadspectrum(joinpath(path, "Al2O3 std.msa"))
        caf2 = loadspectrum(joinpath(path, "CaF2 std.msa"))
        fe = loadspectrum(joinpath(path, "Fe std.msa"))
        mgo = loadspectrum(joinpath(path, "MgO std.msa"))
        sio2 = loadspectrum(joinpath(path, "SiO2 std.msa"))

        det = BasicEDS(
            4096,
            0.0,
            10.0,
            132.0,
            10,
            Dict(
                Shell(1) => n"Be",
                Shell(2) => n"Sc",
                Shell(3) => n"Cs",
                Shell(4) => n"Pu",
            ),
        )
        ff = buildfilter(det)
        e0 = sameproperty(unks, :BeamEnergy)

        ampl = 0.00005
        oroi = charXRayLabels(sio2, n"O", Set( ( n"Si", n"O" ) ), det, e0, ampl = ampl)
        siroi = charXRayLabels(sio2, n"Si", Set( (n"Si", n"O") ), det, e0, ampl = ampl)
        mgroi = charXRayLabels(mgo, n"Mg", Set( (n"Mg", n"O") ), det, e0, ampl = ampl)
        alroi = charXRayLabels(al2o3, n"Al", Set( (n"Al", n"O") ), det, e0, ampl = ampl)
        caroi = charXRayLabels(caf2, n"Ca", Set( (n"Ca", n"F") ), det, e0, ampl = ampl)
        @test length(caroi) == 1
        feroi = charXRayLabels(fe, n"Fe", Set( (n"Fe", ) ), det, e0, ampl = ampl)
        @test length(feroi) == 3

        ok = tophatfilter(oroi, ff, 1.0 / dose(sio2))
        mgk = tophatfilter(mgroi, ff, 1.0 / dose(mgo))
        alk = tophatfilter(alroi, ff, 1.0 / dose(al2o3))
        sik = tophatfilter(siroi, ff, 1.0 / dose(sio2))
        cak = tophatfilter(caroi, ff, 1.0 / dose(caf2))
        fekl = tophatfilter(feroi, ff, 1.0 / dose(fe))

        fds = collect(Iterators.flatten((ok, mgk, alk, sik, cak, fekl)))

        unk = tophatfilter(unks[1], ff, 1.0 / dose(unks[1]))

        ff = filterfit(unk, fds)
        #println("Performing the full generalized fit takes:")
        #@btime filterfit(unk, fds)
        #println("Performing the weighted fit takes:")
        #@btime filterfit(unk, fds, fitcontiguousw)

        @test isapprox(NeXLUncertainties.value(ff, oroi[1]), 0.6624, atol = 0.0001)
        @test isapprox(NeXLUncertainties.value(ff, mgroi[1]), 0.14728, atol = 0.0007)
        @test isapprox(NeXLUncertainties.value(ff, alroi[1]), 0.06679, atol = 0.0006)
        @test isapprox(NeXLUncertainties.value(ff, siroi[1]), 0.35063, atol = 0.0009)
        @test isapprox(NeXLUncertainties.value(ff, caroi[1]), 0.19213, atol = 0.0003)
        @test isapprox(NeXLUncertainties.value(ff, feroi[1]), 0.04185, atol = 0.0004)
        @test isapprox(NeXLUncertainties.value(ff, feroi[2]), 0.06693, atol = 0.0001)
        @test isapprox(NeXLUncertainties.value(ff, feroi[3]), 0.06652, atol = 0.0007)

        @test isapprox(σ(ff, oroi[1]), 0.00082, atol = 0.0001)
        @test isapprox(σ(ff, mgroi[1]), 0.00018, atol = 0.00004)
        @test isapprox(σ(ff, alroi[1]), 0.00016, atol = 0.00003)
        @test isapprox(σ(ff, siroi[1]), 0.00029, atol = 0.00003)
        @test isapprox(σ(ff, caroi[1]), 0.00023, atol = 0.00001)
        @test isapprox(σ(ff, feroi[1]), 0.00044, atol = 0.0001)
        @test isapprox(σ(ff, feroi[2]), 0.00016, atol = 0.00001)
        @test isapprox(σ(ff, feroi[3]), 0.00078, atol = 0.0002)

        # Compare to naive peak integration
        fekkr = kratio(unks[1], fe, 593:613, 636:647, 669:690)
        @test isapprox(
            NeXLUncertainties.value(ff, feroi[2]),
            NeXLUncertainties.value(fekkr),
            atol = 0.0005,
        )
        @test isapprox(σ(ff, feroi[2]), σ(fekkr), atol = 0.00004)

        cakkr = kratio(unks[1], caf2, 334:347, 365:375, 422:439)
        @test isapprox(
            NeXLUncertainties.value(ff, caroi[1]),
            NeXLUncertainties.value(cakkr),
            atol = 0.0008,
        )
        @test isapprox(σ(ff, caroi[1]), σ(cakkr), atol = 0.00007)
    end

    @testset "ADM6005a" begin
        path = joinpath(@__DIR__, "ADM6005a spectra")
        unks = loadspectrum.(joinpath(path, "ADM-6005a_$(i).msa") for i = 1:15)
        al = loadspectrum(joinpath(path, "Al std.msa"))
        caf2 = loadspectrum(joinpath(path, "CaF2 std.msa"))
        fe = loadspectrum(joinpath(path, "Fe std.msa"))
        ge = loadspectrum(joinpath(path, "Ge std.msa"))
        si = loadspectrum(joinpath(path, "Si std.msa"))
        sio2 = loadspectrum(joinpath(path, "SiO2 std.msa"))
        ti = loadspectrum(joinpath(path, "Ti trimmed.msa"))
        zn = loadspectrum(joinpath(path, "Zn std.msa"))

        det = matching(
            unks[1],
            128.0,
            110,
            Dict(
                Shell(1) => n"Be",
                Shell(2) => n"Sc",
                Shell(3) => n"Cs",
                Shell(4) => n"Pu",
            ),
        )
        ff = buildfilter(det)

        ampl = 1e-4
        e0 = sameproperty(unks, :BeamEnergy)
        alroi = charXRayLabels(al, n"Al", Set( ( n"Al", )), det, e0, ampl = ampl)
        caroi = charXRayLabels(caf2, n"Ca", [ n"Ca", n"F" ], det, e0, ampl = ampl)
        feroi = charXRayLabels(fe, n"Fe", Set( ( n"Fe", )), det, e0, ampl = ampl)
        geroi = charXRayLabels(ge, n"Ge", Set( ( n"Ge", )), det, e0, ampl = ampl)
        oroi = charXRayLabels(sio2, n"O", Set( ( n"Si", n"O", )), det, e0, ampl = ampl)
        siroi = charXRayLabels(si, n"Si", Set( ( n"Si", )), det, e0, ampl = ampl)
        tiroi = charXRayLabels(ti, n"Ti", Set( ( n"Ti", )), det, e0, ampl = ampl)
        znroi = charXRayLabels(zn, n"Zn", [ n"Zn" ], det, e0, ampl = ampl)

        alk = tophatfilter(alroi, ff, 1.0 / dose(al))
        cak = tophatfilter(caroi, ff, 1.0 / dose(caf2))
        felk = tophatfilter(feroi, ff, 1.0 / dose(fe))
        gelk = tophatfilter(geroi, ff, 1.0 / dose(ge))
        ok = tophatfilter(oroi, ff, 1.0 / dose(sio2))
        sik = tophatfilter(siroi, ff, 1.0 / dose(si))
        tilk = tophatfilter(tiroi, ff, 1.0 / dose(ti))
        znlk = tophatfilter(znroi, ff, 1.0 / dose(zn))

        fds = collect(Iterators.flatten((alk, cak, felk, gelk, ok, sik, tilk, znlk)))

        res = FilterFitResult[]
        for i = 1:15
            unk = tophatfilter(unks[i], ff, 1.0 / dose(unks[i]))
            push!(res, filterfit(unk, fds))
        end

        # Compare against DTSA-II values
        @test isapprox(mean(values(res, oroi[1])), 0.4923, rtol = 0.003)
        @test isapprox(mean(values(res, siroi[1])), 0.0214, atol = 0.013)
        @test isapprox(mean(values(res, alroi[1])), 0.0281, atol = 0.001)
        @test isapprox(mean(values(res, caroi[1])), 0.1211, rtol = 0.0025)
        @test isapprox(mean(values(res, znroi[1])), 0.0700, rtol = 0.05)
        @test isapprox(mean(values(res, znroi[2])), 0.1115, atol = 0.0005)
        @test isapprox(mean(values(res, znroi[3])), 0.1231, rtol = 0.01)
        @test isapprox(mean(values(res, tiroi[1])), 0.0404, atol = 0.001)
        @test isapprox(mean(values(res, tiroi[2])), 0.064, rtol = 0.0002)
        @test isapprox(mean(values(res, tiroi[3])), 0.064, rtol = 0.06)
        @test isapprox(mean(values(res, feroi[1])), 0.0, atol = 0.001)
        @test isapprox(mean(values(res, feroi[2])), 0.0, atol = 0.0004)
        @test isapprox(mean(values(res, feroi[3])), 0.0, atol = 0.001)
        @test isapprox(mean(values(res, geroi[1])), 0.1789, rtol = 0.01)
        @test isapprox(mean(values(res, geroi[2])), 0.2628, atol = 0.001)
        @test isapprox(mean(values(res, geroi[3])), 0.279, atol = 0.011)
    end
       @testset "ADM6005a - Refs" begin
        path = joinpath(@__DIR__, "ADM6005a spectra")
        unks = loadspectrum.(joinpath(path, "ADM-6005a_$(i).msa") for i = 1:15)
        det = matching(
            unks[1],
            128.0,
            110,
            Dict(
                Shell(1) => n"Be",
                Shell(2) => n"Sc",
                Shell(3) => n"Cs",
                Shell(4) => n"Pu",
            ),
        )
        ffp = references(
            [
                reference(n"Al", joinpath(path, "Al std.msa"), mat"Al"),
                reference(n"Ca", joinpath(path, "CaF2 std.msa"), mat"CaF2"),
                reference(n"Fe", joinpath(path, "Fe std.msa"), mat"Fe"),
                reference(n"Ge", joinpath(path, "Ge std.msa"), mat"Ge"),
                reference(n"Si", joinpath(path, "Si std.msa"), mat"Si"),
                reference(n"O", joinpath(path, "SiO2 std.msa"), mat"SiO2"),
                reference(n"Ti", joinpath(path, "Ti trimmed.msa"), mat"Ti"),
                reference(n"Zn", joinpath(path, "Zn std.msa"), mat"Zn"),
            ],
            det,
        )
        res = fit_spectrum(unks, ffp)
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Al K-L3"))),
            0.0279,
            atol = 0.0001,
        )
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Ti K-L3"))),
            0.0641,
            atol = 0.0001,
        )
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Ge K-M3"))),
            0.2734,
            atol = 0.0001,
        )
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Zn K-M3"))),
            0.1209,
            atol = 0.0001,
        )
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Fe L3-M5"))),
            0.00033,
            atol = 0.00001,
        )
        @test isapprox(
            mean(values(res, findlabel(res[1], n"Fe K-L3"))),
            0.0003026,
            atol = 0.00001,
        )
    end

    # Check that the covariance of the filtered spectrum is calculated correctly as F*diagm(S)*transpose(F)
    @testset "Filtered covariance" begin
        spec = loadspectrum(joinpath(@__DIR__, "ADM6005a spectra", "ADM-6005a_1.msa"))
        det = matching(
            spec,
            128.0,
            110,
            Dict(
                Shell(1) => n"Be",
                Shell(2) => n"Sc",
                Shell(3) => n"Cs",
                Shell(4) => n"Pu",
            ),
        )
        filt = buildfilter(VariableWidthFilter, det)
        specdata = counts(spec)
        cov1 = [
            NeXLSpectrum.filteredcovar(filt, specdata, r, c) for
            r in eachindex(specdata), c in eachindex(specdata)
        ]
        filtd = NeXLSpectrum.filterdata(filt)
        cov2 = filtd * diagm(specdata) * transpose(filtd)
        # @show findmax(ii->abs(cov1[ii]-cov2[ii]), eachindex(cov1))
        @test all(
            isapprox(cov1[ii], cov2[ii], rtol = 1.0e-6, atol = 1.0e-12) for
            ii in eachindex(cov1)
        )
    end

    @testset "Repeated refs" begin
        path = joinpath(@__DIR__, "K412 spectra")
        fe = mat"Fe"
        efs = references(
                   [
                       reference(n"Ca", joinpath(path,"III-E K412[0][4].msa"), srm470_k412),
                       reference(n"Fe", joinpath(path,"III-E K412[0][4].msa"), srm470_k412),
                       reference(n"O", joinpath(path, "SiO2 std.msa"), mat"SiO2"),
                       reference(n"Al", joinpath(path, "Al2O3 std.msa"), mat"Al2O3"),
                       reference(n"Ca", joinpath(path, "CaF2 std.msa"), mat"CaF2"),
                       reference(n"Fe", joinpath(path, "Fe std.msa"), fe),
                       reference(n"Mg", joinpath(path, "MgO std.msa"), mat"MgO"),
                       reference(n"Si", joinpath(path, "SiO2 std.msa"), mat"SiO2"),
                   ],
                   132.0,
               )
        @test properties(efs.references[findfirst(r->n"Fe K-L3" in r.label.xrays, efs.references)].label)[:Composition] === srm470_k412
        @test properties(efs.references[findfirst(r->n"Fe K-M3" in r.label.xrays, efs.references)].label)[:Composition] === srm470_k412
        @test properties(efs.references[findfirst(r->n"Fe L3-M5" in r.label.xrays, efs.references)].label)[:Composition] === fe
        @test properties(efs.references[findfirst(r->n"Ca K-L3" in r.label.xrays, efs.references)].label)[:Composition] === srm470_k412
    end

    @testset "Warnings" begin
        s = loadspectrum(joinpath(@__DIR__, "Other", "K411 simulated.msa"))
        @test_logs ( :warn, "The spectrum \"Noisy[MC simulation of bulk K411] #1\" cannot be used as a reference for the ROI \"O K-L3 + 1 other\" due to 2 peak interferences.")
            charXRayLabels(s, n"O", Set( ( n"C", n"O",n"Mg",n"Si",n"Ca",n"Fe")), simpleEDS(2048,10.0,0.0,132.0), 1.0e6, ampl=1.0e-5)
        @test_logs ( :warn, "The spectrum \"Noisy[MC simulation of bulk K411] #1\" cannot be used as a reference for the ROI \"Fe L3-M5 + 11 others\" due to 1 peak interference.")
            charXRayLabels(s, n"Fe", Set( ( n"C", n"O",n"Mg",n"Si",n"Ca",n"Fe")), simpleEDS(2048,10.0,0.0,132.0), 1.0e6, ampl=1.0e-5)
    end

    @testset "Example 2" begin
        path = joinpath(@__DIR__, "Example 2")
        refs = references( [
            reference( [ n"Mg", n"Si", n"Ca", n"Fe" ], joinpath(path, "K411 std.msa"), srm470_k411)...,
            reference( n"O", joinpath(path,"MgO std.msa"), mat"MgO" ),
            reference( n"Fe", joinpath(path,"Fe std.msa"), mat"Fe" ),
            reference( n"Al", joinpath(path,"Al std.msa"), mat"Al" )
        ], 135.0)
        unk = loadspectrum(joinpath(path, "K412 unk.msa"))
        fr = fit_spectrum(unk, refs)
        qr = quantify(fr)
        @test isapprox(value(qr.comp[n"Al"]), 0.0510, atol=0.0001)
        @test isapprox(value(qr.comp[n"Fe"]), 0.0783, atol=0.0001)
        @test isapprox(value(qr.comp[n"Mg"]), 0.1183, atol=0.0001)
        @test isapprox(value(qr.comp[n"O"]), 0.4471, atol=0.0001)
    end
end


text:
using BenchmarkTools
using SignatureGB
using Combinatorics
using Dictionaries
using Singular

SG = SignatureGB

function divmask_benchmark(nvars, nmons)
    suite = BenchmarkGroup()

    suite["monomials"] = BenchmarkGroup()

    order = SG.Grevlex(nvars)
    ctx = SG.monomialctx(exponents = Int64, order=order)
    idx = SG.ixmonomialctx(ctx)

    mons = [SG.Monomial(rand(1:100, SVector{nvars, Int})) for i in 1:nmons]
    idx_mons = [idx(m) for m in mons]
    itr = combinations(idx_mons, 2)

    suite["monomials"]["divis_with_mask"] = @benchmarkable [SG.divides(idx, ts[1], ts[2]) for ts in $(itr)]
    suite["monomials"]["naive_divis"] = @benchmarkable [SG.divides(ctx, idx.table[ts[1]], idx.table[ts[2]]) for ts in $(itr)]
    suite, idx, ctx, mons
end

function sliceddict_benchmark()
    suite = BenchmarkGroup()
    alph = ["a", "b", "c", "d", "e", "f", "g", "h"]
    rng = 1:100000
    keys = [(lett, i) for lett in alph for i in rng]
    dct_1 = Dictionary(keys, keys)
    dct_2 = SG.SlicedDict(keys, keys)
    suite["plain_dict"] = @benchmarkable getindices($(dct_1), filter(a -> a[1] == "a", keys($(dct_1))))
    suite["sliced_dict"] = @benchmarkable $(dct_2)["a"]
    suite, dct_1, dct_2
end

function kd_tree_benchmark(npols, nmons)
    suite = BenchmarkGroup()
    R, vars = PolynomialRing(Fp(101), ["x$(i)" for i in 1:32])
    order = SG.Grevlex(32)
    char = 101
    ctx = SG.idxsigpolynomialctx(SG.Nmod32Γ(char), order = order)
    pols = filter(p -> !(iszero(p)), [rand(R, 1:1, 1:10) for _ in 1:npols])
    sigs = [ctx(i, R(1)) for i in eachindex(pols)]
    for (p, s) in zip(pols, sigs)
        ctx(s, p)
    end
    exp_type = SG.exponenttype(ctx.po.mo)
    exps = exp_type[collect(1:10)...]
    kd_mons = [ctx.po.mo(SG.Monomial(SVector{32, exp_type}([j == i ? rand(exps) : zero(exp_type) for j in 1:32])))
               for i in [rand(1:32) for _ in 1:16]]
    kd_tree = SG.Kd_node(kd_mons, SG.pos_type(ctx))
    [SG.insert_new_basis_element!(ctx, kd_tree, s) for s in sigs]
    mons = [ctx.po.mo(SG.Monomial(rand(1:10, SVector{32, exp_type}))) for _ in 1:nmons]

    suite["div_query_kd_tree"] = @benchmarkable [SG.div_query($(ctx), $(kd_tree), m) for m in $(mons)]
    suite["div_query"] = @benchmarkable [filter(be -> SG.divides($(ctx).po.mo, SG.leadingmonomial($(ctx), be), m),
                                                $(sigs)) for m in $(mons)]
    suite, ctx, kd_tree, mons, sigs
end

ext: jl


text:
using FakePackage, ReTest

retest(FakePackage, recursive=false)
@test FakePackage.RUN == [1, 2]

FakePackage.runtests(r"begin-end")
@test FakePackage.RUN == [1, 2, 1]
FakePackage.runtests(r" for")
@test FakePackage.RUN == [1, 2, 1, 2]

retest(dry=true) # only to update InlineTest.TESTED_MODULES
@test FakePackage.Sub1 in ReTest.TESTED_MODULES
@test FakePackage.Sub2.SubSub in ReTest.TESTED_MODULES

retest(FakePackage, recursive=true)
@test FakePackage.RUN == [1, 2, 1, 2, 1, 2, 3, 4]
retest(FakePackage) # equivalent to recursive=true
@test FakePackage.RUN == [1, 2, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4]

module Included
using ReTest
RUN = []

@testset "include parent" begin
    include("included.jl")
    include(joinpath(@__DIR__, "included.jl"))
    if VERSION >= v"1.5"
        include(identity, "included.jl")
        include(identity, joinpath(@__DIR__, "included.jl"))
    end
end
end

retest(Included)
@test Included.RUN == (VERSION >= v"1.5" ? [0, 0, 0, 0] : [0, 0])


text:
using DrWatson
@quickactivate("HyperspectraWithNeXL")

using HyperspectraWithNeXL
using NeXLSpectrum
using Profile, Unitful

lt = 0.72*4.0*18.0*3600.0/(1024*1024) # 18.0 hours on 4 detectors
hs = NeXLSpectrum.compress(HyperSpectrum(
    LinearEnergyScale(0.0,10.0),
    Dict{Symbol,Any}(
      :TakeOffAngle => deg2rad(35.0),
      :ProbeCurrent => 1.0,
      :LiveTime => lt,
      :BeamEnergy => 20.0e3,
      :Name => "Mn Nodule"
    ),
    readrplraw(joinpath(datadep"MnNodule","map[15]")),
    fov = [ 4.096u"mm", 4.096u"mm"], offset= [ 0.0u"mm", 0.0u"mm" ]
))
hs[:Detector] = matching(hs, 132.0)

hs = hs[1:16:1024,1:16:1024]

refpath = datadep"MnNodule_Standards"
refs = references( [
    reference(n"Ag", joinpath(refpath, "Ag std.msa") ),
    reference(n"Al", joinpath(refpath, "Al std.msa") ),
    reference(n"C", joinpath(refpath, "C std.msa") ),
    reference(n"Ca", joinpath(refpath, "CaF2 std.msa") ),
    reference(n"Ce", joinpath(refpath, "CeO2 std.msa") ),
    reference(n"Cl", joinpath(refpath, "NaCl std.msa") ),
    reference(n"Cr", joinpath(refpath, "Cr std.msa") ),
    reference(n"Cu", joinpath(refpath, "Cu std.msa") ),
    reference(n"Fe", joinpath(refpath, "Fe std.msa") ),
    reference(n"S", joinpath(refpath, "FeS2 std.msa") ),
    reference(n"P", joinpath(refpath, "GaP std.msa") ),
    reference(n"K", joinpath(refpath, "KBr std.msa") ),
    reference(n"Mg", joinpath(refpath, "Mg std.msa") ),
    reference(n"O", joinpath(refpath, "MgO std.msa") ),
    reference(n"Mn", joinpath(refpath, "Mn std.msa") ),
    reference(n"Na", joinpath(refpath, "NaCl std.msa") ),
    reference(n"Ni", joinpath(refpath, "Ni std.msa") ),
    reference(n"Si", joinpath(refpath, "Si std.msa") ),
    reference(n"Ti", joinpath(refpath, "Ti std.msa") ),
    reference(n"Zn", joinpath(refpath, "Zn std.msa") ) ], 132.0
)

@time fit_spectrum(hs, refs, mode = :Full)

resf = @profview fit_spectrum(hs, refs, mode = :Full)


text:
## UChar = Class("Char", 8)
## Stream = List(UChar, Variable, [:v, :u, :v1])

function readBin(x :: Char)
    if x == '0'
        -1.0
    else
        1.0
    end
end

function readChar(char_int :: UInt8)
    Object("u", UChar, [readBin(x) for x in bin(char_int, 8)])
end

function readVar(xs :: Array{UInt8,1})
    if length(xs) == 0
        Stream.empty
    else
        apply(Stream.cons, [readChar(xs[1]), readVar(xs[2:end])])
    end
end

function readVar(xs)
    readVar(string(xs).data)
end

function prog(x :: Symbol)
    [var, readVar(x)]
end

function prog(x :: Expr)
    if x.head == :local
        [definition, readVar(x.args[1].args[1]), prog(x.args[1].args[2])]
    elseif x.head == :(=)
        [assignment, readVar(x.args[1]), prog(x.args[2])]
    elseif x.head == :function
        [procedure, readVar(x.args[1].args[1]), prog(x.args[2])]
    elseif x.head == :call
        [func_call, prog(x.args[1]), prog(x.args[2])]
    elseif x.head == :block
        [Seq.cons, prog(x.args[1]), prog(x.args[2:end])]
    else
        Seq.empty
    end
end

function prog(x :: Array)
    if length(x) == 0
        Seq.empty
    else
        [Seq.cons, prog(x[1]), prog(x[2:end])]
    end
end

type REPL
    env :: Object
    ans :: Object
end

function repl(r :: REPL, command :: String)
    command = replace(command, "var ", "local ")
    t = toTree(prog(parse(command)))
    init_tree!(t)
    eval_tree!(t)
    o = Object("c", Prog, t.value[:value])
    r.ans = apply(f_eval, [o, r.env])
    apply!(s_eval, [o, r.env], r.env)
    r.ans
end

max_stars_repo_name: Non-Contradiction/NextGeneration

max_issues_repo_name: Non-Contradiction/NextGeneration

max_forks_repo_name: Non-Contradiction/NextGeneration


text:
module RR

    using Cxx
    using NativeDebugger
    import NativeDebugger: fallible_load, load, store!, mapped_file, enable, disable, ip
    using NativeDebugger: Location
    using ObjFileBase

    import Base: ==

    function __init__()
        DEPS_DIR = joinpath(dirname(@__FILE__),"..","deps")
        Libdl.dlopen(joinpath(DEPS_DIR,"usr/lib/librr.so"),
            Libdl.RTLD_GLOBAL)
        Cxx.addHeaderDir(joinpath(DEPS_DIR,"src/rr/src"), kind = C_System)
        Cxx.addHeaderDir(joinpath(DEPS_DIR,"build/rr"), kind = C_System)
        cxx"""
            #include <RecordSession.h>
            #include <RecordTask.h>
            #include <ReplayTask.h>
            #include <ReplayTimeline.h>
            #include <AutoRemoteSyscalls.h>
            #include <GdbServer.h>
            #include <algorithm>
            #include <kernel_metadata.h>
        """
    end
    __init__()

    const ReplayTask = pcpp"rr::ReplayTask"
    const AnyTask = Union{ReplayTask,pcpp"rr::RecordTask",pcpp"rr::Task"}
    const ReplaySession = Union{cxxt"rr::ReplaySession::shr_ptr",pcpp"rr::ReplaySession"}
    const RecordSession = Union{cxxt"rr::RecordSession::shr_ptr",pcpp"rr::RecordSession"}
    const ReplayTimeline = Union{pcpp"rr::ReplayTimeline"}
    const Session = Union{ReplaySession, RecordSession}

    function Base.current_task(session::ReplaySession)
        @assert icxx"$session != 0;"
        task = icxx"$session->current_task();"
        @assert Ptr{Void}(task) != C_NULL
        task
    end

    function Base.current_task(session::RecordSession)
        @assert icxx"$session != 0;"
        task = icxx"$session->scheduler().current();"
        @assert Ptr{Void}(task) != C_NULL
        task
    end
    Base.current_task(timeline::ReplayTimeline) =
        current_task(current_session(timeline))
    Base.current_task(task::AnyTask) = task

    current_session(timeline::ReplayTimeline) = icxx"&$timeline->current_session();"
    current_session(session::Session) = session

    function step_until_exec!(session)
        while !icxx"$session->done_initial_exec();"
            step!(session)
        end
    end

    function step!(session::ReplaySession)
        icxx"$session->replay_step(rr::RUN_CONTINUE);"
    end

    function step!(session::RecordSession)
        icxx"$session->record_step();"
    end

    function step!(session::ReplaySession, target; target_is_event = false)
        icxx"""
            rr::ReplaySession::StepConstraints c(rr::RUN_CONTINUE);
            if ($target_is_event)
                c.stop_at_time = $target;
            else
                c.ticks_target = $target;
            $session->replay_step(c);
        """
    end

    function reverse_continue!(timeline)
        icxx"""
        auto stop_filter = [&](rr::ReplayTask* t) -> bool {
            return true;
        };
        auto interrupt_check = [&]() { return false; };
        $timeline->reverse_continue(
            stop_filter, interrupt_check);
        """
    end

    function reverse_single_step!(session, task, timeline)
        icxx"""
        auto stop_filter = [&](rr::ReplayTask* t) -> bool {
            return true;
        };
        auto interrupt_check = [&]() { return false; };
        $timeline->reverse_singlestep(
            $task->tuid(), $task->tick_count(), stop_filter, interrupt_check);
        """
    end
    reverse_single_step!(timeline) = reverse_single_step!(current_session(timeline),
        current_task(current_session(timeline)), timeline)

    function NativeDebugger.single_step!(session::ReplaySession)
        while true
            status = icxx"$session->replay_step(rr::RUN_SINGLESTEP);"
            if icxx"$status.break_status.singlestep_complete == true;"
                return true
            elseif icxx"$status.status == rr::REPLAY_EXITED;"
                return false
            end
        end
    end

    function at_breakpoint(timeline::ReplayTimeline)
        loc = NativeDebugger.Location(timeline,
            ip(icxx"$(current_task(current_session(timeline)))->regs();"))
        haskey(NativeDebugger.bps_at_location, loc)
    end

    function orig_byte(task,addr)
        icxx"""
            auto it = $task->vm()->breakpoints.find($addr);
            assert(it != $task->vm()->breakpoints.end());
            it->second.overwritten_data;
        """
    end

    function emulate_single_step!(timeline::ReplayTimeline,
            vm = current_task(current_session(timeline)))
        task = current_task(current_session(timeline))
        regs = icxx"$(task)->regs();"
        addr = ip(regs)
        icxx"""
            auto it = $task->vm()->breakpoints.find($addr);
            it == $task->vm()->breakpoints.end();
        """ && return false
        insts = load(task, RemotePtr{UInt8}(ip(regs)), 15)
        insts[1] = orig_byte(task, addr)
        NativeDebugger.X86_64.instemulate!(insts, vm, regs) || return false
        icxx"$task->set_regs($regs);"
        return true
    end

    function NativeDebugger.single_step!(timeline::ReplayTimeline)
        # To get past any breakpoints, check if our current location
        # is a breakpoint location and if so, temporarily clear it
        # while stepping past.
        did_disable = false
        loc = NativeDebugger.Location(timeline,
            ip(icxx"$(current_task(current_session(timeline)))->regs();"))
        if haskey(NativeDebugger.bps_at_location, loc)
            disable(loc)
            did_disable = true
        end
        res = NativeDebugger.single_step!(current_session(timeline))
        if did_disable
            enable(loc)
        end
        res
    end

    function NativeDebugger.step_until_bkpt!(session::ReplaySession)
        while disable_sigint() do
                !icxx"$session->replay_step(rr::RUN_CONTINUE).break_status.breakpoint_hit;"
            end
        end
    end

    # We consider the last thread exit to be an implicit breakpoint
    function is_last_thread_exit(status)
        is_task_exit(status) && icxx"$status.task->task_group()->task_set().size() == 1;"
    end

    function is_task_exit(status)
        icxx"$status.task_exit == true;"
    end

    is_break_sig(status) = icxx"$status.signal != nullptr;" && (
      icxx"$status.signal->si_signo == 11;" ||
      icxx"$status.signal->si_signo == 4;")

    function NativeDebugger.step_until_bkpt!(timeline::ReplayTimeline; only_current_tgid = false)
        current_tgid = icxx"$(current_task(timeline))->tgid();"
        while true
            icxx"$timeline->apply_breakpoints_and_watchpoints();"
            icxx"$(current_session(timeline))->set_visible_execution(true);"
            local res
            exited, bp_hit = disable_sigint() do
                res = icxx"$(current_session(timeline))->replay_step(rr::RUN_CONTINUE);"
                (icxx"$res.status == rr::REPLAY_EXITED;",
                 icxx"$res.break_status.breakpoint_hit;" ||
                 icxx"!$res.break_status.watchpoints_hit.empty();" ||
                 is_last_thread_exit(icxx"$res.break_status;") ||
                 is_break_sig(icxx"$res.break_status;"))
            end
            if !only_current_tgid || icxx"$(current_task(timeline))->tgid();" == current_tgid
                exited && return (false, res)
                bp_hit && return (true, res)
            end
            icxx"$timeline->maybe_add_reverse_exec_checkpoint(rr::ReplayTimeline::LOW_OVERHEAD);"
        end
    end

    function step_to_address!(timeline::ReplayTimeline, theip; disable_bps = false)
        if disable_bps
            icxx"$timeline->unapply_breakpoints_and_watchpoints();"
        else
            NativeDebugger.single_step!(timeline)
        end
        icxx"$(current_task(current_session(timeline)))->vm()->add_breakpoint($theip, rr::BKPT_USER);"
        NativeDebugger.step_until_bkpt!(current_session(timeline))
        icxx"$(current_task(current_session(timeline)))->vm()->remove_breakpoint($theip, rr::BKPT_USER);"
        disable_bps && icxx"$timeline->apply_breakpoints_and_watchpoints();"
        nothing
    end

    function NativeDebugger.continue!(timeline::ReplayTimeline; only_current_tgid = false)
        while true
            bp_hit, res = NativeDebugger.step_until_bkpt!(timeline; only_current_tgid = only_current_tgid)
            bp_hit || return res
            regs = icxx"$(current_task(current_session(timeline)))->regs();"
            if process_lowlevel_conditionals(Location(timeline, ip(regs)), regs)
                return res
            end
            # Step past the breakpoint
            NativeDebugger.single_step!(timeline)
        end
    end

    immutable TraceFrameIterator
        reader
    end

    Base.start(it::TraceFrameIterator) = nothing
    function Base.next(it::TraceFrameIterator, _)
        icxx"$(it.reader).read_frame();", nothing
    end
    Base.done(it::TraceFrameIterator, _) = icxx"$(it.reader).at_end();"
    Base.iteratorsize(it::TraceFrameIterator) = Base.SizeUnknown()

    function Base.show(io::IO, frame::Union{rcpp"rr::TraceFrame",vcpp"rr::TraceFrame"})
        print(io,icxx"$frame.ticks();",": ")
        print_with_color(:green,io,String(icxx"$frame.event().str();"))
        if icxx"$frame.event().is_syscall_event();"
             println(io, " ", unsafe_string(
                icxx"rr::state_name($frame.event().Syscall().state);"))
        end
    end
    tid(frame::Union{rcpp"rr::TraceFrame",vcpp"rr::TraceFrame"}) = icxx"$frame.tid();"
    time(frame::Union{rcpp"rr::TraceFrame",vcpp"rr::TraceFrame"}) = icxx"$frame.time();"

    function NativeDebugger.read_exe(task::AnyTask)
        @assert task != C_NULL
        readmeta(IOBuffer(open(read,Cxx.unsafe_string(icxx"$task->vm()->exe_image();"))))
    end

    function NativeDebugger.read_exe(session::Session)
        NativeDebugger.read_exe(current_task(session))
    end

    NativeDebugger.read_exe(timeline::ReplayTimeline) =
        NativeDebugger.read_exe(current_session(timeline))

    # Mapping remote mappings
    function map_remote(task, mapping::cxxt"const rr::KernelMapping*")
        fd = icxx"""
            rr::AutoRemoteSyscalls remote($task);
            rr::ScopedFd local_fd = remote.retrieve_fd($(mapping)->tracee_fd());
            return dup(local_fd.get());
        """
        msize = icxx"$(mapping)->size();";
        addr = ccall(:mmap,
            Ptr{UInt8}, (Ptr{Void},Csize_t,Cint,Cint,Cint,Int64),
            C_NULL, msize, PROT_READ | PROT_WRITE, Base.Mmap.MAP_SHARED, fd, 0)
        systemerror("mmap",addr==Ptr{Void}(-1))
        r = pointer_to_array(addr, (msize,), false)
        ccall(:close, Void, (Cint,), fd)
        r
    end

    immutable TaskUid
        tuid::cxxt"rr::TaskUid"
    end
    ==(a::TaskUid, b::TaskUid) = icxx"$(a.tuid) == $(b.tuid);"
    tuid(task::AnyTask) = TaskUid(icxx"$task->tuid();")

    # Remote memory operations
    function Cxx.cppconvert{T}(ptr::RemotePtr{T})
        icxx"rr::remote_ptr<$T>{$(ptr.ptr)};"
    end
    function Cxx.cppconvert(ptr::RemoteCodePtr)
        icxx"rr::remote_code_ptr{$(ptr.ptr)};"
    end
    Base.convert(::Type{UInt64}, ptr::vcpp"rr::remote_code_ptr") =
        icxx"$ptr.register_value();"
    Base.convert{T}(::Type{UInt64}, ptr::cxxt"rr::remote_ptr<$T>") =
        icxx"$ptr.as_int();"
    Base.convert{T}(::Type{RemotePtr{T}}, ptr::cxxt"rr::remote_ptr<$T>") =
        RemotePtr{T}(UInt(ptr))

    typealias RRRemotePtr{T} Union{RemotePtr{T,UInt32}, RemotePtr{T,UInt64}, cxxt"rr::remote_ptr<$T>"}
    NativeDebugger.RemotePtr{T}(ptr::cxxt"rr::remote_ptr<$T>") = NativeDebugger.RemotePtr{T}(UInt64(ptr))
    NativeDebugger.RemoteCodePtr(ptr::cxxt"rr::remote_code_ptr") = NativeDebugger.RemoteCodePtr(UInt64(ptr))
    function load{T<:Cxx.CxxBuiltinTypes}(vm::AnyTask, ptr::RRRemotePtr{T})
        ok = Ref{Bool}(true)
        res = icxx"$vm->read_mem($ptr,&$ok);"
        ok[] || error("Failed to read memory at address $ptr")
        res
    end
    load(vm::ReplaySession, ptr) = load(current_task(vm), ptr)

    function load{T}(vm::AnyTask, ptr::RRRemotePtr{T})
        ok = Ref{Bool}(true)
        res = Ref{T}()
        icxx"$vm->read_bytes_helper(
            rr::remote_ptr<uint8_t>($(UInt64(ptr))),
            $(sizeof(T)),(uint8_t*)$(Base.unsafe_convert(Ptr{Void},res)),&$ok);"
        ok[] || error("Failed to read memory at address $ptr")
        res[]
    end

    function store!{T}(vm::AnyTask, ptr::RRRemotePtr{T}, val::T)
        ok = Ref{Bool}(true)
        res = Ref{T}(val)
        icxx"$vm->write_bytes_helper(
            rr::remote_ptr<uint8_t>($(UInt64(ptr))),
            $(sizeof(T)),(uint8_t*)&$res,&$ok);"
        ok[] || error("Failed to write memory at address $ptr")
        nothing
    end

    function store!{T}(vm::AnyTask, ptr::RRRemotePtr{T}, vec::Vector{T})
        ok = Ref{Bool}(true)
        icxx"$vm->write_bytes_helper(
            rr::remote_ptr<uint8_t>($(UInt64(ptr))),
            $(sizeof(vec)),$(Ptr{UInt8}(pointer(vec))),&$ok);"
        ok[] || error("Failed to write memory at address $ptr")
        nothing
    end

    function load{T}(vm::AnyTask, ptr::RRRemotePtr{T}, n)
        ok = Ref{Bool}(true)
        res = Vector{T}(n)
        icxx"$vm->read_bytes_helper(
            rr::remote_ptr<uint8_t>($(UInt64(ptr))),
            $(n*sizeof(T)),$(Ptr{UInt8}(pointer(res))),&$ok);"
        ok[] || error("Failed to read memory at address $ptr")
        res
    end

    function fallible_load{T}(vm::AnyTask, ptr::RRRemotePtr{T}, n)
        ok = Ref{Bool}(true)
        res = Vector{T}(n)
        size = icxx"$vm->read_bytes_fallible(
            rr::remote_ptr<uint8_t>($(UInt64(ptr))),
            $(n*sizeof(T)),$(Ptr{UInt8}(pointer(res))));"
        resize!(res, div(size, sizeof(T)))
        res
    end

    load(vm::ReplayTimeline, args...) =
        load(current_task(current_session(vm)), args...)

    store!(vm::ReplayTimeline, args...) =
        store!(current_task(current_session(vm)), args...)

    # Task `12345` mmap_hardlink_1_julia at ip-loc
    function Base.show(io::IO, task::AnyTask)
        modules = get(io, :modules, nothing)
        ip = UInt(NativeDebugger.ip(fixup_RC(task,icxx"$task->regs();")[2]))
        session = icxx"&$task->session();"
        ssession = icxx"$session->as_replay();"
        ssession == C_NULL && (ssession = icxx"$session->as_record();")
        ssession == C_NULL && (ssession = icxx"$session->as_diversion();")
        print(io, "Task `", icxx"$task->tid;", "` (rec ",
            icxx"$task->rec_tid;", ") ",
            unsafe_string(icxx"$task->vm()->exe_image();"),
            " at 0x",hex(ip)," ",modules !== nothing ?
            NativeDebugger.Unwinder.symbolicate(ssession, modules, ip) : "")
    end

    saved_auxv(vm::pcpp"rr::ReplayTask") = convert(Vector{UInt8}, icxx"$vm->vm()->saved_auxv();")

    function mapped_file(vm::AnyTask, ptr)
        @assert icxx"$vm->vm()->has_mapping($ptr);"
        unsafe_string(icxx"""
            auto &mapping = $vm->vm()->mapping_of($ptr);
            return mapping.recorded_map.is_vdso() ?
              "linux-vdso.so.1" : mapping.map.fsname().c_str();
        """)
    end
    mapped_file(vm::ReplayTimeline, ptr) =
        mapped_file(current_task(current_session(vm)), ptr)

    function NativeDebugger.segment_base(vm::AnyTask, ptr)
        ptr = UInt64(ptr)
        @assert icxx"$vm->vm()->has_mapping($ptr);"
        NativeDebugger.RemotePtr(icxx"$vm->vm()->mapping_of($ptr).map.start();")
    end
    NativeDebugger.segment_base(vm::ReplayTimeline, ptr) =
        NativeDebugger.segment_base(current_task(current_session(vm)), ptr)

    import NativeDebugger.GlibcDyldModules: load_library_map, compute_entry_ptr
    function load_library_map(task::pcpp"rr::ReplayTask", imageh)
        slide = compute_entry_ptr(task, saved_auxv(task)) -
            imageh.file.header.e_entry
        load_library_map(task, imageh, slide)
    end

    # Registers
    import NativeDebugger.Registers: ip, invalidate_regs!, set_sp!, set_ip!,
        set_dwarf!, get_dwarf, getarch
    const RRRegisters = Union{rcpp"rr::Registers",vcpp"rr::Registers"}

    getarch(regs::RRRegisters) = icxx"$regs.arch() == rr::x86_64;" ?
        NativeDebugger.X86_64.X86_64Arch() : NativeDebugger.X86_32.X86_32Arch()
    getarch(task::AnyTask) = icxx"$task->arch() == rr::x86_64;" ?
        NativeDebugger.X86_64.X86_64Arch() : NativeDebugger.X86_32.X86_32Arch()
    getarch(timeline::ReplayTimeline) = getarch(current_task(current_session(timeline)))
    Base.copy(regs::RRRegisters) = icxx"rr::Registers{$regs};"
    ip(regs::RRRegisters) = RemoteCodePtr(icxx"$regs.ip();")
    invalidate_regs!(regs::RRRegisters) = nothing # RR does not track validity
    set_sp!(regs::RRRegisters, sp) = icxx"$regs.set_sp($(RemotePtr{Void}(sp)));"
    set_ip!(regs::RRRegisters, ip) = icxx"$regs.set_ip($(RemoteCodePtr(ip)));"
    function set_dwarf!(regs::RRRegisters, regno::Integer, val)
        if isa(getarch(regs), NativeDebugger.X86_64.X86_64Arch)
            # fs_base and gs_base do not have gdb equivalents
            if regno == NativeDebugger.X86_64.inverse_dwarf[:fs_base]
                return icxx"$regs.fs_base();"
            elseif regno == NativeDebugger.X86_64.inverse_dwarf[:gs_base]
                return icxx"$regs.gs_base();"
            end
            gdbregno = NativeDebugger.X86_64.dwarf2gdb(regno)
            valr = Ref{UInt64}(UInt64(val))
            icxx"$regs.write_register((rr::GdbRegister)$gdbregno,&$valr,$(sizeof(UInt64)));"
        else
            gdbregno = NativeDebugger.X86_32.dwarf2gdb(regno)
            @show val
            valr = Ref{UInt32}(UInt32(val))
            icxx"$regs.write_register((rr::GdbRegister)$gdbregno,&$valr,$(sizeof(UInt32)));"
        end
    end
    function get_dwarf(regs::RRRegisters, regno::Integer)
        if isa(getarch(regs), NativeDebugger.X86_64.X86_64Arch)
            # fs_base and gs_base do not have gdb equivalents
            if regno == NativeDebugger.X86_64.inverse_dwarf[:fs_base]
                return icxx"$regs.fs_base();"
            elseif regno == NativeDebugger.X86_64.inverse_dwarf[:gs_base]
                return icxx"$regs.gs_base();"
            end
            gdbregno = NativeDebugger.X86_64.dwarf2gdb(regno)
        else
            gdbregno = NativeDebugger.X86_32.dwarf2gdb(regno)
        end
        buf = Ref{UInt64}(0)
        defined = Ref{Bool}()
        icxx"$regs.read_register((uint8_t*)&$buf, (rr::GdbRegister)$gdbregno, &$defined);"
        buf[]
    end
    function Base.convert(::Type{NativeDebugger.X86_64.BasicRegs}, regs::RRRegisters)
        retregs = NativeDebugger.X86_64.BasicRegs()
        for i in NativeDebugger.X86_64.basic_regs
            set_dwarf!(retregs, i, get_dwarf(regs, i))
        end
        retregs
    end
    Base.show(io::IO, regs::RRRegisters) = show(io, NativeDebugger.X86_64.BasicRegs(regs))

    function NativeDebugger.get_thread_area_base(task::AnyTask, entry)
        icxx"""
            for (auto &area : $task->thread_areas()) {
                if (area.entry_number == $entry) {
                    return area.base_addr;
                }
            }
            return (unsigned int)0;
        """
    end
    NativeDebugger.get_thread_area_base(timeline::ReplayTimeline, entry) =
        NativeDebugger.get_thread_area_base(current_task(timeline), entry)

    """
        RR hooks syscall instructions by inserting a call instruction and
        redirecting to its own mapped page. If we're on that page, do the first
        unwind step manually to get the correct register context.
    """
    function fixup_RC(task::AnyTask, RC)
        RC = copy(RC)
        theip = ip(RC)
        did_fixup = false
        if UInt64(icxx"$task->vm()->rr_page_start();") <= UInt64(theip) <= UInt64(icxx"$task->vm()->rr_page_start() +
                $task->vm()->rr_page_size();")
            set_ip!(RC,load(task, RemotePtr{RemoteCodePtr}(icxx"$RC.sp();")))
            set_sp!(RC,UInt64(icxx"$RC.sp();")+sizeof(Ptr{Void}))
            did_fixup = true
        end
        did_fixup, RC
    end

    when(session::ReplaySession) = UInt64(icxx"$(current_task(session))->tick_count();")
    when(timeline::ReplayTimeline) = when(current_session(timeline))

    function count_total_ticks(reader)
        icxx"""
            ssize_t nticks = 0;
            $reader.rewind();
            rr::TraceFrame frame;
            while (true) {
                rr::TraceFrame next_frame = $reader.read_frame();
                if ($reader.at_end())
                    break;
                frame = next_frame;
                nticks = std::max(nticks, frame.ticks());
            }
            nticks;
        """
    end
    function count_total_ticks(timeline::ReplayTimeline)
        session = current_session(timeline)
        count_total_ticks(icxx"rr::TraceReader{$session->trace_reader()};")
    end

    function proto_mark(timeline)
        icxx"$timeline->proto_mark();"
    end

    function seek(timeline, mark::cxxt"rr::ReplayTimeline::ProtoMark")
        icxx"$timeline->seek_to_proto_mark($mark);"
    end

    immutable AddressSpaceUid
        pid::Int32
        serial::UInt32
        exec_count::UInt32
    end

    NativeDebugger.current_asid(task::AnyTask) =
        reinterpret(AddressSpaceUid, [icxx"$task->vm()->uid();".data])[]

    NativeDebugger.current_asid(session::ReplaySession) =
        NativeDebugger.current_asid(current_task(session))

    NativeDebugger.current_asid(timeline::ReplayTimeline) =
        NativeDebugger.current_asid(current_session(timeline))

    silence!(timeline) =
        icxx"$(current_session(timeline))->set_visible_execution(false);"

    NativeDebugger.ip(task::AnyTask) = NativeDebugger.ip(icxx"$task->regs();")
    NativeDebugger.ip(timeline::ReplayTimeline) = NativeDebugger.ip(current_task(current_session(timeline)))

    function NativeDebugger.breakpoint(timeline::RR.ReplayTimeline, modules, fname::Symbol)
        syms = NativeDebugger.lookup_syms(timeline, modules, fname)
        bp = NativeDebugger.Breakpoint()
        for (h, base, sym) in syms
            addr = NativeDebugger.RemoteCodePtr(base + ObjFileBase.symbolvalue(sym,
                ObjFileBase.Sections(ObjFileBase.handle(h))))
            NativeDebugger.add_location(bp, NativeDebugger.Location(timeline, addr))
        end
        bp
    end

    function NativeDebugger.breakpoint(timeline::RR.ReplayTimeline, addr)
        bp = NativeDebugger.Breakpoint()
        NativeDebugger.add_location(bp, NativeDebugger.Location(timeline, addr))
        bp
    end

    function NativeDebugger.enable(timeline::RR.ReplayTimeline, loc::Location)
        icxx"$timeline->add_breakpoint(
                $(current_task(current_session(timeline))), $(loc.addr));"
    end

    function NativeDebugger.disable(timeline::RR.ReplayTimeline, loc::Location)
        icxx"$timeline->remove_breakpoint(
                $(current_task(current_session(timeline))), $(loc.addr));"
    end

    function NativeDebugger.print_location(io::IO, vm::RR.ReplayTimeline, loc)
        print(io, "In RR timeline at address ")
        show(io, loc.addr)
        println(io)
    end

    NativeDebugger.getregs(task::AnyTask) = icxx"$task->regs();"
    NativeDebugger.getregs(timeline::ReplayTimeline) =
        NativeDebugger.getregs(current_task(timeline))

    function replay(trace_dir=""; step_to_entry=true)
        session = icxx"""rr::ReplaySession::create($(pointer(trace_dir)));"""
        timeline = icxx"""new rr::ReplayTimeline{std::move($session),rr::ReplaySession::Flags{}};""";
        session = nothing
        icxx"$timeline->maybe_add_reverse_exec_checkpoint(rr::ReplayTimeline::LOW_OVERHEAD);"
        icxx"$(current_session(timeline))->set_visible_execution(true);"
        icxx"""
            rr::ReplaySession::Flags result;
            result.redirect_stdio = true;
            $(current_session(timeline))->set_flags(result);
        """
        RR.step_until_exec!(current_session(timeline))
        task = current_task(current_session(timeline))
        if step_to_entry
          icxx"$timeline->maybe_add_reverse_exec_checkpoint(rr::ReplayTimeline::LOW_OVERHEAD);"
          entrypt = compute_entry_ptr(task,RR.saved_auxv(task))
          icxx"$timeline->add_breakpoint($task, $entrypt);"
          NativeDebugger.step_until_bkpt!(timeline)
          icxx"$timeline->remove_breakpoint($task, $entrypt);"
          icxx"$timeline->maybe_add_reverse_exec_checkpoint(rr::ReplayTimeline::LOW_OVERHEAD);"
        end
        imageh = NativeDebugger.read_exe(current_session(timeline))
        modules = NativeDebugger.GlibcDyldModules.load_library_map(task, imageh)

        timeline, modules
    end

    const preload_thread_locals = 0x70001000
    const stub_scratch_1 = preload_thread_locals
    function unwind_step_rr_extended_jump(sess, base, RC)
        RC′ = copy(RC)
        # First figure out which stub we're in and how far
        # we're into the stub
        arch = NativeDebugger.getarch(sess)
        stub_size = isa(arch, NativeDebugger.X86_64.X86_64Arch) ?
          79 : 44
        ptrT = NativeDebugger.intptr(arch)
        ip = UInt64(NativeDebugger.ip(RC))
        procrel = rem(ip-base, stub_size)
        if procrel != 0
            set_dwarf!(RC′, :rsp, NativeDebugger.load(sess,
              RemotePtr{ptrT,ptrT}(preload_thread_locals+sizeof(ptrT))))
        end
        local return_addr
        if isa(arch, NativeDebugger.X86_64.X86_64Arch)
            lo = NativeDebugger.load(sess, RemotePtr{UInt32, UInt64}(ip-procrel+53))
            hi = NativeDebugger.load(sess, RemotePtr{UInt32, UInt64}(ip-procrel+61))
            return_addr = (UInt64(hi) << 32) | lo
            jump_stub = NativeDebugger.load(sess, RemotePtr{UInt64, UInt64}(ip-procrel+71))
            if NativeDebugger.load(sess, RemotePtr{UInt32, UInt64}(jump_stub+5)) == 0x5e5a5c5a
                # This is the _syscall_hook_trampoline_5a_5e_c3, which doesn't
                # return to where it was called, because the function is too short
                # Just pretend we're before the syscall.
                return_addr -= 5
            end
        else
            return_addr = NativeDebugger.load(sess, RemotePtr{UInt32, UInt32}(ip-procrel+35))
        end
        set_ip!(RC′, return_addr)
        RC′
    end

    function get_synthetic_modules(session)
        modules = Dict{RR.AddressSpaceUid,Dict{NativeDebugger.RemotePtr{Void},NativeDebugger.SyntheticModule}}()
        reader = icxx"rr::TraceReader{$(RR.current_session(session))->trace_reader()};"
        icxx"$reader.rewind();"
        for frame in RR.TraceFrameIterator(reader)
            start = Ref{UInt64}(0)
            size = Ref{UInt64}(0)
            while icxx"""
                bool found = false;
                auto km = $reader.read_mapped_region(nullptr, &found);
                if (found) {
                  $start = km.start().as_int();
                  $size = km.size();
                }
                return found;
                """
                if icxx"$frame.event().type() == rr::EV_PATCH_SYSCALL;"
                    symbolicate = (session, RC)->(true, "RR Syscall Stub")
                    get_proc_bounds = (session, ip)->(0x1:0x1000)-1
                    # XXX: This is incorrect for multi-as replays
                    asid = NativeDebugger.current_asid(session)
                    !haskey(modules, asid) &&
                      (modules[asid] = Dict{UInt64,NativeDebugger.SyntheticModule}())
                    modules[asid][NativeDebugger.RemotePtr{Void}(start[])] =
                      NativeDebugger.SyntheticModule(start[], size[],
                        unwind_step_rr_extended_jump, symbolicate, get_proc_bounds)
                end
            end
        end
        modules
    end

end # module


text:
A,B,C=-1,-2,3

ret=map(x->begin
        if x<0 && iseven(x)
            return 0
        elseif x==0
            return 0
        else
            return x
        end
    end,[A,B,C])

println(ret)

ret=map([A,B,C]) do x
    if x<0 && iseven(x)
        return 0
    elseif x==0
        return 0
    else
        return x
    end
end

println(ret)

max_stars_repo_name: terasakisatoshi/juliaExer

max_issues_repo_name: terasakisatoshi/juliaExer

max_forks_repo_name: terasakisatoshi/juliaExer


text:
# using JuNix
using Test
using Base: SHA1
using SHA
using Pkg.GitTools: tree_hash

function dirs_generator(path)
    (relpath(joinpath(root, dir), path) for (root, dirs, files) in walkdir(path) for dir in dirs)
end

function files_generator(path)
    (relpath(joinpath(root, file), path) for (root, dirs, files) in walkdir(path) for file in files)
end
function verify_tree2(a, b)
    diff = Dict{String,Union{Missing,Tuple{SHA1,SHA1}}}()
    for rpath in files_generator(a)
        apath = joinpath(a, rpath)
        bpath = joinpath(b, rpath)
        ahash = bytes2hex(sha256(read(apath)))
        bhash = bytes2hex(sha256(read(apath)))
        ahash == bhash || (diff[rpath]=(ahash,bhash))
    end
    return diff
end

function verify_tree(a, b)
    diff = Dict{String,Union{Missing,Tuple{SHA1,SHA1}}}()
    for rpath in dirs_generator(a)
        apath = joinpath(a, rpath)
        bpath = joinpath(b, rpath)
        if isdir(bpath)
            ahash = SHA1(tree_hash(apath))
            bhash = SHA1(tree_hash(bpath))
            ahash == bhash || (diff[rpath]=(ahash,bhash))
        else
            diff[rpath]=missing
        end
    end
    return diff
end
#
# @testset "JuNix.jl" begin
#     # Write your tests here.
# end


text:
# This code is part of QuantumCircuits.
#
# (C) Copyright Rafał Pracht 2022.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

module Instructions

using QuantumCircuits.QCircuits.QBase
using QuantumCircuits.QCircuits.Gates
using QuantumCircuits.QCircuits.Gates: ParamT
using QuantumCircuits.QCircuits.Registers
using QuantumCircuits.QCircuits.Math

import Base: show, length, inv

import QuantumCircuits.QCircuits.Gates: getqubits, getqubitsids, appendparams!,
       appendRandParams!, setparameters!, getArgs

import QuantumCircuits.QCircuits.QBase: tomatrix, simplify, standardGateError,
       decompose

export Barrier

"Universary two qubits gate."
struct Barrier <: QuantumGate
    qubits::Vector{Qubit}
end

getArgs(g::Barrier) = ((g.qubits, ), )

"Return the qubits on which operate the gate"
getqubits(::Barrier) = ([getid(qubit) for qubit in g.qubits]..., )

"Return the qubits ids on which operate the gate"
getqubitsids(gate::Barrier) = [getid(qubit) for qubit in gate.qubits]

Base.show(io::IO, ::Barrier) = print(io, "Barrier")

decompose(gate::Barrier) = [gate]

tomatrix(gate::Barrier) = eye(2^length(gate.qubits))

inv(gate::Barrier) = gate

"The simplify the gates."
simplify(gate::Barrier) = [gate]

end  # module Instructions


text:
addprocs(1)

exp=5
print("Testing repeated 10^$exp connect/disconnects....")

(port, server) = open_any_tcp_port(8000)

@spawnat(1,
    begin
        while (true)
            close(accept(server))
        end
    end
    )

@sync begin
    @async begin
        for i in 1:10^exp
            close(connect("localhost", port));
#            println("finished $i")
        end
    end
end

close(server)
println(" : OK")

(port, server) = open_any_tcp_port(8000)

exp=9
size = 10^exp
block = 10^(exp - 4)

print("Testing open, send of 10^$exp bytes and closing. Other side should recv complete amount...")

rr_rcd = RemoteRef()
rr_sent = RemoteRef()

@spawnat(1,
    begin
        while (true)
            s=accept(server)
            bread = 0
            while bread < size
                data = read(s, Uint8, block)
                @assert length(data) == block
                bread += block
            end
#            println("process $(myid()) received $bread bytes")
            close(s)
            put!(rr_rcd, true)
        end
    end)

@spawnat(2,
    begin
        s = connect("localhost", port)
        data = fill!(zeros(Uint8, block), int8(65))
        bwritten = 0
        while bwritten < size
            write(s, data)
            bwritten += block
        end
        close(s)
#        println("process $(myid()) sent $bwritten bytes")
        put!(rr_sent, true)
    end)

wait(rr_rcd)
wait(rr_sent)

close(server)

println(": OK")

require("sockxfer.jl")
xfer_exp = 9
print("Testing 10^$(xfer_exp) bytes of concurrent bidirectional transfers over a single socket connection...")

(port, server) = open_any_tcp_port(8000)

rr_server = RemoteRef()
rr_client = RemoteRef()

@spawnat(1,
    begin
        while (true)
            s=accept(server)
            @async begin xfer(s, xfer_exp); put!(rr_server, true); take!(rr_client); close(s); put!(rr_server, true);   end
        end
    end)

@spawnat(workers()[1], begin s = connect("localhost", port); xfer(s, xfer_exp); put!(rr_client, true); take!(rr_server); close(s); put!(rr_client, true);  end)

take!(rr_server)
take!(rr_client)

close(server)

println(" : OK")

@unix_only include("memtest.jl")

max_stars_repo_name: jakebolewski/julia

max_issues_repo_name: jakebolewski/julia

max_forks_repo_name: jakebolewski/julia


text:
module Brushes

using ..Ahorn, Maple

displayName = "Brushes"
group = "Brushes"

drawingLayers = Ahorn.Layer[]

toolsLayer = nothing
targetLayer = nothing
material = '0'

const brushes = Ahorn.Brush[
    Ahorn.Brush(
        "Pencil",
        hcat(1)
    ),

    Ahorn.Brush(
        "Dither",
        [
            1 0;
            0 1
        ]
    ),
    Ahorn.Brush(
        "Ahorn",
        [
            0 0 1 0 0 0 0;
            0 1 0 1 0 0 0;
            0 0 0 1 1 0 1;
            0 0 1 1 1 0 0;
            0 1 1 1 1 1 0;
            0 1 1 1 1 1 0;
            1 1 1 1 1 1 0;
            1 1 1 1 1 0 0;
            0 1 1 1 0 0 0;
        ]
    )
]

selectedBrush = brushes[1]

hoveringBrush = nothing
const phantomBrushes = Dict{Tuple{Integer, Integer}, Ahorn.Brush}()

lastX, lastY = nothing, nothing

function applyPhantomBrushes()
    if !isempty(phantomBrushes)
        Ahorn.History.addSnapshot!(Ahorn.History.RoomSnapshot("Brush ($(selectedBrush.name), $material)", Ahorn.loadedState.room))
    end

    for (pos, brush) in phantomBrushes
        x, y = pos

        Maple.updateTileSize!(Ahorn.loadedState.room, '0', '0')
        roomTiles = Ahorn.roomTiles(targetLayer, Ahorn.loadedState.room)
        Ahorn.applyBrush!(brush, roomTiles, material, x, y)
    end

    if !isempty(phantomBrushes)
        empty!(phantomBrushes)

        Ahorn.redrawLayer!(targetLayer)
        Ahorn.redrawLayer!(toolsLayer)
    end
end

function drawBrushes(layer::Ahorn.Layer, room::Ahorn.DrawableRoom, camera::Ahorn.Camera)
    if !isa(hoveringBrush, Nothing)
        x, y, brush = hoveringBrush

        Ahorn.drawBrush(brush, layer, x, y)
    end

    # Make sure we should actually render the phantom brushes
    if Ahorn.mouseButtonHeld(0x1)
        for (pos, brush) in phantomBrushes
            x, y = pos

            Ahorn.drawBrush(brush, layer, x, y)
        end
    end
end

function addPhantomBrush(x::Int, y::Int, selectedBrush::Ahorn.Brush)
    if !haskey(phantomBrushes, (x, y))
        phantomBrushes[(x, y)] = deepcopy(selectedBrush)

        return true
    end

    return false
end

function addPhantomBrush(points::Array{Tuple{Int, Int}, 1}, selectedBrush::Ahorn.Brush)
    res = false

    for (x, y) in points
        res |= addPhantomBrush(x, y, selectedBrush)
    end

    return res
end

function addBrushOffset(bx, bw, ox, box, by, bh, oy, boy)
    return div(bx, bw) * bw + ox - box + 1, div(by, bh) * bh + oy - boy + 1
end

function cleanup()
    global hoveringBrush = nothing
    empty!(phantomBrushes)

    Ahorn.redrawLayer!(toolsLayer)
end

function setMaterials!(layer::Ahorn.Layer)
    tileNames = Ahorn.tileNames(layer)
    displayNames = Ahorn.getBrushMaterialsNames(layer, get(Ahorn.config, "sort_brush_tiles_by_name", true))

    Ahorn.setMaterialList!(displayNames, row -> row[1] == get(tileNames, material, nothing))
end

function mouseMotion(x::Number, y::Number)
    if hoveringBrush === nothing || x != hoveringBrush[1] || y != hoveringBrush[2]
        global hoveringBrush = (x, y, deepcopy(selectedBrush))

        Ahorn.redrawLayer!(toolsLayer)
    end
end

function middleClick(x::Number, y::Number)
    tiles = Ahorn.roomTiles(targetLayer, Ahorn.loadedState.room)
    tileNames = Ahorn.tileNames(targetLayer)
    target = get(tiles.data, (y, x), '0')

    global material = target
    layerName = Ahorn.layerName(targetLayer)
    Ahorn.persistence["brushes_material_$(layerName)"] = material
    Ahorn.selectMaterialList!(tileNames[target])
end

function leftClick(x::Number, y::Number)
    layer = Ahorn.layerName(targetLayer)
    Ahorn.History.addSnapshot!(Ahorn.History.RoomSnapshot("Brush $(selectedBrush.name)", Ahorn.loadedState.room))

    Maple.updateTileSize!(Ahorn.loadedState.room, '0', '0')
    roomTiles = Ahorn.roomTiles(targetLayer, Ahorn.loadedState.room)
    Ahorn.applyBrush!(selectedBrush, roomTiles, material, x, y)

    Ahorn.redrawLayer!(targetLayer)
end

function selectionMotion(x1::Number, y1::Number, x2::Number, y2::Number)
    box, boy = selectedBrush.offset

    startX = x1
    startY = y1

    pixels = rotr90(selectedBrush.pixels, selectedBrush.rotation)

    bh, bw = size(pixels)
    ox, oy = mod(startX, bw), mod(startY, bh)

    bx, by = addBrushOffset(x2, bw, ox, box, y2, bw, oy, boy)

    smoothWithLines = get(Ahorn.config, "tools_brushes_smoother_brushes", true)
    redraw = false

    if smoothWithLines && lastX !== nothing && lastY !== nothing
        pointsRaw = Ahorn.pointsOnLine(Ahorn.Line(x2, y2, lastX, lastY))
        points = Tuple{Int, Int}[]

        for (x, y) in pointsRaw
            push!(points, addBrushOffset(x, bw, ox, box, y, bw, oy, boy))
        end

        redraw = addPhantomBrush(points, selectedBrush)

    else
        redraw = addPhantomBrush(bx, by, selectedBrush)
    end

    if redraw
        Ahorn.redrawLayer!(toolsLayer)
    end

    global lastX, lastY = x2, y2
end

function selectionFinish(rect::Ahorn.Rectangle)
    global lastX, lastY = nothing, nothing

    applyPhantomBrushes()
end

function toolSelected(subTools::Ahorn.ListContainer, layers::Ahorn.ListContainer, materials::Ahorn.ListContainer)
    layerName = Ahorn.layerName(targetLayer)
    tileNames = Ahorn.tileNames(targetLayer)
    global material = get(Ahorn.persistence, "brushes_material_$(layerName)", tileNames["Air"])[1]

    wantedBrush = get(Ahorn.persistence, "brushes_brushes_brush", brushes[1].name)
    Ahorn.updateTreeView!(subTools, [brush.name for brush in brushes], row -> row[1] == wantedBrush)

    wantedLayer = get(Ahorn.persistence, "brushes_layer", "fgTiles")
    Ahorn.updateLayerList!(["fgTiles", "bgTiles"], row -> row[1] == wantedLayer)

    Ahorn.redrawingFuncs["tools"] = drawBrushes
    Ahorn.redrawLayer!(toolsLayer)
end

function layerSelected(list::Ahorn.ListContainer, materials::Ahorn.ListContainer, selected::String)
    global targetLayer = Ahorn.getLayerByName(drawingLayers, selected)
    Ahorn.persistence["brushes_layer"] = selected

    tileNames = Ahorn.tileNames(targetLayer)
    layerName = Ahorn.layerName(targetLayer)
    global material = get(Ahorn.persistence, "brushes_material_$(layerName)", tileNames["Air"])[1]
    setMaterials!(targetLayer)
end

function materialSelected(list::Ahorn.ListContainer, selected::String)
    tileNames = Ahorn.tileNames(targetLayer)
    layerName = Ahorn.layerName(targetLayer)
    Ahorn.persistence["brushes_material_$(layerName)"] = tileNames[selected]
    global material = tileNames[selected]
end

function materialFiltered(list::Ahorn.ListContainer)
    tileNames = Ahorn.tileNames(targetLayer)
    Ahorn.selectRow!(list, row -> row[1] == get(tileNames, material, '0'))
end

function getFavorites()
    if targetLayer !== nothing
        key = "favorites_brushes_$(targetLayer.name)"

        return Ahorn.getFavorites(Ahorn.persistence, key)
    end

    return []
end

function materialDoubleClicked(material::String)
    if targetLayer !== nothing
        key = "favorites_brushes_$(targetLayer.name)"
        tileNames = Ahorn.tileNames(targetLayer)

        Ahorn.toggleFavorite(Ahorn.persistence, key, material)
        setMaterials!(targetLayer)
        Ahorn.updateMaterialFilter!(targetLayer.name)
    end
end

function layersChanged(layers::Array{Ahorn.Layer, 1})
    wantedLayer = get(Ahorn.persistence, "brushes_layer", "fgTiles")

    global drawingLayers = layers
    global toolsLayer = Ahorn.getLayerByName(layers, "tools")
    global targetLayer = Ahorn.selectLayer!(layers, wantedLayer, "fgTiles")
end

function subToolSelected(list::Ahorn.ListContainer, selected::String)
    for brush in brushes
        if brush.name == selected
            Ahorn.persistence["brushes_brushes_brush"] = selected
            global selectedBrush = brush
        end
    end
end

function keyboard(event::Ahorn.eventKey)
    shouldRedraw = false

    if event.keyval == Ahorn.keyval("l")
        selectedBrush.rotation = mod(selectedBrush.rotation - 1, 4)

        shouldRedraw |= true

    elseif event.keyval == Ahorn.keyval("r")
        selectedBrush.rotation = mod(selectedBrush.rotation + 1, 4)

        shouldRedraw |= true
    end

    if shouldRedraw && hoveringBrush !== nothing
        global hoveringBrush = (hoveringBrush[1], hoveringBrush[2], deepcopy(selectedBrush))

        Ahorn.redrawLayer!(toolsLayer)
    end
end

end

max_stars_repo_name: CelestialCartographers/Ahorn

max_issues_repo_name: CelestialCartographers/Ahorn

max_forks_repo_name: CelestialCartographers/Ahorn


text:
@doc """
    @> expr

Enhances [`|>`](@ref) operation, and allows more flexible function chaining.

!!! note
    `@>` macro is _almost_ compatible with the original [`|>`](@ref) operation,
    except that chaining arguments by wrapping in an anonymous function like
    `val |> x -> f(x, val)` **should be avoided**.

See also: [`|>`](@ref), [`@>>`](@ref)

# Example

- compatible with usual [`|>`](@ref) operations

```julia
julia> λ(val) == val |> λ == @> val |> λ
```

- given function call, injects chained value into the first argument (no need to write `val |> x -> λ(x, val2)`)

```julia
julia> val |> x -> λ(x) == @> val |> λ()
julia> val |> x -> λ(x, val2) == @> val |> λ(val2)
julia> val |> x -> λ(x, val2) |> x -> λ2(x, val3) == @> val |> λ(val2) |> λ2(val3)
```

- using tuple, the argument position can be specified

```julia
julia> val |> x -> λ(x) == @> val |> (1, λ())
julia> val |> x -> λ(val2, x) == @> val |> (2, λ(val2))
julia> val |> x -> λ(val2, x) |> x -> λ2(x, val3) == @> val |> (2, λ(val2)) |> λ2(val3)
```

- keyword argument should be specified with symbol

```julia
julia> val |> x -> λ(val2; keyword = x) == @> val |> (:keyword, λ(val2))
julia> val |> x -> λ(val2; keyword = x) |> x -> λ2(val3; keyword2 = x) == @> val |> (:keyword, λ(val2)) |> (:keyword2, λ2(val3))
```

- [dot-fusing](https://docs.julialang.org/en/v1/manual/functions/#man-vectorized-1) is fully supported

```julia
julia> (ary .|> string .|> x -> λ(x, val)) == @> ary .|> string .|> λ(val)
```
"""
macro >(expr)
    processexpr!(expr)
    :( $(esc(expr)) )
end

@doc """
    @>> exprs...

Allows [`@>`](@ref) chaining without [`|>`](@ref) operators.

See also: [`@>`](@ref)

# Example

!!! note
    Equivalent to the examples of [`@>`](@ref)

```julia
julia> λ(val) == val |> λ == @>> val λ
```

```julia
julia> val |> x -> λ(x) == @>> val λ()
julia> val |> x -> λ(x, val2) == @>> val λ(val2)
julia> val |> x -> λ(x, val2) |> x -> λ2(x, val3) == @>> val λ(val2) λ2(val3)
```

```julia
julia> val |> x -> λ(x) == @>> val (1, λ())
julia> val |> x -> λ(val2, x) == @>> val (2, λ(val2))
julia> val |> x -> λ(val2, x) |> x -> λ2(x, val3) == @>> val (2, λ(val2)) λ2(val3)
```

```julia
julia> val |> x -> λ(val2; keyword = x) == @>> val (:keyword, λ(val2))
julia> val |> x -> λ(val2; keyword = x) |> x -> λ2(val3; keyword2 = x) == @>> val (:keyword, λ(val2)) (:keyword2, λ2(val3))
```
"""
macro >>(exprs...)
    expr = concatexpr(exprs)
    :( $(esc(expr)) )
end

@doc """
    @.>> exprs...

Works almost same as [`@>>`](@ref) except that implicit [`|>`](@ref) is all
[dot-fused](https://docs.julialang.org/en/v1/manual/functions/#man-vectorized-1).

See also: [`@>`](@ref), [`@>>`](@ref)

# Example

!!! note
    Equivalent to the last example of [`@>`](@ref)

```julia
julia> (ary .|> string .|> x -> λ(val, x)) == @.>> ary string (2, λ(val))
```
"""
macro .>>(exprs...)
    expr = concatexpr(exprs, true)
    :( $(esc(expr)) )
end


text:
@safetestset "isbase32" begin
    using Validator

    @test Validator.isbase32("ZG======") === true
    @test Validator.isbase32("JBSQ====") === true
    @test Validator.isbase32("JBSWY===") === true
    @test Validator.isbase32("JBSWY3A=") === true
    @test Validator.isbase32("JBSWY3DP") === true
    @test Validator.isbase32("JBSWY3DPEA======") === true
    @test Validator.isbase32("K5SWYY3PNVSSA5DPEBXG6ZA=") === true
    @test Validator.isbase32("K5SWYY3PNVSSA5DPEBXG6===") === true

    @test Validator.isbase32("12345") === false
    @test Validator.isbase32("") === false
    @test Validator.isbase32("JBSWY3DPtesting123") === false
    @test Validator.isbase32("ZG=====") === false
    @test Validator.isbase32("Z======") === false
    @test Validator.isbase32("Zm=8JBSWY3DP") === false
    @test Validator.isbase32("=m9vYg=") === false
    @test Validator.isbase32("Zm9vYm/y====") === false
end


text:
module Unicycle

using NeuralNetworkAnalysis, LaTeXStrings
using NeuralNetworkAnalysis: UniformAdditivePostprocessing
import Plots, DifferentialEquations
using Plots: plot, plot!, scatter!, lens!, bbox, savefig, font, Measures.mm

# problem

@taylorize function unicycle!(dx, x, p, t)
    x₁, x₂, x₃, x₄, w, u₁, u₂ = x

    dx[1] = x₄ * cos(x₃)
    dx[2] = x₄ * sin(x₃)
    dx[3] = u₂
    dx[4] = u₁ + w
    dx[5] = zero(x[5])
    dx[6] = zero(x[6])
    dx[7] = zero(x[7])
    return dx
end

controller = read_nnet_sherlock(@modelpath "" "controller_Unicycle_sherlock")

X₀ = Hyperrectangle(low=[ 9.5,  -4.5,  2.1,  1.5, -1e-4],
                    high=[9.55, -4.45, 2.11, 1.51, 1e-4])
U₀ = ZeroSet(2)
vars_idx = Dict(:state_vars=>1:4, :input_vars=>[5], :control_vars=>6:7)
ivp = @ivp(x' = unicycle!(x), dim: 7, x(0) ∈ X₀ × U₀)

control_postprocessing = UniformAdditivePostprocessing(-20.0)

period = 0.2
T = 10.0
T_warmup = 2 * period

prob = ControlledPlant(ivp, controller, vars_idx, period;
                       postprocessing=control_postprocessing)

target_states = cartesian_product(Hyperrectangle(zeros(4), [0.6, 0.2, 0.06, 0.3]),
                                  Universe(3))

predicate_sol_suff = sol -> sol[end][end] ⊆ target_states

# simulation

println("simulation")

res = @timed simulate(prob, T=T; trajectories=1, include_vertices=false)
sim1 = res.value
print_timed(res)

res = @timed simulate(prob, T=T; trajectories=10, include_vertices=true)
sim = res.value
print_timed(res)

# reachability analysis

alg = TMJets(abstol=1e-15, orderT=10, orderQ=1)
alg_nn = Ai2()
splitter = BoxSplitter([3, 1, 8, 1])

function benchmark(; T=T, silent::Bool=false)
    silent || println("flowpipe construction")
    res = @timed solve(prob, T=T, alg_nn=alg_nn, alg=alg,
                                 splitter=splitter)
    sol = res.value
    silent || print_timed(res)

    silent || println("property checking")
    res = @timed predicate_sol_suff(sol)
    silent || print_timed(res)
    if res.value
        silent || println("The property is satisfied.")
    else
        silent || println("The property may be violated.")
    end
    return sol
end

benchmark(T=T_warmup, silent=true)
res = @timed benchmark()
sol = res.value
println("total analysis time")
print_timed(res)

# plotting

function plot_sim!(p, sim, vars)
    plot_simulation!(p, sim; vars=vars, color=:black, lab="")
end

function plot_sol!(p, sol, vars)
    plot!(p, sol, vars=vars, c=:orange, alpha=0.1)
    plot!(p, sol[end][end], vars=vars, c=:red, alpha=0.3)
end

function plot_zoom!(p, vars)
    if vars == [1, 2]
        lens!(p, [9.45, 9.6], [-4.55, -4.4], inset=(1, bbox(0.7, 0.2, 0.3, 0.25)), xticks=[9.5, 9.55], yticks=[-4.5, -4.45], subplot=2)
        lens!(p, [0.3, 0.7], [-0.3, 0.3], inset=(1, bbox(0.1, 0.3, 0.3, 0.3)), xticks=[0.4, 0.6], subplot=3)
    elseif vars == [3, 4]
        plot!(xlims=[-0.2, 2.7], ylims=[-1.5, 3], leg=:topleft)
        lens!(p, [2.095, 2.12], [1.49, 1.52], inset=(1, bbox(0.4, 0.1, 0.3, 0.3)), xticks=[2.10, 2.11, 2.12], subplot=2)
        lens!(p, [-0.1, 0.1], [-0.5, -0.1], inset=(1, bbox(0.6, 0.6, 0.3, 0.3)), xticks=[-0.1, 0, 0.1], subplot=3)
    end
end

function plot_X0!(p, vars; lab=L"X_0")
    plot!(p, project(X₀, vars), c=:blue, lab=lab, alpha=0.6)
end

function label(var)
    var == 1 && return L"x"
    var == 2 && return L"y"
    var == 3 && return L"\theta"
    var == 4 && return L"v"
    throw(ArgumentError(var))
end

function plot_all(sol, sim, vars)
    p = plot(xlab=label(vars[1]), ylab=label(vars[2]),
             tickfont=font(15, "Times"),
             guidefontsize=15,
             xguidefont=font(15, "Times"),
             yguidefont=font(15, "Times"),
             bottom_margin=1mm, left_margin=0mm, right_margin=0mm, top_margin=0mm)
    plot_X0!(p, vars)
    plot!(p, project(target_states, vars), c=:green, lab=L"X_T")

    !isnothing(sol) && plot_sol!(p, sol, vars)
    !isnothing(sol) && plot_X0!(p, vars, lab="")
    plot_sim!(p, sim, vars)
    plot_zoom!(p, vars)
    return p
end

plot_all(nothing, sim, [1, 2])
savefig("unicycle_sim_x1x2.png")

plot_all(nothing, sim, [3, 4])
savefig("unicycle_sim_x3x4.png")

plot_all(sol, sim1, [1, 2])
savefig("unicycle_reach_x1x2.png")

plot_all(sol, sim1, [3, 4])
savefig("unicycle_reach_x3x4.png")

tdom = range(0, T - period, step=period)
fig = plot(xlab=L"t", ylab=L"u_1(t) / u_2(t)",
           leg=:topleft,
           legendfontsize=5,
           tickfont=font(15, "Times"),
           guidefontsize=15,
           xguidefont=font(15, "Times"),
           yguidefont=font(15, "Times"),
           bottom_margin=2mm, left_margin=0mm, right_margin=5mm, top_margin=1mm)
[plot!(fig, tdom, [c[1] for c in controls(sim, i)], lab=(i == 1 ? L"u_1" : ""), c=:red, seriestype=:steppre) for i in 1:length(sim)]
[plot!(fig, tdom, [c[2] for c in controls(sim, i)], lab=(i == 1 ? L"u_2" : ""), c=:blue, seriestype=:steppre) for i in 1:length(sim)]
plot!()
savefig("unicycle_controls.png")

end
nothing


text:
abstract type AbstractTimeIntegrator end

function integrate!(plasma, field::AbstractField, integrator::AbstractTimeIntegrator, dt=nothing)
  return integrator(plasma, field, dt)
end

struct LeapFropTimeIntegrator <: AbstractTimeIntegrator
  dt::Float64
end

function LeapFrogTimeIntegrator(p::Plasma, f::AbstractField; cflmultiplier=1)
  return LeapFropTimeIntegrator(cellsize(f) / maxspeed(p) * cflmultiplier)
end

timestep(ti::LeapFropTimeIntegrator) = ti.dt

function (ti::LeapFropTimeIntegrator)(plasma, field::AbstractField{BC}, dt=nothing) where {BC}
  isnothing(dt) && (dt = timestep(ti))

  bc = BC(0.0, domainsize(field))

  pushposition!(plasma, dt / 2, bc)
  zero!(field)
  deposit!(field, plasma)
  solve!(field)
  pushvelocity!(plasma, field, dt)
  pushposition!(plasma, dt / 2, bc)
  return nothing
end

@concrete struct SemiImplicit2ndOrderTimeIntegrator <: AbstractTimeIntegrator
  dt::Float64
  atol::Float64
  rtol::Float64
  maxiters::Int64
  fieldcopy
end

function SemiImplicit2ndOrderTimeIntegrator(p::Plasma, f::AbstractField; maxiters=10, atol=0, rtol=sqrt(eps()))
  return SemiImplicit2ndOrderTimeIntegrator(cellsize(f) / maxspeed(p), atol, rtol, maxiters, deepcopy(p))
end

timestep(ti::SemiImplicit2ndOrderTimeIntegrator) = ti.dt

function (ti::SemiImplicit2ndOrderTimeIntegrator)(plasma, field, dt=nothing)
  isnothing(dt) && (dt = timestep(ti))
  bc = BC(0.0, domainsize(field))
  copy!(ti.fieldcopy, field)
  firstiteration = true
  while firstiteration || iters < ti.maxiters
    firstiteration = false
    zero!(field.chargedensity)
    for (i,s) ∈ enumerate(plasma.species)
      for (j, particle) ∈ enumerate(s.particles)
        p₀ = deepcopy(particle)
        copy!(particle, p₀)
        pushposition!(particle, dt/2, bc) # push half with starting velocity
        pushvelocity!(particle, field, dt) # accelerate with middle velocity
        pushposition!(particle, dt/2, bc) # now push other half timestep with end timestep velocity
        deposit!(field, (p₀ + particle) / 2)
      end
    end
    solve!(field) # solve for the mid point electric field
    if isapprox(field, fieldcopy, atol=ti.atol, rtol=ti.rtol)
      pushposition!(plasma, dt / 2, bc)
      pushvelocity!(plasma, field, dt)
      pushposition!(plasma, dt / 2, bc)
      break
    end
    copy!(ti.fieldcopy, field)
  end
  return nothing
end


text:
using Documenter, Clustering

makedocs(
    source = "source",
    format = Documenter.HTML(prettyurls=false),
    sitename = "Clustering.jl",
    modules = [Clustering],
    pages = [
        "Clustering.jl Docs" => "index.md",
        "Algorithms" => [
            "algorithms.md",
            "init.md",
            "kmeans.md",
            "kmedoids.md",
            "hclust.md",
            "mcl.md",
            "affprop.md",
            "dbscan.md",
            "fuzzycmeans.md",
        ],
        "Evaluation & Validation" => [
            "validate.md",
            "silhouette.md",
            "randindex.md",
            "varinfo.md",
            "vmeasure.md",
        ]
    ],
    debug = false,
)

deploydocs(
    repo = "github.com/alyst/Clustering.jl.git",
)


text:
using FANN

include("test-data.jl")
include("test-linear.jl")
include("test-xor.jl")
include("test-io.jl")
include("test-trainalg.jl")
include("test-getset-weights.jl")


text:
using BattleSim

white = Knight("Lancelot", 34)
black = Knight("Moriaen", 36)
green = Archer("Robin Hood", 28)

shoot!(green)
shoot!(white)    # knights cannot shoot arrows
dismount!(green) # archers cannot dismount, as they have no horses
dismount!(black)

max_stars_repo_name: ordovician/BattleSim

max_issues_repo_name: ordovician/BattleSim

max_forks_repo_name: ordovician/BattleSim


text:
using Test, SparseVolterraExamples

include("test_JQEJ-operators.jl")
include("test_VolterraEqns-FullKernel.jl")
include("test_VolterraEqns-MonKernel.jl")
include("test_VolterraIntegral-FullKernel.jl")
include("test_VolterraIntegral-MonKernel.jl")


text:
module Metrics

using util
using DataStructures
using PyCall

export createMAP_RecallRate, createBinaryPredictionROC, reset, update_metric!, compute, MAP_RecallRate, BinaryPredictionROC

abstract type Metric end

mutable struct MAP_RecallRate <: Metric
    label::String
    k::Int
    sum_map::Float64
    hits_per_k::Dict{Int, Float64}
    n_duplicates::Int
    group_by_master::Bool
end

function createMAP_RecallRate(;label="MAP_RecallRate", k=20, group_by_master=true)
    return MAP_RecallRate(label, k, 0.0, Dict{Int, Float64}([ (i, 0.0) for i in 1:k]), 0, group_by_master)
end

function reset(metric::MAP_RecallRate)
    metric.hits_per_k = Dict{Int, Float64}([ (i, 0.0) for i in 1:k])
    metric.n_duplicates = 0
    metric.sum_map = 0.0
end

function update_metric!(metric::MAP_RecallRate, query::Report, recommendation_list::Vector{Tuple{Report, Float64}})
    if query.dup_id == query.id
        #println("Simgleton: $(query.id) $([ (r.id, s) for (r, s)  in recommendation_list[1:20]])")
        return
    end

    buckets = Set()
    pos = Inf
    correct_cand = Int64(-1)

    for (cand, score) in recommendation_list
        if metric.group_by_master
            query.dup_id in buckets && continue
            push!(buckets, cand.dup_id)
        else
            push!(buckets, cand.id)
        end

        if cand.dup_id == query.dup_id
            pos = Float64(length(buckets))

            # println("$(query.id) $(cand.id) $(pos) $([(r.id, s) for (r, s)  in recommendation_list[1:20]])")
            # @debug
            # print("{}, {}, {}, {}, {}".format(report_id, pos, cand_id, self.master_id_by_bug_id[report_id], list(
            #     zip(candidates[:30], map(lambda x: self.master_id_by_bug_id[x], candidates[:30]), scores[:30]))))
            correct_cand = cand.id
            break
        end
    end

    for k in 1:metric.k
        k < pos && continue

        metric.hits_per_k[k]+=1.0
    end

    metric.sum_map += 1.0/pos
    metric.n_duplicates += 1

    return pos
end

function compute(metric::MAP_RecallRate)
    recall_rate = Dict{Int,Float64}()

    for k in 1:metric.k
        recall_rate[k] = metric.hits_per_k[k] / metric.n_duplicates
    end

    return Dict("type" => "metric",
            "label" => metric.label,
            "hits_per_k" => metric.hits_per_k,
            "rr" => recall_rate,
            "sum_map" => metric.sum_map,
            "map"=> metric.sum_map / metric.n_duplicates,
            "total" => metric.n_duplicates,
            )
end

mutable struct BinaryPredictionROC <: Metric
    label::String
    queries::MutableLinkedList{Report}
    scores::MutableLinkedList{Float64}
end

function  createBinaryPredictionROC(;label="BinaryPredictionROC")
    return BinaryPredictionROC(label, MutableLinkedList{Report}(), MutableLinkedList{Float64}())
end

function reset!(metric::BinaryPredictionROC)
    metric.queries = MutableLinkedList{Report}()
    metric.scores = MutableLinkedList{Float64}()
end

function update_metric!(metric::BinaryPredictionROC, query::Report, recommendation_list::Vector{Tuple{Report, Float64}})
    top_candidate = recommendation_list[1][1]
    top_score = recommendation_list[1][2]

    push!(metric.queries, query)
    push!(metric.scores, top_score)

        # print("{} {} {} {}".format(query_id, top_candidate, top_score, self.master_id_by_bug_id[query_id] != query_id))

    # print("{}, {}, {}, {}, {}, {}, {}".format(
    #     self.master_id_by_bug_id[query_id] == query_id, query_id,
    #     self.master_id_by_bug_id[query_id], top_candidate, self.master_id_by_bug_id[top_candidate], top_score,
    #     [(k, s) for k, s in zip(candidates[:20], scores[:20])]))

end

function compute(metric::BinaryPredictionROC)
    SKLEARN_METRICS = pyimport("sklearn.metrics")
    y_true = [Int(query.dup_id != query.id) for query in metric.queries]
    fpr, tpr, thresholds = SKLEARN_METRICS.roc_curve(y_true, collect(metric.scores), pos_label=1)

    return Dict("type" => "metric",
                "label" => metric.label,
                "fpr" => fpr,
                "tpr" => tpr,
                "auc" => SKLEARN_METRICS.auc(fpr,tpr),
                "n_queries" =>  length(metric.queries),
                "threshold" => thresholds,
    )

end

end

max_stars_repo_name: irving-muller/FaST

max_issues_repo_name: irving-muller/FaST

max_forks_repo_name: irving-muller/FaST


text:
# xlib integration for interacting with the clipboard

if dlopen_e("libX11") == C_NULL
    error("X11 not found (required for XClipboard.jl)")
end

import DataStructures.OrderedDict

export clipboard

typealias Atom Culong
typealias Time Culong
typealias X11_Bool Cint
typealias XID Culong

typealias Display Ptr{Void}
typealias _XEvent Ptr{Void}
typealias Window XID

# Useful constants

const COPYBUF_MAX_SZ = 128*1024 # 128K buffer for copying data

const CurrentTime = zero(Clong)
const Success =     int32(0)
const BadRequst =   int32(1)
const BadValue =    int32(2)
const BadWindow =   int32(3)
const BadAtom =     int32(5)

const NoneAtom =    int32(0)

const SelectionNotify = int32(31)
const AnyPropertyType = zero(Atom)

# Event types

abstract XEvent

type XAnyEvent <: XEvent
    _type::Cint
    serial::Culong
    send_event::X11_Bool
    disp::Display
    window::Window

    XAnyEvent(xevent::IOBuffer) = fillXEvent(new(), xevent)
end

type XSelectionEvent <: XEvent
    _type::Cint
    serial::Culong
    send_event::X11_Bool
    disp::Display
    window::Window

    selection::Atom
    target::Atom
    property::Atom
    time::Time

    XSelectionEvent(xevent::IOBuffer) = fillXEvent(new(), xevent)
end

# Fill in type values from
function fillXEvent{T<:XEvent}(x::T, xevent::IOBuffer)
    seekstart(xevent)
    p = 0
    for (n,t) in zip(T.names, T.types)
        if t == Ptr{Void}
            x.(n) = pointer(Void, read(xevent, Uint64))
            p += 8
        else
            x.(n) = read(xevent, t)
            p += sizeof(t)
        end
        if p%8 != 0
            read(xevent, Array(Uint8, 8-p%8))
            p += (8-p%8)
        end
    end
    x
end

## Function wrappers
XOpenDisplay(name::String = "")            = ccall((:XOpenDisplay,   :libX11), Display, (Ptr{Void},),    name == "" ? C_NULL : name)
DefaultScreen(disp::Display)               = ccall((:XDefaultScreen, :libX11), Cint,    (Display,),      disp)
RootWindow(disp::Display, screen::Integer) = ccall((:XRootWindow,    :libX11), Window,  (Display, Cint), disp, screen)

XInternAtom(disp::Display,
            name::ASCIIString,
            only_if_exists::Bool = false) =
    ccall((:XInternAtom, :libX11), Atom, (Display, Ptr{Cuchar}, X11_Bool), disp, name, only_if_exists)

XCreateSimpleWindow(disp::Display,
                    parent::Window,
                    x::Integer,
                    y::Integer,
                    width::Integer,
                    height::Integer,
                    border_width::Integer,
                    border::Uint64,
                    background::Uint64) =
    ccall((:XCreateSimpleWindow, :libX11), Window,
          (Display, Window, Cint, Cint, Cuint, Cuint,  Cuint,        Culong, Culong),
           disp,    parent, x,    y,    width, height, border_width, border, background)

BlackPixel(disp::Display, screen::Integer) = ccall((:XBlackPixel, :libX11), Culong, (Display, Cint), disp, screen)

XConvertSelection(display::Display,
                  selection::Atom,
                  target::Atom,
                  property::Atom,
                  requestor::Window,
                  time::Integer) =
    ccall((:XConvertSelection, :libX11), Cint,
          (Display, Atom,      Atom,   Atom,     Window,    Time),
           disp,    selection, target, property, requestor, time)

XFlush(disp::Display) = ccall((:XFlush, :libX11), Cint, (Display,), disp)

# Buffer to pass to XLib
const xevent = IOBuffer(96)

function XNextEvent(disp::Display, xevent::IOBuffer = xevent)
    rv = ccall((:XNextEvent, :libX11), Cint, (Display, _XEvent), disp, xevent.data)

    xevent.size = 96 # this is a lie...
    seekstart(xevent)
    _type = read(xevent, Cint)

    _type == SelectionNotify ? XSelectionEvent(xevent) :
                               XAnyEvent(xevent)

end

XFree(p) = ccall((:XFree, :libX11), Cint, (Ptr{Void},), p)

XPending(d::Display) = ccall((:XPending, :libX11), Cint, (Display,), d)

function XGetAtomName(disp::Display, a::Atom)
    if a == NoneAtom
        return "None"
    end
    p = ccall((:XGetAtomName, :libX11), Ptr{Cuchar}, (Display, Atom), disp, a)
    if p == C_NULL
        error("XGetAtomName: BadAtom (Atom does not exist)")
    end
    rv = bytestring(p)
    XFree(p)
    return rv
end

XGetAtomName(a::Atom) = XGetAtomName(disp, a)

XConnectionNumber(display::Display) =
   ccall((:XConnectionNumber, :libX11), Cint, (Display,), display)

#

type _Property
    _type           ::Atom
    format          ::Cint
    nitems          ::Culong
    bytes_remaining ::Culong
    data            ::Ptr{Cuchar}
end

function XGetWindowProperty(display     ::Display,
                            w           ::Window,
                            property    ::Atom,
                            long_offset ::Integer,
                            long_length ::Integer,
                            delete      ::Bool,
                            req_type    ::Atom)
    actual_type     = Array(Atom, 1)
    actual_format   = Array(Cint, 1)       # = size in bits of one item of actual_type (8, 16, or 32)
    nitems          = Array(Culong, 1)
    bytes_remaining = Array(Culong, 1)
    prop            = Array(Ptr{Cuchar}, 1)

    rv = ccall((:XGetWindowProperty, :libX11), Cint,
               (Display    , Window       , Atom       , Clong          , Clong      , X11_Bool, Atom    ,
                Ptr{Atom}  , Ptr{Cint}    , Ptr{Culong}, Ptr{Culong}    , Ptr{Ptr{Cuchar}}                ),

                display    , w            , property   , long_offset    , long_length, delete  , req_type,
                actual_type, actual_format, nitems,      bytes_remaining, prop                             )

    if rv != Success
        rv == BadAtom   ? error("XGetWindowProperty: BadAtom") :
        rv == BadValue  ? error("XGetWindowProperty: BadValue") :
        rv == BadWindow ? error("XGetWindowProperty: BadWindow") :
        error("XGetWindowProperty: Unknown error ($rv)")
    end

    return _Property(actual_type[1], actual_format[1], nitems[1], bytes_remaining[1], prop[1])
end

type Property{T}
    _type           ::Atom
    data            ::Array{T}
end

function GetWindowProperty(display     ::Display,
                           w           ::Window,
                           property    ::Atom,
                           delete      ::Bool = false,
                           req_type    ::Atom = AnyPropertyType)
    # query size

    prop = XGetWindowProperty(display, w, property, 0, 0, delete, req_type)

    if prop._type == NoneAtom
        @assert prop.bytes_remaining == 0
        @assert prop._type == 0
        error("Requested atom ($(XGetAtomName(disp, req_type)) does not exist")
    end

    # This is actually rather screwy.

    # On 64-bit systems (at least), some of the return values are actually 64-bits
    # However, XLib doesn't actually know anything about 64-bit values,
    # but instead returns the number of bytes _as_if_ the return values were 32-bits
    # (Really?!)

    # Just to be sure, we use prop.format, which is the reported return value size in bits,
    # and scale by sizeof(Culong)/(prop.format/8), which should be 1 on a 32-bit system
    # and 2 on a 64-bit system

    # This is at least true for XA_ATOM and XA_TARGETS.
    # If there are other property types which have the same characteristics, they
    # they should be added here.

    # See also: http://tronche.com/gui/x/xlib/window-information/XGetWindowProperty.html

    if prop._type == XA_ATOM || prop._type == XA_TARGETS
        sz = div(prop.bytes_remaining,(prop.format>>3))*sizeof(Atom)
    else
        sz = prop.bytes_remaining # size in bytes of items read
    end

    # Do the actual read
    # Get all data in one shot

    # TODO: this might cause problems very very big copy buffers...
    # TODO: it might be better to set this up as a feed to an IOBuffer

    # Here, the request size is set as the number of 32-bit items to retrieve.
    # (Why, Xlib, do you make our life difficult in this way????)

    prop = XGetWindowProperty(display, w, property, 0, sz>>2+1, delete, req_type)

    @assert prop.bytes_remaining == 0

    # Copy the data

    buf = Array(Uint8, sz)
    unsafe_copy!(pointer(buf), prop.data, sz)
    XFree(prop.data)

    return Property(prop._type, buf)

    # # Should we convert the buffer here or somewhere else

    # if prop.format == 8
    #     return Property(prop._type, buf)
    # elseif prop.format == 16
    #     return Property(prop._type, reinterpret(Uint16, buf))
    # elseif prop.format == 32
    #     return Property(prop._type, reinterpret(Uint32, buf))
    # #elseif prop.format == 64            # This doesn't exist yet...
    #     #return Property(prop._type, reinterpret(Uint64, buf))
    # else
    #     error("Unknown type size: $prop.format")
    # end
end

# Initialization
const disp = XOpenDisplay()
const screen = DefaultScreen(disp)
const root = RootWindow(disp, screen)

# We need a window to exist, but it doesn't have to be visible (mapped)
const w = XCreateSimpleWindow(disp, root, 0, 0, 100, 100, 0, BlackPixel(disp, screen), BlackPixel(disp, screen))

# clipboard related atoms
const sel        = XInternAtom(disp, "CLIPBOARD", false)
const XA_TARGETS = XInternAtom(disp, "TARGETS", false)
const XA_ATOM    = convert(Atom,  4) # Xatom.h
const XA_STRING  = convert(Atom, 31) # Xatom.h
const XA_UTF8_STRING = 0x100|XA_STRING

#
function pick_target_from_list(disp::Display, atom_list::Vector{Atom}, datatypes::Array)

    atom_names  = OrderedDict{String,Atom}()
    for atom in atom_list
        atom_names[XGetAtomName(disp, atom)] = atom
    end

    for d in datatypes
        if (atom = get(atom_names, d, 0)) > 0
            return atom
        end
    end
    return NoneAtom
end

function pick_target_from_targets(disp::Display, p::Property, datatypes::Array)
    # work around broken implementations

    if((p._type != XA_ATOM && p._type != XA_TARGETS) || sizeof(T) != 4)
        if "UTF8_STRING" in datatypes
            return XA_UTF8_STRING
        elseif "STRING" in datatypes
            return XA_STRING
        else
            return NoneAtom
        end
    end

    return pick_target_from_list(disp, reinterpret(Atom, p.data), datatypes)
end

function xclipboard_request(request_atom::Atom, timeout::Int=2)

    XConvertSelection(disp, sel, request_atom, sel, w, CurrentTime)
    XFlush(disp)

    x11_fd = XConnectionNumber(disp)

    while true
        # Are any events pending?
        if XPending(disp) == 0
            # poll Xlib with a 2 second timeout)
            if poll_fd(RawFD(x11_fd), timeout, readable=true) == 0
                return Uint8[]
            end
        end

        e = XNextEvent(disp)

        e._type != SelectionNotify && continue
        e.property == NoneAtom && return Uint8[]
        e.target != request_atom && continue

        prop = GetWindowProperty(disp, w, sel)

        return prop.data
    end
end


text:
using DataFrames, CSV
using Statistics
using BlackBoxOptim

using Infiltrator
using ModelParameters

include("./network_creation.jl")

climate_data = DataFrame!(CSV.File("../tests/data/climate/climate_historic.csv",
                          comment="#",
                          dateformat="YYYY-mm-dd"))

hist_dam_levels = DataFrame!(CSV.File("../tests/data/dam/historic_levels_for_fit.csv", dateformat="YYYY-mm-dd"))
hist_dam_releases = DataFrame!(CSV.File("../tests/data/dam/historic_releases.csv", dateformat="YYYY-mm-dd"))

# Subset to same range
last_date = hist_dam_levels.Date[end]
climate_data = climate_data[climate_data.Date .<= last_date, :]
hist_dam_releases = hist_dam_releases[hist_dam_releases.Date .<= last_date, :]

climate = Climate(climate_data, "_rain", "_evap")

function obj_func(params)
    global climate
    global hist_dam_levels
    global hist_dam_releases
    global mg
    global g
    global v_id

    node = get_prop(mg, v_id, :node)
    node = deepcopy(node)
    update_params!(node, params)  # node = ModelParameters.update(node, params)
    set_prop!(mg, v_id, :node, node)

    timesteps = sim_length(climate)
    for ts in (1:timesteps)
        run_node!(mg, g, v_id, climate, ts; water_order=hist_dam_releases)
    end

    # Calculate score (NSE)
    hist_levels = hist_dam_levels[:, "Dam Level [mAHD]"]
    score = -(1.0 - sum((node.level .- hist_levels).^2) / sum((hist_levels .- mean(hist_levels)).^2))

    # reset to clear stored values
    # reset!(get_prop(mg, v_id, :node))

    return score
end

function calibrate(g, mg, v_id)

    inlets = inneighbors(g, v_id)
    if !isempty(inlets)
        for ins in inlets
            calibrate(g, mg, ins)
        end
    end

    @infiltrate

    # Create new optimization function
    opt_func = x -> obj_func(x, climate, mg, g, v_id, discharge, levels)

    target_node = Model(get_prop(mg, v_id, :node))
    score = bboptimize(opt_func; SearchRange=collect(target_node.bounds))

    return score
end

match = collect(filter_vertices(mg, :name, "406000"))
v_id = match[1]
# target_node = Model(get_prop(mg, v_id, :node))

@info calibrate(g, mg, v_id)

max_stars_repo_name: ConnectedSystems/camgraph

max_issues_repo_name: ConnectedSystems/camgraph

max_forks_repo_name: ConnectedSystems/camgraph


text:
import CompatHelper
import Test

import Base64
import Dates
import GitHub
import HTTP
import JSON
import Pkg
import Printf
import Random
import TimeZones

const registries_1 = Pkg.RegistrySpec[Pkg.RegistrySpec(name = "General",
                                                       uuid = "23338594-aafe-5451-b93e-139f81909106",
                                                       url = "https://github.com/JuliaRegistries/General.git")]

const registries_2 = Pkg.RegistrySpec[Pkg.RegistrySpec(name = "General",
                                                       uuid = "23338594-aafe-5451-b93e-139f81909106",
                                                       url = "https://github.com/JuliaRegistries/General.git"),
                                      Pkg.RegistrySpec(name = "BioJuliaRegistry",
                                                       uuid = "ccbd2cc2-2954-11e9-1ccf-f3e7900901ca",
                                                       url = "https://github.com/BioJulia/BioJuliaRegistry.git")]

include("integration-test-utilities.jl")

Test.@testset "CompatHelper.jl" begin
    Test.@testset "CompatHelper.jl unit tests" begin
        @info("Running the CompatHelper.jl unit tests")
        include("unit-tests.jl")
    end

    COMPATHELPER_RUN_INTEGRATION_TESTS = get(ENV, "COMPATHELPER_RUN_INTEGRATION_TESTS", "")::String
    if COMPATHELPER_RUN_INTEGRATION_TESTS == "true"
        Test.@testset "CompatHelper.jl integration tests" begin
            @info("Running the CompatHelper.jl integration tests")
            include("integration-tests.jl")
        end
    end
end