text:
using Turing

function param(α,β)
    μ = α/(α+β)
    σ² = (α*β)/((α+β)^2 *(α+β+1))
    return [μ σ²]
end
    @model function BayesDisp(R,μ_scale = 1)
        k = length(R)
        μ ~ filldist(truncated(Normal(0,μ_scale),0,Inf),k)
        σ ~ filldist(Exponential(1),k)
        for i in 1:k
            R[i] .~ Normal(μ[i],σ[i])
        end
    end

    @model function BetaDisp(R)
        k = length(R)
        α ~ filldist(Uniform(1,100),k)#Gamma(2,2)
        β ~ filldist(Uniform(1,100),k)#Gamma(2,2)
        for i in 1:k
            R[i] .~ Beta(α[i],β[i])
        end
    end
function bayesdisp(disp,μ_scale,n_samples = 1000,n_chains = 4, sampler = NUTS())
    mod = BayesDisp(disp.residuals, μ_scale)
    chns = sample(mod,sampler, MCMCThreads(),n_samples,n_chains)
    D = Dict()
    for (i,level) in enumerate(string.(disp.levels))
        D["μ[$(i)]"] = level * "_μ"
        D["σ[$(i)]"] = level * "_σ"
    end

    return replacenames(chns, D)
end

function bayesdisp(disp,n_samples = 1000,n_chains = 4, sampler = NUTS())
    mod = BetaDisp(disp.residuals)
    chns = sample(mod,sampler, MCMCThreads(),n_samples,n_chains)
    D = Dict()
    for (i,level) in enumerate(string.(disp.levels))
        D["α[$(i)]"] = level * "_μ"
        D["β[$(i)]"] = level * "_σ"
    end

    return replacenames(chns, D)
end

export bayesdisp, param


text:
"""
# Usage

    solve(prob::FIEProblem, n, SpectralUltraspherical())

Using Ultrasphereical spectral method solve fractional integral equations.

### References

```tex
@article{Hale2018AFA,
  title={A Fast and Spectrally Convergent Algorithm for Rational-Order Fractional Integral and Differential Equations},
  author={Nicholas Hale and Sheehan Olver},
  journal={SIAM J. Sci. Comput.},
  year={2018},
  volume={40},
  pages={A2456-A2491}
}
```
"""
struct SpectralUltraspherical <: FractionalDiffEqAlgorithm end

function solve(FIE::FIEProblem, n, ::SpectralUltraspherical)
	@unpack parameters, orders, rightfun, tspan = FIE
	# Initialize operators
	Z = spzeros(n, n)
	I1 = sparse(Matrix(I, n, n))
	II = [I1 Z; Z I1]
	Q05(lam) = Qmat(n, 0.5, lam)
	QQ05 = [Z Q05(1); Q05(0.5) Z]
	QQ = m->QQ05^(2*m)

	judgeorders(j) = j == 1 ? (return II) : (return QQ(j))

	A = zeros(2*n, 2*n)

	for (i, j) in zip(parameters, orders)
	  A += i*judgeorders(j)
	end

	f = 0
	rhs = [mycoeffs(rightfun, n, 0.5); mycoeffs(f, n, 1)]

	idx = [collect(1:n)'; collect(n+1:2*n)']
	idx = idx[:]
	A = A[idx, idx]

	# Initialize solution
	u=zeros(size(A, 1))
	u[idx] = A\rhs

	# Evaluate solution
	sol = myeval(u, tspan, 2)
	return FIESolution(tspan, sol)
end

function myeval(u, x, q)
	if size(u, 2) > 1
	    uu = zeros(length(x), size(u, 2))
	    for k = 1:size(u, 2)
	        uu[:, k] = myeval(u[:, k], x, q)
	    end
	    return uu
	end

	N::Int64 = length(u)/q

	if q == 2
	    uu = clenshawP(x, u[1:N]) .+ sqrt.(1 .+x).*clenshawU(x, u[N+1:2*N])
	else
    	uu = 0*x
    	for k = 0:q-1
        	uu = uu .+ (1+x).^(k/q).*clenshawJ_special(x, u[k*N+collect(1:N)], k/q)
    	end
	end
	return uu
end

function clenshawP(x, c)
	bk1 = 0*x
	bk2 = copy(bk1)
	N = size(c, 1)-1
	for k = N:-1:1
	    bk = c[k+1] .+ (2*k+1)/(k+1)*x.*bk1 .- (k+1)/(k+2)*bk2
	    bk2 = bk1
	    bk1 = bk
	end
	y = c[1] .+ x.*bk1 - 0.5*bk2
	return y
end

function clenshawU(x, c)
	bk1 = 0*x
	bk2 = copy(bk1)
	N = size(c, 1)-1
	for k = N:-1:1
	    bk = c[k+1] .+ 2*x.*bk1 - bk2
	    bk2 = bk1
	    bk1 = bk
	end
	y = c[1] .+ 2*x.*bk1 - bk2
	return y
end

function clenshawJ_special(x, c, b)
	bk1 = 0*x
	bk2 = copy(bk1)
	N = size(c, 1)-1
	for k = N:-1:1
	    Ak = (2*k+3)
	    Bk = (1-2*b)/(2*k+1)
	    Ck = (k+2-b)*(k+1+b)*(2*k+5)/((k+3)*(2*k+3))
	    bk = c[k+1] + ((Ak*x+Bk).*bk1 - Ck*bk2)/(k+2)
	    bk2 = bk1
	    bk1 = bk
	end
	Ck = (2-b)*(1+b)*5/18
	y = c[1] + 0.5*(3*x+1-2*b).*bk1 - Ck*bk2
	return y
end

ext: jl


text:
# [note1]: Expects julia.exe in path (e.g. use mingw with export PATH="$PATH:/c/julia/bin"
# [note2]: https://discourse.julialang.org/t/interpolating-an-empty-string-inside-backticks/2428/3
using octsock5
using Base.Test
function runTwoProcess(args::Cmd, cSharp::Bool, tcpIp::Bool) # [note2]
    # start server process [note1]

    if cSharp
        assert(false == tcpIp);
        # run C# server implementation
        a1::Tuple = open(`csharp.exe`);
    else
        # run Julia server implementation
        if tcpIp
            a1 = open(`julia.exe main.jl server tcpip`);
        else
            a1 = open(`julia.exe main.jl server`);
        end
    end
    # wait until server has bound the port
    resp1::String = readline(a1[1]);
    assert(contains(resp1, "SERVER_READY"));

    # start client process
    #sleep(0.5);
    if tcpIp
        a2::Tuple = open(`julia.exe main.jl client $args tcpip`);
    else
        a2 = open(`julia.exe main.jl client $args`);
    end
    # read all output
    while true
        resp1 = readline(a1[1]);
        if contains(resp1, "SERVER_EXIT")
            break;
        end
    end
    resp2::String = readstring(a2[2]);

    # clean shutdown
    close(a1[1]);
    close(a2[1]);
    return (resp1, resp2)
end

function runLatency(cSharp::Bool, tcpIp::Bool)
    info("running latency test");
    (a,b) = runTwoProcess(`roundtrip`, cSharp, tcpIp);
    info("Server reports: " * a)
    info("Client reports: " * b);
    return true
end

function runThroughput(cSharp::Bool, tcpIp::Bool)
    info("running throughput test");
    (a, b) = runTwoProcess(`throughput`, cSharp, tcpIp)
    info("Server reports: " * a)
    info("Client reports: " * b);
    return true
end

function runAllTypes(cSharp::Bool, tcpIp::Bool)
    info("running all types test");
    (a, b) = runTwoProcess(`alltypes`, cSharp, tcpIp)
    info("Server reports: " * a)
    info("Client reports: " * b);
    return true
end

# tests NaN, Inf
function runSpecials(cSharp::Bool, tcpIp::Bool)
    info("running +/-Inf, NaN test");
    (a, b) = runTwoProcess(`specials`, cSharp, tcpIp)
    info("Server reports: " * a)
    info("Client reports: " * b);
    return true
end

# tests a large data package (blocking problems?
function runLarge(cSharp::Bool, tcpIp::Bool)
    info("running large data package test");
    (a, b) = runTwoProcess(`large`, cSharp, tcpIp)
    info("Server reports: " * a)
    info("Client reports: " * b);
    return true
end

# opens and closes repeatedly. Tests, whether handles are cleanly released.
function reopen(tcpIp::Bool)
    if tcpIp
        portNum::Int64 = 20000;
    else
        portNum = -12345;
    end
    for i = 1 : 100
        iServer::octsock5_cl = octsock5_new(isServer=true, portNum=portNum);
        iClient::octsock5_cl = octsock5_new(isServer=false, portNum=portNum);
        octsock5_accept(iServer);
        s1::String = "Hello world this is a test.";
        s2::String = "Lorem Ipsum. Bonk.";
        octsock5_write(iServer, s1);
        octsock5_write(iClient, s2);
        assert(octsock5_read(iClient) == s1);
        assert(octsock5_read(iServer) == s2);
        octsock5_delete(iServer);
        octsock5_delete(iClient);
    end
    return true;
end

# pass == 1 : Julia loopback server, windows named pipes
# pass == 2 : Julia loopback server, TCP/IP (slow)
# pass == 3 : C# loopback server, windows named pipes
for pass = 1 : 3
    tcpIp::Bool = (pass == 2);
    cSharp::Bool = (pass == 3);

    if pass < 3
        @test reopen(tcpIp);
    end

    @test runLarge(cSharp, tcpIp);
    @test runLatency(cSharp, tcpIp);
    @test runThroughput(cSharp, tcpIp);
    @test runSpecials(cSharp, tcpIp);
    @test runAllTypes(cSharp, tcpIp);
    if pass < 3
        @test reopen(tcpIp);
    end
end


text:
## Basis data structures.

abstract type AbstractSzbasis end

"""
Basis of occupation vectors.
"""
struct Szbasis <: AbstractSzbasis
    "Number of sites."
    K::Int
    "Number of particles."
    N::Int

    "Number of basis vectors."
    D::Int

    "Occupation vectors (K by D)."
    vectors::Matrix{Int}
end

"""
    Szbasis(K::Int, N::Int)

Create a basis for `K` sites and `N` particles.
"""
function Szbasis(K::Int, N::Int)
    K >= 1 || throw(DomainError(K, "At least 1 site is required."))
    N >= 0 || throw(DomainError(N, "At least 0 particles are required."))

    # Basis size.
    D = num_vectors(N, K)

    v = zeros(Int, K)
    v[1] = N
    vectors = Matrix{Int}(undef, K, D)
    vectors[:, 1] .= v

    for i in 2:D
        if v[1] > 0
            v[1] -= 1
            v[2] += 1
        else
            j = findfirst(!iszero, v)

            v[1] = v[j] - 1
            v[j] = 0
            v[j+1] += 1
        end
        vectors[:, i] .= v
    end

    Szbasis(K, N, D, vectors)
end

"""
Basis of occupation vectors with a site occupation restriction.
"""
struct RestrictedSzbasis <: AbstractSzbasis
    "Number of sites."
    K::Int
    "Number of particles."
    N::Int
    "Site capacity."
    M::Int

    "Number of basis vectors."
    D::Int

    "Occupation vectors (K by D)."
    vectors::Matrix{Int}
end

"""
    RestrictedSzbasis(K::Int, N::Int, M::Int)

Create a basis for `K` sites and `N` particles, with no more than `M` particles
per site.
"""
function RestrictedSzbasis(K::Int, N::Int, M::Int)
    K >= 1 || throw(DomainError(K, "At least 1 site is required."))
    N >= 0 || throw(DomainError(N, "At least 0 particles are required."))
    N <= K * M || throw(DomainError(N, "Particles do not fit on the sites."))

    # Basis size.
    D = num_vectors(N, K, M)
    dNM = M > 0 ? div(N, M) : 1

    v = zeros(Int, K)
    for j in 1:dNM
        v[j] = M
    end
    if 1 <= (dNM + 1) <= K
        v[dNM+1] = N - M * dNM
    end
    vectors = Matrix{Int}(undef, K, D)
    vectors[:, 1] .= v

    for i in 2:D
        if v[1] > 0
            if v[1] < M
                delta = M - v[1]
                v[1] = M
            else
                delta = 0
            end

            j = findfirst(!iszero, v .< M)

            v[j] += 1
            v[j-1] -= 1 + delta
        else
            j = findfirst(!iszero, v)
            k = j + findfirst(!iszero, @view(v[(j+1):end]) .< M)

            v[k-j] = v[j] - 1
            v[k] += 1
            for l in 1:(k-j-1)
                v[l] = M
            end
            # The indices after the first one differ from those in the paper.
            for l in (k-j+1):(k-1)
                v[l] = 0
            end
        end
        vectors[:, i] .= v
    end

    RestrictedSzbasis(K, N, M, D, vectors)
end


text:
module TwitterCard

using Conda
using PyCall: pyimport

const tbuf = IOBuffer()

function twitter_meta(prop, content)
    write(tbuf, "<meta name=\"twitter:")
    write(tbuf, prop)
    write(tbuf, "\" content=\"")
    write(tbuf, content)
    write(tbuf, "\"></meta>")
end

function twitter_card(content)
    twitter_meta("card", "summary")
end

function get_twitter_tools()
    try
        return pyimport("twitter")
    catch
        Conda.pip("install", "twitter")
        return pyimport("twitter")
    end
end

end


text:
module Ocean

using ..BalanceLaws
using ..Problems

export AbstractOceanModel,
    AbstractOceanProblem,
    AbstractOceanCoupling,
    Uncoupled,
    Coupled,
    AdvectionTerm,
    NonLinearAdvectionTerm,
    InitialConditions

abstract type AbstractOceanModel <: BalanceLaw end
abstract type AbstractOceanProblem <: AbstractProblem end

abstract type AbstractOceanCoupling end
struct Uncoupled <: AbstractOceanCoupling end
struct Coupled <: AbstractOceanCoupling end

abstract type AdvectionTerm end
struct NonLinearAdvectionTerm <: AdvectionTerm end

function ocean_init_state! end
function ocean_init_aux! end
function ocean_boundary_state! end

function coriolis_parameter end
function kinematic_stress end
function surface_flux end

include(joinpath("Domains", "Domains.jl"))
include(joinpath("Fields", "Fields.jl"))

include("OceanBC.jl")

include("HydrostaticBoussinesq/HydrostaticBoussinesq.jl")

using .HydrostaticBoussinesq: HydrostaticBoussinesqModel, Forcing

include("ShallowWater/ShallowWaterModel.jl")
include("SplitExplicit/SplitExplicitModel.jl")
include("SplitExplicit01/SplitExplicitModel.jl")
include("OceanProblems/OceanProblems.jl")

include("SuperModels.jl")

using .OceanProblems: InitialConditions
using .SuperModels:
    HydrostaticBoussinesqSuperModel, current_time, current_step, Δt

include("JLD2Writer.jl")

using .JLD2Writers: JLD2Writer, OutputTimeSeries, write!

end

max_forks_repo_name: jm-c/CLIMA


text:
# Train/test dataset paths

const HOMERENTALS = joinpath(DATADIR, "home_rentals", "home_rentals.csv")

const GENESEQUENCES = joinpath(DATADIR, "splice_junction_gene_sequences",
                               "splice_junction_gene_sequences.csv")

const CIFAR100TRAIN = joinpath(DATADIR, "cifar_100", "train.csv")
const CIFAR100TEST = joinpath(DATADIR, "cifar_100", "test.csv")

const IMDBTRAIN = joinpath(DATADIR, "imdb_movie_review", "train.tsv")
const IMDBTEST = joinpath(DATADIR, "imdb_movie_review", "test.tsv")

const QUORATRAIN = joinpath(DATADIR, "quora_questions", "train.csv")
const QUORATEST = joinpath(DATADIR, "quora_questions", "test.csv")

const DOCTRAIN = joinpath(DATADIR, "default_of_credit", "train.csv")
const DOCTEST = joinpath(DATADIR, "default_of_credit", "test.csv")

const PORTRAIN = joinpath(DATADIR, "prediction_of_return", "train.txt")
const PORTEST = joinpath(DATADIR, "prediction_of_return", "test.txt")

const SENSOR = joinpath(DATADIR, "sensor_data", "sensor_data.csv")
const SPAM = joinpath(DATADIR, "spam", "spam.csv")

function house_rentals()
    df = CSV.read(HOMERENTALS)
end

function splice_junction()
    df = CSV.read(GENESEQUENCES)
end

function cifar100()
    trn = CSV.read(CIFAR100TRAIN)
    tst = CSV.read(CIFAR100TEST)
    trn, tst
end

function imdb_movie_review()
    trn = CSV.read(IMDBTRAIN)
    tst = CSV.read(IMDBTEST)
    trn, tst
end

function quora_questions()
    trn = CSV.read(QUORATRAIN)
    tst = CSV.read(QUORATEST)
    trn, tst
end

function default_of_credit()
    trn = CSV.read(DOCTRAIN)
    tst = CSV.read(DOCTEST)
    trn, tst
end

function prediction_of_return()
    trn = readtext(PORTRAIN)
    tst = readtext(PORTEST)
    trn, tst
end

function sensor()
    df = CSV.read(SENSOR)
end

function spam()
    df = CSV.read(SPAM)
end

max_stars_repo_name: alperkesen/AutoML

max_issues_repo_name: alperkesen/AutoML


text:
VERSION ≥ v"1.4" && @testset "evalpoly" begin
    # test fallbacks for when code generation fails
    @testset "fallbacks for $T" for T in (Float64, ComplexF64)
        x, p = randn(T), Tuple(randn(T, 10))
        y_fb, ys_fb = ChainRules._evalpoly_intermediates_fallback(x, p)
        y, ys = ChainRules._evalpoly_intermediates(x, p)
        @test y_fb ≈ y
        @test collect(ys_fb) ≈ collect(ys)

        Δy, ys = randn(T), Tuple(randn(T, 9))
        ∂x_fb, ∂p_fb = ChainRules._evalpoly_back_fallback(x, p, ys, Δy)
        ∂x, ∂p = ChainRules._evalpoly_back(x, p, ys, Δy)
        @test ∂x_fb ≈ ∂x
        @test collect(∂p_fb) ≈ collect(∂p)
    end

    @testset "x dim: $(nx), pi dim: $(np), type: $T" for T in (Float64, ComplexF64), nx in (tuple(), 3), np in (tuple(), 3)
        # skip x::Matrix, pi::Number case, which is not supported by evalpoly
        isempty(np) && !isempty(nx) && continue
        m = 5
        x = randn(T, nx..., nx...)
        p = [randn(T, np..., np...) for _ in 1:m]
        test_frule(evalpoly, x, p)
        test_frule(evalpoly, x, Tuple(p))
        test_rrule(evalpoly, x, p)
        test_rrule(evalpoly, x, Tuple(p))
    end

    # mixed type inputs
    test_rrule(evalpoly, rand(ComplexF64), rand(3))
end


text:
"""
    This is a port of the IDL script 'TSC.pro' by Joop Schaye from Feb 1999.
    Original: https://idlastro.gsfc.nasa.gov/ftp/pro/math/tsc.pro
"""

module TriangularShapedCloudInterpolation

    export TSCInterpolation,
           get_tsc_positions

    """
        get_tsc_positions(pos::Array{<:Real}, res_elements::Array{<:Integer})

    Returns an array with interpolation positions for a given `pos` array.
    Number of interpolation positions in each dimension is given by the `res_elements` array.
    """
    function get_tsc_positions(pos::Array{<:Real}, res_elements::Array{<:Integer})

        dim   = size(pos,1)
        Npart = size(pos,2)

        pos_tsc = Array{eltype(pos[1]),2}(undef, Npart, dim)

        @inbounds for i = 1:dim
            minx = minimum(pos[i,:])
            maxx = maximum(pos[i,:]) .* (1.0+1.e-6)
            dx   = -(minx - maxx) / res_elements[i]
            for j = 1:Npart
                pos_tsc[j,i] = ( pos[i,j] .- minx ) ./ dx
            end
        end

        return pos_tsc
    end

    """
        get_tsc_positions(pos::Array{<:Real}, res_elements::Integer)

    Helper function if resolution elements in all dimensions are the same.
    """
    get_tsc_positions(pos::Array{<:Real}, res_elements::Integer) = get_tsc_positions(pos, [res_elements, res_elements, res_elements])

    @inline function get_distances_weights(pos::Array{<:Real}, n_steps::Integer;
                                           wraparound::Bool, isolated::Bool)

        # Coordinates of nearest grid point (ngp)
        if wraparound
            ng = floor.(pos  .+ 0.5)
        else
            ng = floor.(pos) .+ 0.5
        end

        # Distance from sample to ngp.
        dng = ng .- pos

        if wraparound
            k2 = ng
        else
            k2 = floor.(Int64, ng .- 0.5)
        end

        # Weight of ngp.
        w2 = 0.75 .- dng .* dng

        # Point before ngp.
        k1 = k2 .- 1  # Index.
        dx  = 1.0 .- dng  # Distance to sample.
        w1 = 0.5 .* (1.5 .- dx).^2  # TSC-weight.

        # Point after ngp.
        k3 = k2 .+ 1  # Index.
        dx  = 1.0 .+ dng # Distance to sample.
        w3 = 0.5 .* (1.5 .- dx).^2  # TSC-weight.

        # Periodic boundary conditions.
        bad = findall( k2 .== 0 )
        if length(bad) > 0
            k1[bad] .= n_steps - 1
            if isolated
                w1[bad] .= 0.0
            end
        end

        bad = findall( k2 .== (n_steps - 1) )
        if length(bad) > 0
            k3[bad] .= 0
            if isolated
                w3[bad] .= 0.0
            end
        end

        if wraparound
            bad = findall( k2 .== n_steps )
            if length(bad) > 0
                k2[bad] .= 0
                k3[bad] .= 1
            end
        end

        return k1, k2, k3, w1, w2, w3

    end

    """
        calculate_weights!(field::Array{<:Real}, tottscweight::Array{<:Real},
                                        index::Array{<:Integer}, tscweight::Array{<:Real},
                                        value::Array{<:Real}, Nsamples::Integer;
                                        average::Bool=false)

    Helper function to update the weights arrays.
    """
    @inline function calculate_weights!(field::Array{<:Real}, tottscweight::Array{<:Real},
                                        index::Array{<:Integer}, tscweight::Array{<:Real},
                                        value::Array{<:Real}, Nsamples::Integer;
                                        average::Bool=false)

        for j = 1:Nsamples
            field[index[j]]  += tscweight[j] * value[j]
        end

        if average
            for j = 1:Nsamples
                tottscweight[index[j]] += tscweight[j]
            end
        end

        return field, tottscweight
    end

    """
        find_dim_bounds(dim::Integer)

    Helper function to get loop limits for dimension loops.
    """
    @inline function find_dim_bounds(dim::Integer)

        if dim == 1
            dimx = 3
            dimy = 1
            dimz = 1
        elseif dim == 2
            dimx = 3
            dimy = 3
            dimz = 1
        elseif dim == 3
            dimx = 3
            dimy = 3
            dimz = 3
        end

        return dimx, dimy, dimz
    end

    """
        TSCInterpolation( value::Array{<:Real},
                          pos::Array{<:Real},
                          res_elements::Array{<:Integer},
                          average::Bool=true,
                          wraparound::Bool=false,
                          isolated::Bool=false    )

    Runs a TSC interpolation on the `value` array based on the provided positions.
    Returns a 3D array with interpolated values.
    """
    function TSCInterpolation(  value::Array{<:Real},
                                pos::Array{<:Real},
                                res_elements::Array{<:Integer};
                                average::Bool=true,
                                wraparound::Bool=false,
                                isolated::Bool=false    )

        Nsamples = length(value)

        # allocate arrays for indices and weights
        kx = zeros(Int64, Nsamples,3)
        ky = zeros(Int64, Nsamples,3)
        kz = zeros(Int64, Nsamples,3)

        wx = ones(Nsamples,3)
        wy = ones(Nsamples,3)
        wz = ones(Nsamples,3)

        dim = size(pos,2)

        nx = res_elements[1]

        # x direction
        kx[:,1], kx[:,2], kx[:,3], wx[:,1], wx[:,2], wx[:,3] = get_distances_weights(pos[:,1], res_elements[1],
                                                                                wraparound = wraparound,
                                                                                isolated   = isolated)

        # y direction
        if dim > 1
            ny = res_elements[2]

            ky[:,1], ky[:,2], ky[:,3], wy[:,1], wy[:,2], wy[:,3] = get_distances_weights(pos[:,2], res_elements[2],
                                                                    wraparound = wraparound,
                                                                    isolated   = isolated)
        else
            ny = 1
        end

        # z direction
        if dim > 2
            nz = res_elements[3]

            kz[:,1], kz[:,2], kz[:,3], wz[:,1], wz[:,2], wz[:,3] = get_distances_weights(pos[:,3], res_elements[3],
                                                                    wraparound = wraparound,
                                                                    isolated   = isolated)
        else
            nz = 1
        end

        nxny = nx*ny

        # find maximum indices for dimension loop
        dimx, dimy, dimz = find_dim_bounds(dim)

        # allocate arrays for interpolated data and weighting
        field = zeros( nx * ny * nz )
        tottscweight = zeros( nx * ny * nz )

        # run interpolation
        @inbounds @fastmath for i = 1:dimx, j = 1:dimy, k = 1:dimz

            index     = @. kx[:,i] + ky[:,j] * nx + kz[:,k] * nxny + 1
            tscweight = @. wx[:,i] * wy[:,j] * wz[:,k]

            calculate_weights!( field, tottscweight,
                                index, tscweight, value, Nsamples,
                                average=average )

        end

        if average
            good = findall(tottscweight .> 0.0)
            field[good] ./= tottscweight[good]
        end

        return reshape(field, (nx, ny, nz))
    end

    """
        TSCInterpolation( value::Array{<:Real},
                          pos::Array{<:Real},
                          res_elements::Integer;
                          average::Bool=true,
                          wraparound::Bool=false,
                          isolated::Bool=false  )

    Helper function to run TSC interpolation with the same number of resolution elements in all dimensions.
    """
    function TSCInterpolation(  value::Array{<:Real},
                                pos::Array{<:Real},
                                res_elements::Integer;
                                average::Bool=true,
                                wraparound::Bool=false,
                                isolated::Bool=false    )

        dim = length(pos[1,:])

        res = res_elements .* ones(Int64, dim)

        return TSCInterpolation(value, pos, res,
                                average=average,
                                wraparound=wraparound,
                                isolated=isolated )
    end

end # module


text:
struct AVKUserModel

    # data for user -
    alphavantage_api_email::String
    alphavantage_api_key::String

    # constructor -
    function AVKUserModel(api_email::String="teamalphavantage@gmail.com",api_key::String="SKHF9DT43T4S8IR6")
        new(api_email,api_key)
    end
end

function _check_if_user_dictionary_contains_correct_keys(user_data_dictionary::Dict{String,Any})::Result{Bool}

    # the user dictionary should contain a user_data root, and a alpha_vantage_api_key child -
    # TODO: fill me in ...
    return Result{Bool}(true)
end

# -- PUBLIC METHODS HERE ------------------------------------------------------- #
"""
    build_api_user_model(path) -> {Result{AVKError} or Result{AVKUserModel}}

Takes a `path` string which points to a JSON configuration file.
Returns either a Result{AVKError} if something went wrong, or a Result{AVKUserModel} object holding the user email and AlphaVantage API key.
The AVKError and AVKUserModel can be accessed using the `value` field on the Result return wrapper
"""
function build_api_user_model(path_to_configuration_file::String)::(Union{T, Nothing} where T<:Any)

    # some users checks -
    # did the user pass in a legit path?
    check_result = _is_path_valid(path_to_configuration_file)
    if (typeof(check_result.value) == Bool && check_result.value == false)
        error_message = "error: $(path_to_configuration_file) in not a valid path"
        return Result{AVKError}(AVKError(error_message))
    end

    # ok, path seems legit - load the default user information from the config.json file -
    user_json_dictionary = JSON.parsefile(path_to_configuration_file)

    # does the user dictionary contain the correct keys?
    if (_check_if_user_dictionary_contains_correct_keys(user_json_dictionary) == false)
        error_message = "error: missing keys in user configuration dictionary"
        return Result{AVKError}(AVKError(error_message))
    end

    # -- DO NOT EDIT BELOW THIS LINE ------------------------------------------#

    # grab the user data -
    alpha_vantage_api_key = user_json_dictionary["user_data"]["alpha_vantage_api_key"]
    alpha_vantage_api_email = user_json_dictionary["user_data"]["alpha_vantage_api_email"]

    # build APIUserModel -
    api_user_model = AVKUserModel(alpha_vantage_api_email, alpha_vantage_api_key)

    # return the user_data_dictionary -
    return Result{AVKUserModel}(api_user_model)
    # -------------------------------------------------------------------------#
end

max_stars_repo_name: teamalphavantage/AlphaVantageAPI

max_issues_repo_name: teamalphavantage/AlphaVantageAPI

max_forks_repo_name: teamalphavantage/AlphaVantageAPI


text:
function kci!(freqs::Vector{<:Integer}, kci_peak::Float64)
    sort!(freqs)
    return length(freqs) > 10 ? freqs[div(length(freqs), 2)] / kci_peak : -1.0
end

kci(freqs::Vector{<:Integer}, kci_peak::Float64) = kci!(copy(freqs), kci_peak)

function kci(indexed_counts::IndexedCounts{M,C}, name::Symbol, sequence::BioSequence, kci_peak::Float64) where {M,C}
    freqs = project_count(indexed_counts, name, sequence)
    filter!(!iszero, freqs)
    return kci!(freqs, kci_peak)
end


text:
@testset "Reach-set constructors" begin
    X = BallInf(ones(2), 1.0)

    # constructor with a time interval
    R = ReachSet(X, 0 .. 1)
    @test tspan(R) == 0 .. 1

    # constructor with a time point
    R = ReachSet(X, 1.0)
    @test tspan(R) == interval(1.0)

    # if the time is an integer, it is converted to a float
    R = ReachSet(X, 1)
    @test tspan(R) == interval(1.0)
end

@testset "Reach-set projections" begin
    X = BallInf(zeros(5), 1.0)
    B = BallInf(zeros(2), 1.0)

    R = ReachSet(X, 0 .. 1)

    # concrete projection of a set
    p1 = project(X, [1, 2])
    p2 = project(X, (1, 2))
    p3 = project(X, vars=(1, 2))
    p4 = project(X, vars=[1, 2])
    @test all(x -> isequivalent(B, x), [p1, p2, p3, p4])

    # lazy projection of a set
    p1 = Projection(X, [1, 2])
    p2 = Projection(X, (1, 2))
    p3 = Projection(X, vars=(1, 2))
    p4 = Projection(X, vars=[1, 2])
    @test all(x -> isequivalent(B, overapproximate(x, Hyperrectangle)), [p1, p2, p3, p4])

    # concrete projection of a reach-set
    p1 = project(R, [1, 2])
    p2 = project(R, (1, 2))
    p3 = project(R, vars=(1, 2))
    p4 = project(R, vars=[1, 2])
    @test all(x -> isequivalent(B, set(x)), [p1, p2, p3, p4])

    # concrete projection of a reach-set along a spatial variable and the time variable
    Bt = Hyperrectangle(low=[0.0, -1.0], high=[1.0, 1.0])
    p1 = project(R, [0, 1])
    p2 = project(R, (0, 1))
    p3 = project(R, vars=(0, 1))
    p4 = project(R, vars=[0, 1])
    @test all(x -> isequivalent(Bt, set(x)), [p1, p2, p3, p4])

    # lazy projection of a reach-set
    p1 = Projection(R, [1, 2])
    p2 = Projection(R, (1, 2))
    p3 = Projection(R, vars=(1, 2))
    p4 = Projection(R, vars=[1, 2]);
    @test all(x -> isequivalent(B, overapproximate(set(x), Hyperrectangle)), [p1, p2, p3, p4])

    # lazy projection of a reach-set along a spatial variable and the time variable
    p1 = Projection(R, [0, 1])
    p2 = Projection(R, (0, 1))
    p3 = Projection(R, vars=(0, 1))
    p4 = Projection(R, vars=[0, 1]);
    @test all(x -> isequivalent(Bt, overapproximate(set(x), Hyperrectangle)), [p1, p2, p3, p4])
end

@testset "Conversion of Taylor model reach-sets" begin
    H = Hyperrectangle(ones(2), [0.2, 0.4])
    a = overapproximate(H, TaylorModelReachSet)
    b = convert(TaylorModelReachSet, H)

    @test tspan(a) == tspan(b) == 0 .. 0
    @test isequivalent(set(overapproximate(a, Hyperrectangle)), H)
    @test isequivalent(set(overapproximate(b, Hyperrectangle)), H)

    X = box_approximation(a)
    Y = overapproximate(a, Hyperrectangle)
    @test tspan(X) == tspan(Y)
    @test isequivalent(set(X), set(Y))

    R = ReachSet(H, 0 .. 1)
    c = overapproximate(R, TaylorModelReachSet)
    d = convert(TaylorModelReachSet, R)

    @test tspan(c) == tspan(d) == 0..1
    @test isequivalent(set(overapproximate(c, Hyperrectangle)), set(R))
    @test isequivalent(set(overapproximate(d, Hyperrectangle)), set(R))

    Z = convert(Zonotope, H)
    a = overapproximate(Z, TaylorModelReachSet)
    b = convert(TaylorModelReachSet, Z)

    @test tspan(a) == tspan(b) == 0 .. 0
    @test isequivalent(set(overapproximate(a, Zonotope)), Z)
    @test isequivalent(set(overapproximate(b, Zonotope)), Z)

    R = ReachSet(Z, 0 .. 1)
    c = overapproximate(R, TaylorModelReachSet)
    d = convert(TaylorModelReachSet, R)

    @test tspan(c) == tspan(d) == 0..1
    @test isequivalent(set(overapproximate(c, Zonotope)), set(R))
    @test isequivalent(set(overapproximate(d, Zonotope)), set(R))

end


text:
module StructuredSingularValues

using LinearAlgebra

include("common.jl")
include("lowerbounds.jl")
include("upperbounds.jl")

export lowerboundSSV

end

max_stars_repo_name: hurak/StructuredSingularValues

max_issues_repo_name: hurak/StructuredSingularValues

max_forks_repo_name: hurak/StructuredSingularValues


text:
module NumericalAnalysis
using Documenter
using ForwardDiff
using LinearAlgebra
using PyCall
using Conda
using SymPy
using Latexify

include("basic.jl")
export my_f, derivative_my_f, ξ, absolute_error, relative_error, NthDerivative, TaylorPolynomials

# Write your package code here.

include("SEq1.jl")

export Bisection, fixed_point, Newton, Secant, FalsePos, ModifiedNewton

include("Polynomial.jl")

export Lagrange, Neville, NDDF, NCSpline, CCSpline

include("NCalculus.jl")

export ThreePoint, FivePoint, Trapezoidal, Simpson, Newton_cotes, Romberg,Gaussian_Quad, SimpsonDoubleIntegral, MutipleIntegralSimple, GaussianDoubleIntegral

end


text:
using Revise
#]add RiskyCommunity
using RiskyCommunity
using Parameters
using CSV
using DataFrames
using Statistics
using Glob
using JuMP
using Gurobi
using Query
using StatsBase

###########FUNCTIONS TO USE#########
    function read_result(model::JuMP.Model)
        cost_AGG = JuMP.value(model[:cost_AGG])
        cost_DA_market = sum(JuMP.value.(model[:cost_DA_market]))
        cost_RT_market = sum(JuMP.value.(model[:cost_RT_market]))
        @show cost_AGG, cost_DA_market, cost_RT_market

        soc = JuMP.value.(model[:soc])
        cap = model.ext[:param][:capsₑ][1]

        viol_balance_up = soc.data[soc.data .> (0.001 + cap)]
        viol_balance_down = soc.data[soc.data .< -0.001]
        sum_balance_down = sum(viol_balance_down)
        sum_balance_up = sum(viol_balance_up)
        numb_balance_up = length(viol_balance_up)
        numb_balance_down = length(viol_balance_down)

        reliability_in = (1-(numb_balance_up + numb_balance_down)/(5*96*65))

        @show numb_balance_up, numb_balance_down

        in_res_dict = Dict("cost_AGG" => cost_AGG, "reliability_in" => reliability_in , "cost_DA_market" => cost_DA_market, "cost_RT_market" => cost_RT_market,
        "numb_viol_UP" => numb_balance_up, "numb_viol_DOWN" => numb_balance_down, "sum_balance_down" => sum_balance_down, "sum_balance_up" => sum_balance_up)

        return in_res_dict
    end

    function read_test_result(model::JuMP.Model)
        cost_AGG = JuMP.value(model[:true_cost_AGG])
        cost_DA_market = sum(JuMP.value.(model[:cost_DA_market]))
        cost_RT_market = sum(JuMP.value.(model[:cost_RT_market]))

        slack_DOWN = JuMP.value.(model.ext[:dec_vars][:slack_down])
        slack_UP = JuMP.value.(model.ext[:dec_vars][:slack_up])

        #Violations of Balance
        numb_upper_violation = length(slack_DOWN.data[slack_DOWN.data .≠ 0])
        numb_lower_violation = length(slack_UP.data[slack_UP.data .≠ 0])
        numb_viol = (numb_upper_violation + numb_lower_violation)

        max_DOWN = max(slack_DOWN...)
        max_UP = max(slack_UP...)

        sum_DOWN = sum(slack_DOWN)
        sum_UP= sum(slack_UP)

        @show cost_AGG, cost_DA_market, cost_RT_market

        res_dict = Dict("numb_viol" => numb_viol,  "cost_AGG" => cost_AGG, "cost_DA_market" => cost_DA_market, "cost_RT_market" => cost_RT_market,
         "max_DOWN" => max_DOWN, "max_UP" => max_UP, "sum_UP" => sum_UP, "sum_DOWN" => sum_DOWN)
        return res_dict
    end

##############INIT THE MARKET DATA################
    period = Period(number_of_days = 1, number_of_t_steps= 96)
    # ##########OBTAIN THE RT IMBALANCE PRICE SCENARIOS FOR THE TRAINING SET##########
    scenarios_train = Dict()
    scen_price_RT_train = Dict()
    dat = glob("*.csv","data/imb_price_scenarios/cluster_train_5/")
    train_dat = dat
    for d in 1:length(train_dat)
        scenario_train = DataFrame!(CSV.File(train_dat[d], header = true, decimal='.'))
        scenario_train = scenario_train[2:end, :]
        scenarios_train[d] = scenario_train
        scen_price_RT_train[d] = Scenario(id = "RT_price_$d", counter = d, prob = 1/length(train_dat)) #Equal probability
    end

    prices_RT_train = []
    for d in 1:length(train_dat)
        push!(prices_RT_train, Price(period = period,  scenario = scen_price_RT_train[d], id = scen_price_RT_train[d].id))
        prices_RT_train[d].values[1,:] = scenarios_train[d].values
    end

##########CHOSING ONE FORECAST FOR DA PRICES##########
    da_price_init =  DataFrame!(CSV.File("data/Day-ahead Prices_202003150000-202003160000.csv", header = true, decimal=','))[:,2]
    da_price = [parse(Float64, da_price_init[i]) for i in range(1, stop = length(da_price_init))]
    da_price_96 = repeat(da_price, inner = [4]) #EUR/MWh
    #Single scenario for DA price
    scen_price_DA = Scenario(id = :DA_price, prob = 1.0) #Single DA scenario
    #Init DA price forecast
    price_DA = Price(period = period)
    price_DA.values[1,:] = da_price_96

    #Use this form when passing to the JuMP model
    prices_train = Dict(:DA => price_DA, :RT => prices_RT_train)

###########INIT THE DER AGENT DATA AS A SINGLE AGENT WITH BESS#############
    dat_agent = DataFrame!(CSV.File("data/WCVaR_dat/single_site/63/da_fc/63_err_1.csv", header=true, delim= ","))
    values(dat_agent[1,2:end])
    # sort(dat_agent, [:timestamp])
    loads = collect(values(dat_agent[1,2:end]) ./1000) # To get something reasonable in MW
    my_der_agent = DER_Agent(period = period)
    my_der_agent.demand.values[1,:] = loads
    my_der_agent.dev_ID.values[1,:] = zeros(length(loads)) #No ID dev
    my_der_agent.dev_RT.values[1,:] = zeros(length(loads)) #No RT dev
    #The model complains if there are no scenarios added as it expects
    scen_ID_dev_1 = Scenario(id = :ID_dev_1, counter = 1, prob = 1.0)
    dev_ID_1 = Demand(period = period, id= :ID_dev_1, scenario = scen_ID_dev_1, values = zeros(period.number_of_days, period.number_of_t_steps))
    deviations_ID = [dev_ID_1]
    my_der_agent.dev_ID = deviations_ID

#####LOADING RT DEV SCENARIOS##########FOR  WCVAR OPT#######
    # loc = "data/WCVaR_dat/diff_per_site/mix2/split"
    # # loc = "data/WCVaR_dat/single_site/58/clustered"
    # groups = readdir(loc)
    #
    # errors_dict = Dict()
    # # errors_dict["$curr_group"] = glob("*.csv","data/WCVaR_dat/diff_per_site/mix2/split/$curr_group")
    # err_scenarios_train = Dict()
    # scen_err_RT_train = Dict()
    # global counter = 0
    # for g in groups
    #     curr_group = g
    #     errors_dict["$curr_group"] = glob("*.csv","$loc/$curr_group")
    #     for d in 1:length(errors_dict["$curr_group"])
    #         global counter = counter + 1
    #         ###GET THE PROBABILITY OF THE SCENARIO STORED IN THE NAME######
    #         str = errors_dict[g][d]
    #         prob_str = last(str, 10)
    #         prob_str = first(prob_str, 6)
    #         prob_float = parse(Float64, prob_str)
    #         #########UPDATE VALUES##########
    #         err_scenario_train = DataFrame!(CSV.File(errors_dict["$curr_group"][d], header = true, decimal='.'))
    #         err_scenario_train = err_scenario_train[1:end, :].values .* loads
    #         err_scenarios_train[g,d] = err_scenario_train
    #         scen_err_RT_train[g,d] = Scenario(id = "RT_error_SITE$curr_group", counter = counter, prob = prob_float/5) #Equal probability
    #     end
    # end
    #
    # errors_RT_train = []
    # for g in groups
    #     curr_group = g
    #     for d in 1:length(errors_dict["$curr_group"])
    #         push!(errors_RT_train, Demand(period = period,  scenario = scen_err_RT_train[g,d], id = scen_err_RT_train[g,d].id, values = permutedims(err_scenarios_train[g,d])))
    #         # errors_RT_train[d].values[1,:] = err_scenarios_train[g,d]
    #     end
    # end

    # FOR  CVAR OPT#######
    error_dat = glob("*.csv","data/WCVaR_dat/diff_per_site/mix2/")
    err_scenarios_train = Dict()
    scen_err_RT_train = Dict()
    for d in 1:length(error_dat)
        ###GET THE PROBABILITY OF THE SCENARIO STORED IN THE NAME######
        str = error_dat[d]
        prob_str = last(str, 10)
        prob_str = first(prob_str, 6)
        prob_float = parse(Float64, prob_str)
        #########UPDATE VALUES##########
        err_scenario_train = DataFrame!(CSV.File(error_dat[d], header = true, decimal='.'))
        err_scenario_train = err_scenario_train[1:end, :].values .* loads
        err_scenarios_train[d] = err_scenario_train
        scen_err_RT_train[d] = Scenario(id = "RT_error_group_1", counter = d, prob = prob_float/5) #Equal probability
    end

    errors_RT_train = []
    for d in 1:length(error_dat)
        push!(errors_RT_train, Demand(period = period,  scenario = scen_err_RT_train[d], id = scen_err_RT_train[d].id))
        errors_RT_train[d].values[1,:] = err_scenarios_train[d]
    end

    my_der_agent.dev_RT = errors_RT_train

##########BESS SETTING############
    batt_1 = Battery(capacity_power = 3.0, capacity_energy = 0.4, efficiency_dch = 0.99, efficiency_ch = 0.99)
    my_der_agent.battery = batt_1
    my_der_agent_2 = deepcopy(my_der_agent)
    my_aggregator = Aggregator(name ="Mihalys_aggregator")
    DER_agents = [my_der_agent]
    my_aggregator.DER_agents = DER_agents

#OPEXs OF THE DIFFERENT STAGES
    my_aggregator.DER_agents[1].OPEX_DA = mean(price_DA.values) .* 0.01
    my_aggregator.DER_agents[1].OPEX_ID = 0.001
    my_aggregator.DER_agents[1].OPEX_RT = sum(mean(prices_RT_train[j].values) for j in [1,2,3,4,5])./5 .* 0.05

    my_aggregator.preferences.bigM = 3000.0 #For the VaR version
    my_aggregator.max_DA = 2 * max(loads...)
    my_aggregator.max_RT = 0.5 * max(Iterators.flatten([errors_RT_train[j].values for j in range(1, 65, step = 1)])...) #Max error among the scenarios

#TESTS
NEW_in_DICT_CVAR = Dict()
NEW_out_DICT_CVAR = Dict()
# NEW_in_DICT_WCVAR = Dict()
# NEW_out_DICT_WCVAR = Dict()
#
# ADD_in_DICT_WCVAR = Dict()
# ADD_out_DICT_WCVAR = Dict()

#OR LOAD THE EXISTING RESULTS
# using JLD
# out_DICT_CVAR = load("data/results/WCVaR/out_DICT_CVAR.jld")["NEW_out_DICT_CVAR"]
# NEW_out_DICT_WCVAR = load("data/results/WCVaR/out_DICT_WCVAR.jld")["NEW_out_DICT_WCVAR"]
# NEW_in_DICT_WCVAR = load("data/results/WCVaR/in_DICT_WCVAR.jld")["NEW_in_DICT_WCVAR"]
#
# NEW_out_DICT_CVAR = load("data/results/WCVaR/out_DICT_CVAR.jld")["NEW_out_DICT_CVAR"]
# NEW_in_DICT_CVAR = load("data/results/WCVaR/in_DICT_CVAR.jld")["NEW_in_DICT_CVAR"]
#
# ADD_out_DICT_WCVAR = load("data/results/WCVaR/ADD_out_DICT_WCVAR.jld")["ADD_out_DICT_WCVAR"]
# ADD_in_DICT_WCVAR = load("data/results/WCVaR/ADD_in_DICT_WCVAR.jld")["ADD_in_DICT_WCVAR"]

sites_test = ["12", "58", "40", "17", "29", "70", "68"]
for s in sites_test
    in_results_CVAR =  DataFrame(ϵ = [], reliability = [], cost = [], numb_violation_UP = [],  numb_violation_DOWN = [], sum_UP = [], sum_DOWN = [], t_build = [], t_solve = [])
    out_results_CVAR = DataFrame(ϵ = [], numb_viol = [], iter = [], cost_AGG = [], max_upper = [], max_lower = [], sum_upper = [], sum_lower = [])
    rng = range(0.01, 0.1, step = 0.01)
    for epsilon in rng
            println("epsilon was $epsilon")
            # epsilon = 0.1
            my_aggregator.preferences.ϵ = epsilon
            (aggregator_model) , time_to_build = @timed build_model(my_aggregator, period, prices_train)
            #Some debugging options
            # aggregator_model.ext[:param][:scens_RT_dev][1][2].values[1,2]
            # aggregator_model.ext[:param][:group_RT_dev]
            # aggregator_model[:energy_balance]
            #
            # aggregator_model.ext[:param][:ϵ]
            #
            # aggregator_model.ext[:param][:scens_RT_market][1]

            optimizer = Gurobi.Optimizer
            max_time = 4000
            psolve = -1
            meth = -1
            #CVAR MODEL
            (solved_aggregator_model) , time_to_solve = @timed solve_model(aggregator_model, optimizer; timelimit = max_time, method = meth, presolve = psolve)
            in_res = read_result(solved_aggregator_model)
            push!(in_results_CVAR, [epsilon, in_res["reliability_in"], in_res["cost_AGG"], in_res["numb_viol_UP"], in_res["numb_viol_DOWN"], in_res["sum_balance_up"], in_res["sum_balance_down"], time_to_build, time_to_solve])

            #Copying the aggregator for the out-of-sample tests
            test_aggregator = deepcopy(my_aggregator)
            test_DER_agent = deepcopy(my_der_agent)
            test_aggregator.DER_agents[1].dev_RT
            test_DER_agent.dev_RT = []

            #Attaining test data
            error_dat_test = glob("*.csv", "data/WCVaR_dat/single_site/$s")
            err_scenarios_test = Dict()
            scen_err_RT_test = Dict()
            for d in 1:length(error_dat_test)
                err_scenario_test = DataFrame!(CSV.File(error_dat_test[d], header = true, decimal='.'))
                err_scenario_test = err_scenario_test[1:end, :].values .* loads
                err_scenarios_test[d] = err_scenario_test
                scen_err_RT_test[d] = Scenario(id = "RT_error_group_1", counter = d, prob = 1) #Equal probability
            end

            # results_CVAR = DataFrame(ϵ = [], iter = [], cost_AGG = [], max_upper = [], max_lower = [], sum_upper = [], sum_lower = [])

            for i in 1:length(error_dat_test)
                errors_RT_test = []
                errors_RT_test = Demand(period = period,  scenario = scen_err_RT_test[i], id = scen_err_RT_test[i].id)
                errors_RT_test.values[1,:] = err_scenarios_test[i]
                test_DER_agent.dev_RT = []
                test_DER_agent.dev_RT = [errors_RT_test]
                test_aggregator.DER_agents = [test_DER_agent]
                test_aggregator_model = build_test_model(test_aggregator, period, prices_train)
                ####FIX DA DECISIONS TO CVAR MODEL#########
                # fix_model_variable!(test_aggregator_model, :fᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :aᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:aᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :chᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :dchᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴰᴬ]))

                # fix_model_variable!(test_aggregator_model, :fᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴵᴰ]))
                # fix_model_variable!(test_aggregator_model, :chᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴵᴰ]))
                # fix_model_variable!(test_aggregator_model, :dchᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴵᴰ]))
                #Fixing everything except the imbalance market position
                # fix_model_variable!(test_aggregator_model, :soc, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:soc])[:,:,:,1,:,:])
                # fix_model_variable!(test_aggregator_model, :chᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴿᵀ])[:,:,:,1,:,:])
                # fix_model_variable!(test_aggregator_model, :dchᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴿᵀ])[:,:,:,1,:,:])
                fix_model_variable!(test_aggregator_model, :aᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:aᴿᵀ]))
                # fix_model_variable!(test_aggregator_model, :fᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴿᵀ]))

                test_solved_aggregator_model = solve_model(test_aggregator_model, optimizer, timelimit = max_time)
                term_stat = termination_status(test_solved_aggregator_model)

                # Gurobi.compute_conflict(test_solved_aggregator_model.moi_backend.optimizer.model)
                # MOI.get(test_solved_aggregator_model.moi_backend, Gurobi.ConstraintConflictStatus(), soc1.index)
                # test_solved_aggregator_model.computeIIS()
                # using Gurobi
                # compute_conflict!(test_solved_aggregator_model)
                # grb_model = test_solved_aggregator_model.moi_backend.optimizer.model
                # computeIIS(grb_model.inner)
                # test_solved_aggregator_model.feasRelax()
                # Gurobi.compute_conflict(grb_model)
                # feasRelax(grb_model.inner)
                # Gurobi.computeIIS(test_solved_aggregator_model)
                if term_stat == MOI.OPTIMAL || term_stat == MOI.LOCALLY_SOLVED
                    println("######The result was feasible at the $i-th iteration...####")
                    cost_AGG = JuMP.value(test_solved_aggregator_model[:true_cost_AGG])
                    println("######The cost was $cost_AGG...####")

                    res = read_test_result(test_solved_aggregator_model)
                    # res["sum_UP"]
                    # res["sum_DOWN"]
                    # res["max_UP"]
                    # res["max_DOWN"]
                    # real_cost = res["cost_AGG"]
                    push!(out_results_CVAR, [test_aggregator.preferences.ϵ, res["numb_viol"], i,  res["cost_AGG"], res["max_UP"],  res["max_DOWN"], res["sum_UP"], res["sum_DOWN"]])
                else
                    println("######The result was infeasible at the $i-th iteration...####")
                end
            end
    end

    NEW_in_DICT_CVAR["SITE$s","MIX2"] = in_results_CVAR
    NEW_out_DICT_CVAR["SITE$s","MIX2"] = out_results_CVAR
end

##########FROM HERE

rels_CVAR = Dict()
rels_WCVAR = Dict()
rels_norm_CVAR = Dict()
rels_norm_WCVAR = Dict()
cost_scaled_CVAR = Dict()
cost_scaled_WCVAR = Dict()
mean_costs_CVAR = Dict()
mean_costs_WCVAR = Dict()

mean_viol_CVAR_dict = Dict()
mean_viol_WCVAR_dict = Dict()
perc95_viol_CVAR_dict = Dict()
perc95_viol_WCVAR_dict = Dict()
add_perc95_viol_WCVAR_dict = Dict()

max_viol_CVAR_dict = Dict()
max_viol_WCVAR_dict = Dict()
add_max_viol_WCVAR_dict = Dict()

add_rels_WCVAR = Dict()
add_mean_costs_WCVAR = Dict()

numb_viol_CVAR_dict = Dict()
numb_viol_WCVAR_dict = Dict()
add_numb_viol_WCVAR_dict = Dict()
add_mean_viol_WCVAR_dict = Dict()

epsilons = rng = range(0.01, 0.1, step = 0.01)

sites  = ["70", "17", "68", "29", "12", "40", "58"]
mixes = ["MIX2"]

using PyCall, PyPlot
sns = pyimport("seaborn")
pygui(true)
plt.style.use("default")

for s in sites
    for m in mixes
        out_results_CVAR = NEW_out_DICT_CVAR["SITE$s","$m"]
        in_results_CVAR = NEW_in_DICT_CVAR["SITE$s","$m"]
        out_results_WCVAR = NEW_out_DICT_WCVAR["SITE$s","$m"]
        in_results_WCVAR = NEW_in_DICT_WCVAR["SITE$s","$m"]

        add_out_results_WCVAR = ADD_out_DICT_WCVAR["SITE$s","$m"]
        add_in_results_WCVAR = ADD_in_DICT_WCVAR["SITE$s","$m"]

        mean_cost = []
        mean_viol = []
        reliability_CVAR = []
        reliability_CVAR_perc_up = []
        reliability_CVAR_perc_down = []
        numb_viol_CVAR = []
        perc95_viol_CVAR = []
        max_viol_CVAR = []

        mean_cost_WCVAR = []
        mean_viol_WCVAR = []
        reliability_WCVAR = []
        reliability_WCVAR_perc_up = []
        reliability_WCVAR_perc_down = []
        numb_viol_WCVAR = []
        perc95_viol_WCVAR = []
        max_viol_WCVAR = []

        add_mean_cost_WCVAR = []
        add_mean_viol_WCVAR = []
        add_reliability_WCVAR = []
        add_reliability_WCVAR_perc_up = []
        add_reliability_WCVAR_perc_down = []
        add_numb_viol_WCVAR = []
        add_perc95_viol_WCVAR = []
        add_max_viol_WCVAR = []

        for eps in rng

            CVAR = out_results_CVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

            push!(mean_cost, mean(CVAR.cost_AGG))
            rel = (1 - sum(CVAR.numb_viol)/(length(CVAR.ϵ).*5 .* 96))
            push!(reliability_CVAR, rel)
            push!(reliability_CVAR_perc_up, percentile((1 .- (CVAR.numb_viol)./(length(CVAR.ϵ).*5 .* 96)), 95))
            push!(reliability_CVAR_perc_down, percentile((1 .- (CVAR.numb_viol)./(length(CVAR.ϵ).*5 .* 96)), 5))

            push!(numb_viol_CVAR, sum(CVAR.numb_viol))
            push!(perc95_viol_CVAR, percentile((CVAR.sum_upper + CVAR.sum_lower), 99))

            push!(mean_viol, mean(CVAR.sum_upper + CVAR.sum_lower))
            push!(max_viol_CVAR, max(CVAR.max_upper..., CVAR.max_lower...))

        end

        for eps in rng

            WCVAR = out_results_WCVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

            push!(mean_cost_WCVAR, mean(WCVAR.cost_AGG))
            rel = (1 - sum(WCVAR.numb_viol)/(length(WCVAR.ϵ).*5 .* 96))
            push!(reliability_WCVAR, rel)
            push!(reliability_WCVAR_perc_up, percentile((1 .- (WCVAR.numb_viol)./(length(WCVAR.ϵ).*5 .* 96)), 95))
            push!(reliability_WCVAR_perc_down, percentile((1 .- (WCVAR.numb_viol)./(length(WCVAR.ϵ).*5 .* 96)), 5))

            push!(numb_viol_WCVAR, sum(WCVAR.numb_viol))
            push!(mean_viol_WCVAR, mean(WCVAR.sum_upper + WCVAR.sum_lower))
            push!(perc95_viol_WCVAR, percentile((WCVAR.sum_upper + WCVAR.sum_lower), 99))
            push!(max_viol_WCVAR, max(WCVAR.max_upper..., WCVAR.max_lower...))

        end

        for eps in range(0.1, 0.17; step =0.01)

            add_WCVAR = add_out_results_WCVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

              push!(add_mean_cost_WCVAR, mean(add_WCVAR.cost_AGG))
              rel = (1 - sum(add_WCVAR.numb_viol)/(length(add_WCVAR.ϵ).*5 .* 96))
              push!(add_reliability_WCVAR, rel)
              push!(add_reliability_WCVAR_perc_up, percentile((1 .- (add_WCVAR.numb_viol)./(length(add_WCVAR.ϵ).*5 .* 96)), 95))
              push!(add_reliability_WCVAR_perc_down, percentile((1 .- (add_WCVAR.numb_viol)./(length(add_WCVAR.ϵ).*5 .* 96)), 5))
              push!(add_mean_viol_WCVAR, mean(add_WCVAR.sum_upper + add_WCVAR.sum_lower))

              push!(add_numb_viol_WCVAR, sum(add_WCVAR.numb_viol))
              push!(add_perc95_viol_WCVAR, percentile((add_WCVAR.sum_upper + add_WCVAR.sum_lower), 99))
              push!(add_max_viol_WCVAR, max(add_WCVAR.max_upper..., add_WCVAR.max_lower...))

        end

        mean_costs_CVAR["SITE$s","$m"] = -mean_cost
        mean_costs_WCVAR["SITE$s","$m"] = -mean_cost_WCVAR
        add_mean_costs_WCVAR["SITE$s","$m"] = -add_mean_cost_WCVAR

        # cost_scaled_CVAR["SITE$s","$m"]  = scaled_cost_CVAR
        # cost_scaled_WCVAR["SITE$s","$m"]  = scaled_cost_WCVAR

        rels_CVAR["SITE$s","$m"] = reliability_CVAR
        rels_WCVAR["SITE$s","$m"] = reliability_WCVAR
        add_rels_WCVAR["SITE$s","$m"] = add_reliability_WCVAR

        numb_viol_CVAR_dict["SITE$s","$m"] = numb_viol_CVAR
        numb_viol_WCVAR_dict["SITE$s","$m"] = numb_viol_WCVAR
        add_numb_viol_WCVAR_dict["SITE$s","$m"] = add_numb_viol_WCVAR

        mean_viol_CVAR_dict["SITE$s","$m"] = mean_viol
        mean_viol_WCVAR_dict["SITE$s","$m"] = mean_viol_WCVAR
        add_mean_viol_WCVAR_dict["SITE$s","$m"] = add_mean_viol_WCVAR

        perc95_viol_CVAR_dict["SITE$s","$m"] = perc95_viol_CVAR
        perc95_viol_WCVAR_dict["SITE$s","$m"] = perc95_viol_WCVAR
        add_perc95_viol_WCVAR_dict["SITE$s","$m"] = add_perc95_viol_WCVAR

        max_viol_CVAR_dict["SITE$s","$m"] = max_viol_CVAR
        max_viol_WCVAR_dict["SITE$s","$m"] = max_viol_WCVAR
        add_max_viol_WCVAR_dict["SITE$s","$m"] = add_max_viol_WCVAR

        # rels_norm_CVAR["SITE$s","$m"] = rel_out_vs_in_CVAR
        # rels_norm_WCVAR["SITE$s","$m"] = rel_out_vs_in_WCVAR
    end
end

cvar = NEW_out_DICT_CVAR["SITE29","MIX2"] |>
  @filter(_.cost_AGG <= -440) |>
  DataFrame

wcvar = NEW_out_DICT_WCVAR["SITE29","MIX2"] |>
    @filter(_.cost_AGG <= -440) |>
    DataFrame

add_wcvar = ADD_out_DICT_WCVAR["SITE29","MIX2"] |>
    @filter(_.cost_AGG <= -440) |>
    DataFrame

#adding up the dataframes
wcvar = [wcvar; add_wcvar]

sns.set_context("paper", font_scale=1.8)
sns.set_style("whitegrid")
sns.set_style("ticks")
sns.color_palette()

p = sns.JointGrid()
sns.kdeplot(color = "tab:blue", x=-wcvar.cost_AGG[:], y=max((wcvar.max_upper, wcvar.max_lower)...),  ax=p.ax_joint, alpha = 0.8, stat = "density", common_norm = true, label = "WCVaR")
sns.kdeplot(color = "tab:orange", x=-cvar.cost_AGG[:], y=max((cvar.max_upper, cvar.max_lower)...), ax=p.ax_joint, alpha = 0.8, stat = "density", common_norm = true, label = "CVaR")

sns.histplot(color = "tab:blue", y=max((wcvar.max_upper, wcvar.max_lower)...),  ax=p.ax_marg_y, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:orange", y=max((cvar.max_upper, cvar.max_lower)...),  ax=p.ax_marg_y, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:blue", x=-wcvar.cost_AGG[:],  ax=p.ax_marg_x, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:orange", x=-cvar.cost_AGG[:],  ax=p.ax_marg_x, element="step", stat = "density", alpha = 0.5)

# plt.legend()
plt.ylim(0.0, 1.5)
mpatches = pyimport("matplotlib.patches")
blue_patch = mpatches.Patch(color="tab:blue", alpha = 0.5, label="WCVaR")
red_patch = mpatches.Patch(color="tab:orange", alpha = 0.5, label="CVaR")

text = mpatches.Patch(label = "29, d=0.501",fc="w", fill=false, edgecolor="none", linewidth=0)

p.ax_joint.legend(handles=[blue_patch, red_patch, text], ncol =1, loc = "upper right")


text:
function get_new_pots(pots, maps)
    new_pots = copy(pots)
    for i in 3:length(pots)-2
        key = pots[i-2:i+2]
        if key in keys(maps)
            new_pots[i] = maps[key]
        else
            new_pots[i] = 0
        end
    end
    return new_pots
end

function get_summed(pots, offset)

    summed = 0
    for (i, v) in enumerate(pots)
        if v == 1
            summed += i - offset - 1
        end
    end
    return summed
end

function foo()
    lines = readlines(open("data/12.txt", "r"))
    idx = findfirst(isequal(':'), lines[1]) + 2
    x = lines[1][idx:end]
    offset = 1000
    pots = zeros(Int, length(x) + 2 * offset)
    for (i, c) in enumerate(x)
        if c == '#'
            pots[i + offset] = 1
        end
    end
    maps = Dict{Array{Int, 1}, Int}()
    for line in lines[3:end]
        z = zeros(5)
        label = 0
        for (i, c) in enumerate(line[1:5])
            if c == '#'
                z[i] = 1
            end
        end
        if line[end] == '#'
            label = 1
        end
        maps[z] = label
    end

    summs = Array{Int, 1}()
    for q in 1:200
        pots = get_new_pots(pots, maps)
        xs = get_summed(pots, offset)
        push!(summs, xs)
    end
    additive = diff(summs)[end]
    return (50_000_000_000 - 200) * additive + get_summed(pots, offset)
end

println(foo())


text:
##
function badvar1(x::Vector{Float64})
ex2 = 0.0
ex = 0.0
for i=1:length(x)
  ex2 = ex2 + x[i]^2
  ex = ex + x[i]
end
n = length(x)
return 1.0/(n-1)*(ex2 - (ex)^2/n)
end
@show badvar1([0.0,1,2])
@show badvar1([0.0,1,2].+1.0e8)
##
using Random
Random.seed!(2)
x = randn(100)
@show badvar1(x)
@show badvar1(x.+1.0e8)
##
function goodvar(x::Vector{Float64})
  n = length(x); mean = 0.0; m2 = 0.0; N = 0.0
  for i=1:n
    N = N + 1 # running length
    delta = x[i] - mean
    mean = mean + delta/N
    m2 = m2 + delta*(x[i]-mean)
  end
  return m2/(n-1)
end
@show goodvar([0.0,1,2])
@show goodvar([0.0,1,2].+1.0e8)
##
using Random
Random.seed!(2)
x = randn(100)
@show goodvar(x)
@show goodvar(x.+1.0e8)


text:
using Plots: guidefont, series_annotations, PLOTS_SEED

using VisualRegressionTests
using RecipesBase
using StableRNGs
using TestImages
using LibGit2
using Random
using FileIO
using Plots
using Dates
using JSON
using Test
using Gtk

import GeometryBasics
import ImageMagick

@testset "Infrastructure" begin
    @test_nowarn JSON.Parser.parse(
        String(read(joinpath(dirname(pathof(Plots)), "..", ".zenodo.json"))),
    )
end

@testset "Plotly standalone" begin
    @test_nowarn Plots._init_ijulia_plotting()
    @test Plots.plotly_local_file_path[] === nothing
    temp = Plots.use_local_dependencies[]
    withenv("PLOTS_HOST_DEPENDENCY_LOCAL" => true) do
        Plots.__init__()
        @test Plots.plotly_local_file_path[] isa String
        @test isfile(Plots.plotly_local_file_path[])
        @test Plots.use_local_dependencies[] = true
        @test_nowarn Plots._init_ijulia_plotting()
    end
    Plots.plotly_local_file_path[] = nothing
    Plots.use_local_dependencies[] = temp
end

include("test_defaults.jl")
include("test_pipeline.jl")
include("test_axes.jl")
include("test_contours.jl")
include("test_axis_letter.jl")
include("test_components.jl")
include("test_shorthands.jl")
include("integration_dates.jl")
include("test_recipes.jl")
include("test_hdf5plots.jl")
include("test_pgfplotsx.jl")
include("test_plotly.jl")

reference_dir(args...) =
    joinpath(homedir(), ".julia", "dev", "PlotReferenceImages", args...)

function reference_file(backend, i, version)
    refdir = reference_dir("Plots", string(backend))
    fn = "ref$i.png"
    versions = sort(VersionNumber.(readdir(refdir)), rev = true)
    reffn = joinpath(refdir, string(version), fn)
    for v in versions
        tmpfn = joinpath(refdir, string(v), fn)
        if isfile(tmpfn)
            reffn = tmpfn
            break
        end
    end
    return reffn
end

reference_path(backend, version) = reference_dir("Plots", string(backend), string(version))

if !isdir(reference_dir())
    mkpath(reference_dir())
    LibGit2.clone(
        "https://github.com/JuliaPlots/PlotReferenceImages.jl.git",
        reference_dir(),
    )
end

include("imgcomp.jl")
Random.seed!(PLOTS_SEED)

default(show = false, reuse = true)  # don't actually show the plots

is_ci() = get(ENV, "CI", "false") == "true"
const PLOTS_IMG_TOL = parse(Float64, get(ENV, "PLOTS_IMG_TOL", is_ci() ? "1e-4" : "1e-5"))

## Uncomment the following lines to update reference images for different backends

# @testset "GR" begin
#     image_comparison_facts(:gr, tol=PLOTS_IMG_TOL, skip = Plots._backend_skips[:gr])
# end
#
# plotly()
# @testset "Plotly" begin
#     image_comparison_facts(:plotly, tol=PLOTS_IMG_TOL, skip = Plots._backend_skips[:plotlyjs])
# end
#
# pyplot()
# @testset "PyPlot" begin
#     image_comparison_facts(:pyplot, tol=PLOTS_IMG_TOL, skip = Plots._backend_skips[:pyplot])
# end
#
# pgfplotsx()
# @testset "PGFPlotsX" begin
#     image_comparison_facts(:pgfplotsx, tol=PLOTS_IMG_TOL, skip = Plots._backend_skips[:pgfplotsx])
# end

##

@testset "Axes" begin
    p = plot()
    axis = p.subplots[1][:xaxis]
    @test typeof(axis) == Plots.Axis
    @test Plots.discrete_value!(axis, "HI") == (0.5, 1)
    @test Plots.discrete_value!(axis, :yo) == (1.5, 2)
    @test Plots.ignorenan_extrema(axis) == (0.5, 1.5)
    @test axis[:discrete_map] == Dict{Any,Any}(:yo => 2, "HI" => 1)

    Plots.discrete_value!(axis, ["x$i" for i in 1:5])
    Plots.discrete_value!(axis, ["x$i" for i in 0:2])
    @test Plots.ignorenan_extrema(axis) == (0.5, 7.5)
end

@testset "NoFail" begin
    # ensure backend with tested display
    @test unicodeplots() == Plots.UnicodePlotsBackend()
    @test backend() == Plots.UnicodePlotsBackend()

    dsp = TextDisplay(IOContext(IOBuffer(), :color => true))

    @testset "Plot" begin
        plots = [
            histogram([1, 0, 0, 0, 0, 0]),
            plot([missing]),
            plot([missing, missing]),
            plot(fill(missing, 10)),
            plot([missing; 1:4]),
            plot([fill(missing, 10); 1:4]),
            plot([1 1; 1 missing]),
            plot(["a" "b"; missing "d"], [1 2; 3 4]),
        ]
        for plt in plots
            display(dsp, plt)
        end
        @test_nowarn plot(x -> x^2, 0, 2)
    end

    @testset "Bar" begin
        p = bar([3, 2, 1], [1, 2, 3])
        @test p isa Plots.Plot
        @test display(dsp, p) isa Nothing
    end
end

@testset "EmptyAnim" begin
    anim = @animate for i in []
    end

    @test_throws ArgumentError gif(anim)
end

@testset "NaN-separated Segments" begin
    segments(args...) = collect(iter_segments(args...))

    nan10 = fill(NaN, 10)
    @test segments(11:20) == [1:10]
    @test segments([NaN]) == []
    @test segments(nan10) == []
    @test segments([nan10; 1:5]) == [11:15]
    @test segments([1:5; nan10]) == [1:5]
    @test segments([nan10; 1:5; nan10; 1:5; nan10]) == [11:15, 26:30]
    @test segments([NaN; 1], 1:10) == [2:2, 4:4, 6:6, 8:8, 10:10]
    @test segments([nan10; 1:15], [1:15; nan10]) == [11:15]
end

@testset "Utils" begin
    zipped = (
        [(1, 2)],
        [("a", "b")],
        [(1, "a"), (2, "b")],
        [(1, 2), (3, 4)],
        [(1, 2, 3), (3, 4, 5)],
        [(1, 2, 3, 4), (3, 4, 5, 6)],
        [(1, 2.0), (missing, missing)],
        [(1, missing), (missing, "a")],
        [(missing, missing)],
        [(missing, missing, missing), ("a", "b", "c")],
    )
    for z in zipped
        @test isequal(collect(zip(Plots.unzip(z)...)), z)
        @test isequal(collect(zip(Plots.unzip(GeometryBasics.Point.(z))...)), z)
    end
    op1 = Plots.process_clims((1.0, 2.0))
    op2 = Plots.process_clims((1, 2.0))
    data = randn(100, 100)
    @test op1(data) == op2(data)
    @test Plots.process_clims(nothing) ==
          Plots.process_clims(missing) ==
          Plots.process_clims(:auto)

    @test (==)(
        Plots.texmath2unicode(
            raw"Equation $y = \alpha \cdot x + \beta$ and eqn $y = \sin(x)^2$",
        ),
        raw"Equation y = α ⋅ x + β and eqn y = sin(x)²",
    )
end

@testset "Backends" begin
    @testset "UnicodePlots" begin
        @test unicodeplots() == Plots.UnicodePlotsBackend()
        @test backend() == Plots.UnicodePlotsBackend()

        io = IOContext(IOBuffer(), :color => true)

        # lets just make sure it runs without error
        p = plot(rand(10))
        @test p isa Plots.Plot
        @test show(io, p) isa Nothing
        p = bar(randn(10))
        @test p isa Plots.Plot
        @test show(io, p) isa Nothing
        p = plot([1, 2], [3, 4])
        annotate!(p, [(1.5, 3.2, Plots.text("Test", :red, :center))])
        hline!(p, [3.1])
        @test p isa Plots.Plot
        @test show(io, p) isa Nothing
        p = plot([Dates.Date(2019, 1, 1), Dates.Date(2019, 2, 1)], [3, 4])
        hline!(p, [3.1])
        annotate!(p, [(Dates.Date(2019, 1, 15), 3.2, Plots.text("Test", :red, :center))])
        @test p isa Plots.Plot
        @test show(io, p) isa Nothing
        p = plot([Dates.Date(2019, 1, 1), Dates.Date(2019, 2, 1)], [3, 4])
        annotate!(p, [(Dates.Date(2019, 1, 15), 3.2, :auto)])
        hline!(p, [3.1])
        @test p isa Plots.Plot
        @test show(io, p) isa Nothing
    end

    @testset "PlotlyJS" begin
        @test plotlyjs() == Plots.PlotlyJSBackend()
        @test backend() == Plots.PlotlyJSBackend()

        p = plot(rand(10))
        @test p isa Plots.Plot
        @test_broken display(p) isa Nothing
    end

    @testset "GR" begin
        ENV["PLOTS_TEST"] = "true"
        ENV["GKSwstype"] = "100"
        @test gr() == Plots.GRBackend()
        @test backend() == Plots.GRBackend()

        @static if haskey(ENV, "APPVEYOR")
            @info "Skipping GR image comparison tests on AppVeyor"
        else
            image_comparison_facts(
                :gr,
                tol = PLOTS_IMG_TOL,
                skip = Plots._backend_skips[:gr],
            )
        end
    end
end


text:
mutable struct Bitboard
    white :: UInt64 # all white pieces
    P :: Array{UInt64,1}
    R :: Array{UInt64,1}
    N :: Array{UInt64,1}
    B :: Array{UInt64,1}
    Q :: Array{UInt64,1}
    K :: UInt64
    A :: Array{UInt64,1}

    black :: UInt64 # all black pieces
    p :: Array{UInt64,1}
    r :: Array{UInt64,1}
    n :: Array{UInt64,1}
    b :: Array{UInt64,1}
    q :: Array{UInt64,1}
    k :: UInt64
    a :: Array{UInt64,1}

    free :: UInt64  # all free squares
    taken :: UInt64 # all pieces

    white_attacks :: UInt64 # attacked squares
    black_attacks :: UInt64

    player_color :: String

    white_can_castle_queenside :: Bool
    white_can_castle_kingside  :: Bool
    black_can_castle_queenside :: Bool
    black_can_castle_kingside  :: Bool
    white_king_moved :: Bool
    black_king_moved :: Bool

    enpassant_square :: UInt64
    enpassant_done :: Bool

    halfmove_clock :: Int64
    fullmove_clock :: Int64 # start at 1, increment for each black move

    fen :: String
    game :: Array{String,1}
end

function set_board()
    return fen_to_bitboard(
        "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1")
end


text:
#!/usr/bin/env julia

println("hi")

max_stars_repo_name: AbirRazzak/FutureBot

max_issues_repo_name: AbirRazzak/FutureBot

max_forks_repo_name: AbirRazzak/FutureBot


text:
# This file is part of Kpax3. License is MIT.

# Tell Plots/GR that we are in a headless environment
ENV["GKSwstype"] = "100"
ENV["PLOTS_TEST"] = "true"

using Distances
using Plots
using Printf
using Random
using SpecialFunctions
using Statistics
using StatsBase
using Test

gr(size=(800, 600))

using Kpax3

ε = 1.0e-13
Random.seed!(1427371200)

function runtests()
  # this module is used by "model/partition_rows" tests
  include("data/partitions.jl")

  tests = [
    "data_processing/fasta_data_processing";
    "data_processing/csv_data_processing";
    "distances/simovici_jaroszewicz";
    "distances/jaccard";
    "misc/basic_functions";
    "misc/partition_functions";
    "types/settings";
    "types/data";
    "types/prior_col";
    "types/prior_row";
    "types/state";
    "types/state_list";
    "types/support";
    "model/likelihoods";
    "model/partition_cols";
    "model/partition_rows";
    "model/loss_binder"
    "optimizer/local_mode";
    "optimizer/selection";
    "optimizer/crossover";
    "optimizer/mutation";
    "mcmc/partition_ratios";
    "mcmc/log_likelihoods";
    "mcmc/weight";
    "mcmc/merge";
    "mcmc/split";
    "mcmc/gibbs";
    "mcmc/biased_random_walk";
    "mcmc/posterior";
    "mcmc/diagnostics";
    "estimate/write";
    "plots/plots"
  ]

  for t in tests
    f = string(t, ".jl")
    @printf("Going through tests in '%s'... ", f)
    include(f)
    @printf("PASSED!\n")
  end

  nothing
end

runtests()


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

## Code for searching and viewing documentation

using Markdown

using Base.Docs: catdoc, modules, DocStr, Binding, MultiDoc, keywords, isfield, namify, bindingexpr,
    defined, resolve, getdoc, meta, aliasof, signature

import Base.Docs: doc, formatdoc, parsedoc, apropos

using Base: with_output_color, mapany

import REPL

using InteractiveUtils: subtypes

using Unicode: normalize

## Help mode ##

# This is split into helpmode and _helpmode to easier unittest _helpmode
helpmode(io::IO, line::AbstractString) = :($REPL.insert_hlines($io, $(REPL._helpmode(io, line))))
helpmode(line::AbstractString) = helpmode(stdout, line)

const extended_help_on = Ref{Any}(nothing)

function _helpmode(io::IO, line::AbstractString)
    line = strip(line)
    ternary_operator_help = (line == "?" || line == "?:")
    if startswith(line, '?') && !ternary_operator_help
        line = line[2:end]
        extended_help_on[] = line
        brief = false
    else
        extended_help_on[] = nothing
        brief = true
    end
    # interpret anything starting with # or #= as asking for help on comments
    if startswith(line, "#")
        if startswith(line, "#=")
            line = "#="
        else
            line = "#"
        end
    end
    x = Meta.parse(line, raise = false, depwarn = false)
    assym = Symbol(line)
    expr =
        if haskey(keywords, Symbol(line)) || Base.isoperator(assym) || isexpr(x, :error) ||
            isexpr(x, :invalid) || isexpr(x, :incomplete)
            # Docs for keywords must be treated separately since trying to parse a single
            # keyword such as `function` would throw a parse error due to the missing `end`.
            assym
        elseif isexpr(x, (:using, :import))
            (x::Expr).head
        else
            # Retrieving docs for macros requires us to make a distinction between the text
            # `@macroname` and `@macroname()`. These both parse the same, but are used by
            # the docsystem to return different results. The first returns all documentation
            # for `@macroname`, while the second returns *only* the docs for the 0-arg
            # definition if it exists.
            (isexpr(x, :macrocall, 1) && !endswith(line, "()")) ? quot(x) : x
        end
    # the following must call repl(io, expr) via the @repl macro
    # so that the resulting expressions are evaluated in the Base.Docs namespace
    :($REPL.@repl $io $expr $brief)
end
_helpmode(line::AbstractString) = _helpmode(stdout, line)

# Print vertical lines along each docstring if there are multiple docs
function insert_hlines(io::IO, docs)
    if !isa(docs, Markdown.MD) || !haskey(docs.meta, :results) || isempty(docs.meta[:results])
        return docs
    end
    docs = docs::Markdown.MD
    v = Any[]
    for (n, doc) in enumerate(docs.content)
        push!(v, doc)
        n == length(docs.content) || push!(v, Markdown.HorizontalRule())
    end
    return Markdown.MD(v)
end

function formatdoc(d::DocStr)
    buffer = IOBuffer()
    for part in d.text
        formatdoc(buffer, d, part)
    end
    Markdown.MD(Any[Markdown.parse(seekstart(buffer))])
end
@noinline formatdoc(buffer, d, part) = print(buffer, part)

function parsedoc(d::DocStr)
    if d.object === nothing
        md = formatdoc(d)
        md.meta[:module] = d.data[:module]
        md.meta[:path]   = d.data[:path]
        d.object = md
    end
    d.object
end

## Trimming long help ("# Extended help")

struct Message  # For direct messages to the terminal
    msg    # AbstractString
    fmt    # keywords to `printstyled`
end
Message(msg) = Message(msg, ())

function Markdown.term(io::IO, msg::Message, columns)
    printstyled(io, msg.msg; msg.fmt...)
end

trimdocs(doc, brief::Bool) = doc

function trimdocs(md::Markdown.MD, brief::Bool)
    brief || return md
    md, trimmed = _trimdocs(md, brief)
    if trimmed
        line = extended_help_on[]
        line = isa(line, AbstractString) ? line : ""
        push!(md.content, Message("Extended help is available with `??$line`", (color=Base.info_color(), bold=true)))
    end
    return md
end

function _trimdocs(md::Markdown.MD, brief::Bool)
    content, trimmed = [], false
    for c in md.content
        if isa(c, Markdown.Header{1}) && isa(c.text, AbstractArray) && !isempty(c.text)
            item = c.text[1]
            if isa(item, AbstractString) &&
                lowercase(item) ∈ ("extended help",
                                   "extended documentation",
                                   "extended docs")
                trimmed = true
                break
            end
        end
        c, trm = _trimdocs(c, brief)
        trimmed |= trm
        push!(content, c)
    end
    return Markdown.MD(content, md.meta), trimmed
end

_trimdocs(md, brief::Bool) = md, false

"""
    Docs.doc(binding, sig)

Return all documentation that matches both `binding` and `sig`.

If `getdoc` returns a non-`nothing` result on the value of the binding, then a
dynamic docstring is returned instead of one based on the binding itself.
"""
function doc(binding::Binding, sig::Type = Union{})
    if defined(binding)
        result = getdoc(resolve(binding), sig)
        result === nothing || return result
    end
    results, groups = DocStr[], MultiDoc[]
    # Lookup `binding` and `sig` for matches in all modules of the docsystem.
    for mod in modules
        dict = meta(mod)
        if haskey(dict, binding)
            multidoc = dict[binding]
            push!(groups, multidoc)
            for msig in multidoc.order
                sig <: msig && push!(results, multidoc.docs[msig])
            end
        end
    end
    if isempty(groups)
        # When no `MultiDoc`s are found that match `binding` then we check whether `binding`
        # is an alias of some other `Binding`. When it is we then re-run `doc` with that
        # `Binding`, otherwise if it's not an alias then we generate a summary for the
        # `binding` and display that to the user instead.
        alias = aliasof(binding)
        alias == binding ? summarize(alias, sig) : doc(alias, sig)
    else
        # There was at least one match for `binding` while searching. If there weren't any
        # matches for `sig` then we concatenate *all* the docs from the matching `Binding`s.
        if isempty(results)
            for group in groups, each in group.order
                push!(results, group.docs[each])
            end
        end
        # Get parsed docs and concatenate them.
        md = catdoc(mapany(parsedoc, results)...)
        # Save metadata in the generated markdown.
        if isa(md, Markdown.MD)
            md.meta[:results] = results
            md.meta[:binding] = binding
            md.meta[:typesig] = sig
        end
        return md
    end
end

# Some additional convenience `doc` methods that take objects rather than `Binding`s.
doc(obj::UnionAll) = doc(Base.unwrap_unionall(obj))
doc(object, sig::Type = Union{}) = doc(aliasof(object, typeof(object)), sig)
doc(object, sig...)              = doc(object, Tuple{sig...})

function lookup_doc(ex)
    if isa(ex, Expr) && ex.head !== :(.) && Base.isoperator(ex.head)
        # handle syntactic operators, e.g. +=, ::, .=
        ex = ex.head
    end
    if haskey(keywords, ex)
        return parsedoc(keywords[ex])
    elseif Meta.isexpr(ex, :incomplete)
        return :($(Markdown.md"No documentation found."))
    elseif !isa(ex, Expr) && !isa(ex, Symbol)
        return :($(doc)($(typeof)($(esc(ex)))))
    end
    if isa(ex, Symbol) && Base.isoperator(ex)
        str = string(ex)
        isdotted = startswith(str, ".")
        if endswith(str, "=") && Base.operator_precedence(ex) == Base.prec_assignment && ex !== :(:=)
            op = chop(str)
            eq = isdotted ? ".=" : "="
            return Markdown.parse("`x $op= y` is a synonym for `x $eq x $op y`")
        elseif isdotted && ex !== :(..)
            op = str[2:end]
            return Markdown.parse("`x $ex y` is akin to `broadcast($op, x, y)`. See [`broadcast`](@ref).")
        end
    end
    binding = esc(bindingexpr(namify(ex)))
    if isexpr(ex, :call) || isexpr(ex, :macrocall)
        sig = esc(signature(ex))
        :($(doc)($binding, $sig))
    else
        :($(doc)($binding))
    end
end

# Object Summaries.
# =================

function summarize(binding::Binding, sig)
    io = IOBuffer()
    if defined(binding)
        binding_res = resolve(binding)
        !isa(binding_res, Module) && println(io, "No documentation found.\n")
        summarize(io, binding_res, binding)
    else
        println(io, "No documentation found.\n")
        quot = any(isspace, sprint(print, binding)) ? "'" : ""
        println(io, "Binding ", quot, "`", binding, "`", quot, " does not exist.")
    end
    md = Markdown.parse(seekstart(io))
    # Save metadata in the generated markdown.
    md.meta[:results] = DocStr[]
    md.meta[:binding] = binding
    md.meta[:typesig] = sig
    return md
end

function summarize(io::IO, λ::Function, binding::Binding)
    kind = startswith(string(binding.var), '@') ? "macro" : "`Function`"
    println(io, "`", binding, "` is a ", kind, ".")
    println(io, "```\n", methods(λ), "\n```")
end

function summarize(io::IO, TT::Type, binding::Binding)
    println(io, "# Summary")
    T = Base.unwrap_unionall(TT)
    if T isa DataType
        println(io, "```")
        print(io,
            Base.isabstracttype(T) ? "abstract type " :
            Base.ismutabletype(T)  ? "mutable struct " :
            Base.isstructtype(T) ? "struct " :
            "primitive type ")
        supert = supertype(T)
        println(io, T)
        println(io, "```")
        if !Base.isabstracttype(T) && T.name !== Tuple.name && !isempty(fieldnames(T))
            println(io, "# Fields")
            println(io, "```")
            pad = maximum(length(string(f)) for f in fieldnames(T))
            for (f, t) in zip(fieldnames(T), fieldtypes(T))
                println(io, rpad(f, pad), " :: ", t)
            end
            println(io, "```")
        end
        subt = subtypes(TT)
        if !isempty(subt)
            println(io, "# Subtypes")
            println(io, "```")
            for t in subt
                println(io, Base.unwrap_unionall(t))
            end
            println(io, "```")
        end
        if supert != Any
            println(io, "# Supertype Hierarchy")
            println(io, "```")
            Base.show_supertypes(io, T)
            println(io)
            println(io, "```")
        end
    elseif T isa Union
        println(io, "`", binding, "` is of type `", typeof(TT), "`.\n")
        println(io, "# Union Composed of Types")
        for T1 in Base.uniontypes(T)
            println(io, " - `", Base.rewrap_unionall(T1, TT), "`")
        end
    else # unreachable?
        println(io, "`", binding, "` is of type `", typeof(TT), "`.\n")
    end
end

function find_readme(m::Module)::Union{String, Nothing}
    mpath = pathof(m)
    isnothing(mpath) && return nothing
    !isfile(mpath) && return nothing # modules in sysimage, where src files are omitted
    path = dirname(mpath)
    top_path = pkgdir(m)
    while true
        for file in readdir(path; join=true, sort=true)
            isfile(file) && (basename(lowercase(file)) in ["readme.md", "readme"]) || continue
            return file
        end
        path == top_path && break # go no further than pkgdir
        path = dirname(path) # work up through nested modules
    end
    return nothing
end
function summarize(io::IO, m::Module, binding::Binding; nlines::Int = 200)
    readme_path = find_readme(m)
    if isnothing(readme_path)
        println(io, "No docstring or readme file found for module `$m`.\n")
    else
        println(io, "No docstring found for module `$m`.")
    end
    exports = filter!(!=(nameof(m)), names(m))
    if isempty(exports)
        println(io, "Module does not export any names.")
    else
        println(io, "# Exported names")
        print(io, "  `")
        join(io, exports, "`, `")
        println(io, "`\n")
    end
    if !isnothing(readme_path)
        readme_lines = readlines(readme_path)
        isempty(readme_lines) && return  # don't say we are going to print empty file
        println(io, "# Displaying contents of readme found at `$(readme_path)`")
        for line in first(readme_lines, nlines)
            println(io, line)
        end
        length(readme_lines) > nlines && println(io, "\n[output truncated to first $nlines lines]")
    end
end

function summarize(io::IO, @nospecialize(T), binding::Binding)
    T = typeof(T)
    println(io, "`", binding, "` is of type `", T, "`.\n")
    summarize(io, T, binding)
end

# repl search and completions for help

quote_spaces(x) = any(isspace, x) ? "'" * x * "'" : x

function repl_search(io::IO, s::Union{Symbol,String})
    pre = "search:"
    print(io, pre)
    printmatches(io, s, map(quote_spaces, doc_completions(s)), cols = _displaysize(io)[2] - length(pre))
    println(io, "\n")
end
repl_search(s) = repl_search(stdout, s)

function repl_corrections(io::IO, s)
    print(io, "Couldn't find ")
    quot = any(isspace, s) ? "'" : ""
    print(io, quot)
    printstyled(io, s, color=:cyan)
    print(io, quot, '\n')
    print_correction(io, s)
end
repl_corrections(s) = repl_corrections(stdout, s)

# inverse of latex_symbols Dict, lazily created as needed
const symbols_latex = Dict{String,String}()
function symbol_latex(s::String)
    if isempty(symbols_latex) && isassigned(Base.REPL_MODULE_REF)
        for (k,v) in Iterators.flatten((REPLCompletions.latex_symbols,
                                        REPLCompletions.emoji_symbols))
            symbols_latex[v] = k
        end

        # Overwrite with canonical mapping when a symbol has several completions (#39148)
        merge!(symbols_latex, REPLCompletions.symbols_latex_canonical)
    end

    return get(symbols_latex, s, "")
end
function repl_latex(io::IO, s0::String)
    # This has rampant `Core.Box` problems (#15276). Use the tricks of
    # https://docs.julialang.org/en/v1/manual/performance-tips/#man-performance-captured
    # We're changing some of the values so the `let` trick isn't applicable.
    s::String = s0
    latex::String = symbol_latex(s)
    if isempty(latex)
        # Decompose NFC-normalized identifier to match tab-completion
        # input if the first search came up empty.
        s = normalize(s, :NFD)
        latex = symbol_latex(s)
    end
    if !isempty(latex)
        print(io, "\"")
        printstyled(io, s, color=:cyan)
        print(io, "\" can be typed by ")
        printstyled(io, latex, "<tab>", color=:cyan)
        println(io, '\n')
    elseif any(c -> haskey(symbols_latex, string(c)), s)
        print(io, "\"")
        printstyled(io, s, color=:cyan)
        print(io, "\" can be typed by ")
        state::Char = '\0'
        with_output_color(:cyan, io) do io
            for c in s
                cstr = string(c)
                if haskey(symbols_latex, cstr)
                    latex = symbols_latex[cstr]
                    if length(latex) == 3 && latex[2] in ('^','_')
                        # coalesce runs of sub/superscripts
                        if state != latex[2]
                            '\0' != state && print(io, "<tab>")
                            print(io, latex[1:2])
                            state = latex[2]
                        end
                        print(io, latex[3])
                    else
                        if '\0' != state
                            print(io, "<tab>")
                            state = '\0'
                        end
                        print(io, latex, "<tab>")
                    end
                else
                    if '\0' != state
                        print(io, "<tab>")
                        state = '\0'
                    end
                    print(io, c)
                end
            end
            '\0' != state && print(io, "<tab>")
        end
        println(io, '\n')
    end
end
repl_latex(s::String) = repl_latex(stdout, s)

macro repl(ex, brief::Bool=false) repl(ex; brief=brief) end
macro repl(io, ex, brief) repl(io, ex; brief=brief) end

function repl(io::IO, s::Symbol; brief::Bool=true)
    str = string(s)
    quote
        repl_latex($io, $str)
        repl_search($io, $str)
        $(if !isdefined(Main, s) && !haskey(keywords, s) && !Base.isoperator(s)
               :(repl_corrections($io, $str))
          end)
        $(_repl(s, brief))
    end
end
isregex(x) = isexpr(x, :macrocall, 3) && x.args[1] === Symbol("@r_str") && !isempty(x.args[3])
repl(io::IO, ex::Expr; brief::Bool=true) = isregex(ex) ? :(apropos($io, $ex)) : _repl(ex, brief)
repl(io::IO, str::AbstractString; brief::Bool=true) = :(apropos($io, $str))
repl(io::IO, other; brief::Bool=true) = esc(:(@doc $other))
#repl(io::IO, other) = lookup_doc(other) # TODO

repl(x; brief::Bool=true) = repl(stdout, x; brief=brief)

function _repl(x, brief::Bool=true)
    if isexpr(x, :call)
        x = x::Expr
        # determine the types of the values
        kwargs = nothing
        pargs = Any[]
        for arg in x.args[2:end]
            if isexpr(arg, :parameters)
                kwargs = mapany(arg.args) do kwarg
                    if kwarg isa Symbol
                        kwarg = :($kwarg::Any)
                    elseif isexpr(kwarg, :kw)
                        lhs = kwarg.args[1]
                        rhs = kwarg.args[2]
                        if lhs isa Symbol
                            if rhs isa Symbol
                                kwarg.args[1] = :($lhs::(@isdefined($rhs) ? typeof($rhs) : Any))
                            else
                                kwarg.args[1] = :($lhs::typeof($rhs))
                            end
                        end
                    end
                    kwarg
                end
            elseif isexpr(arg, :kw)
                if kwargs === nothing
                    kwargs = Any[]
                end
                lhs = arg.args[1]
                rhs = arg.args[2]
                if lhs isa Symbol
                    if rhs isa Symbol
                        arg.args[1] = :($lhs::(@isdefined($rhs) ? typeof($rhs) : Any))
                    else
                        arg.args[1] = :($lhs::typeof($rhs))
                    end
                end
                push!(kwargs, arg)
            else
                if arg isa Symbol
                    arg = :($arg::(@isdefined($arg) ? typeof($arg) : Any))
                elseif !isexpr(arg, :(::))
                    arg = :(::typeof($arg))
                end
                push!(pargs, arg)
            end
        end
        if kwargs === nothing
            x.args = Any[x.args[1], pargs...]
        else
            x.args = Any[x.args[1], Expr(:parameters, kwargs...), pargs...]
        end
    end
    #docs = lookup_doc(x) # TODO
    docs = esc(:(@doc $x))
    docs = if isfield(x)
        quote
            if isa($(esc(x.args[1])), DataType)
                fielddoc($(esc(x.args[1])), $(esc(x.args[2])))
            else
                $docs
            end
        end
    else
        docs
    end
    :(REPL.trimdocs($docs, $brief))
end

"""
    fielddoc(binding, field)

Return documentation for a particular `field` of a type if it exists.
"""
function fielddoc(binding::Binding, field::Symbol)
    for mod in modules
        dict = meta(mod)
        if haskey(dict, binding)
            multidoc = dict[binding]
            if haskey(multidoc.docs, Union{})
                fields = multidoc.docs[Union{}].data[:fields]
                if haskey(fields, field)
                    doc = fields[field]
                    return isa(doc, Markdown.MD) ? doc : Markdown.parse(doc)
                end
            end
        end
    end
    fields = join(["`$f`" for f in fieldnames(resolve(binding))], ", ", ", and ")
    fields = isempty(fields) ? "no fields" : "fields $fields"
    Markdown.parse("`$(resolve(binding))` has $fields.")
end

# As with the additional `doc` methods, this converts an object to a `Binding` first.
fielddoc(object, field::Symbol) = fielddoc(aliasof(object, typeof(object)), field)

# Search & Rescue
# Utilities for correcting user mistakes and (eventually)
# doing full documentation searches from the repl.

# Fuzzy Search Algorithm

function matchinds(needle, haystack; acronym::Bool = false)
    chars = collect(needle)
    is = Int[]
    lastc = '\0'
    for (i, char) in enumerate(haystack)
        while !isempty(chars) && isspace(first(chars))
            popfirst!(chars) # skip spaces
        end
        isempty(chars) && break
        if lowercase(char) == lowercase(chars[1]) &&
           (!acronym || !isletter(lastc))
            push!(is, i)
            popfirst!(chars)
        end
        lastc = char
    end
    return is
end

longer(x, y) = length(x) ≥ length(y) ? (x, true) : (y, false)

bestmatch(needle, haystack) =
    longer(matchinds(needle, haystack, acronym = true),
           matchinds(needle, haystack))

avgdistance(xs) =
    isempty(xs) ? 0 :
    (xs[end] - xs[1] - length(xs)+1)/length(xs)

function fuzzyscore(needle, haystack)
    score = 0.
    is, acro = bestmatch(needle, haystack)
    score += (acro ? 2 : 1)*length(is) # Matched characters
    score -= 2(length(needle)-length(is)) # Missing characters
    !acro && (score -= avgdistance(is)/10) # Contiguous
    !isempty(is) && (score -= sum(is)/length(is)/100) # Closer to beginning
    return score
end

function fuzzysort(search::String, candidates::Vector{String})
    scores = map(cand -> (fuzzyscore(search, cand), -Float64(levenshtein(search, cand))), candidates)
    candidates[sortperm(scores)] |> reverse
end

# Levenshtein Distance

function levenshtein(s1, s2)
    a, b = collect(s1), collect(s2)
    m = length(a)
    n = length(b)
    d = Matrix{Int}(undef, m+1, n+1)

    d[1:m+1, 1] = 0:m
    d[1, 1:n+1] = 0:n

    for i = 1:m, j = 1:n
        d[i+1,j+1] = min(d[i  , j+1] + 1,
                         d[i+1, j  ] + 1,
                         d[i  , j  ] + (a[i] != b[j]))
    end

    return d[m+1, n+1]
end

function levsort(search::String, candidates::Vector{String})
    scores = map(cand -> (Float64(levenshtein(search, cand)), -fuzzyscore(search, cand)), candidates)
    candidates = candidates[sortperm(scores)]
    i = 0
    for outer i = 1:length(candidates)
        levenshtein(search, candidates[i]) > 3 && break
    end
    return candidates[1:i]
end

# Result printing

function printmatch(io::IO, word, match)
    is, _ = bestmatch(word, match)
    for (i, char) = enumerate(match)
        if i in is
            printstyled(io, char, bold=true)
        else
            print(io, char)
        end
    end
end

function printmatches(io::IO, word, matches; cols::Int = _displaysize(io)[2])
    total = 0
    for match in matches
        total + length(match) + 1 > cols && break
        fuzzyscore(word, match) < 0 && break
        print(io, " ")
        printmatch(io, word, match)
        total += length(match) + 1
    end
end

printmatches(args...; cols::Int = _displaysize(stdout)[2]) = printmatches(stdout, args..., cols = cols)

function print_joined_cols(io::IO, ss::Vector{String}, delim = "", last = delim; cols::Int = _displaysize(io)[2])
    i = 0
    total = 0
    for outer i = 1:length(ss)
        total += length(ss[i])
        total + max(i-2,0)*length(delim) + (i>1 ? 1 : 0)*length(last) > cols && (i-=1; break)
    end
    join(io, ss[1:i], delim, last)
end

print_joined_cols(args...; cols::Int = _displaysize(stdout)[2]) = print_joined_cols(stdout, args...; cols=cols)

function print_correction(io::IO, word::String)
    cors = map(quote_spaces, levsort(word, accessible(Main)))
    pre = "Perhaps you meant "
    print(io, pre)
    print_joined_cols(io, cors, ", ", " or "; cols = _displaysize(io)[2] - length(pre))
    println(io)
    return
end

print_correction(word) = print_correction(stdout, word)

# Completion data

moduleusings(mod) = ccall(:jl_module_usings, Any, (Any,), mod)

filtervalid(names) = filter(x->!occursin(r"#", x), map(string, names))

accessible(mod::Module) =
    Symbol[filter!(s -> !Base.isdeprecated(mod, s), names(mod, all=true, imported=true));
           map(names, moduleusings(mod))...;
           collect(keys(Base.Docs.keywords))] |> unique |> filtervalid

doc_completions(name) = fuzzysort(name, accessible(Main))
doc_completions(name::Symbol) = doc_completions(string(name))

# Searching and apropos

# Docsearch simply returns true or false if an object contains the given needle
docsearch(haystack::AbstractString, needle) = occursin(needle, haystack)
docsearch(haystack::Symbol, needle) = docsearch(string(haystack), needle)
docsearch(::Nothing, needle) = false
function docsearch(haystack::Array, needle)
    for elt in haystack
        docsearch(elt, needle) && return true
    end
    false
end
function docsearch(haystack, needle)
    @warn "Unable to search documentation of type $(typeof(haystack))" maxlog=1
    false
end

## Searching specific documentation objects
function docsearch(haystack::MultiDoc, needle)
    for v in values(haystack.docs)
        docsearch(v, needle) && return true
    end
    false
end

function docsearch(haystack::DocStr, needle)
    docsearch(parsedoc(haystack), needle) && return true
    if haskey(haystack.data, :fields)
        for doc in values(haystack.data[:fields])
            docsearch(doc, needle) && return true
        end
    end
    false
end

## doc search

## Markdown search simply strips all markup and searches plain text version
docsearch(haystack::Markdown.MD, needle) = docsearch(stripmd(haystack.content), needle)

"""
    stripmd(x)

Strip all Markdown markup from x, leaving the result in plain text. Used
internally by apropos to make docstrings containing more than one markdown
element searchable.
"""
stripmd(@nospecialize x) = string(x) # for random objects interpolated into the docstring
stripmd(x::AbstractString) = x  # base case
stripmd(x::Nothing) = " "
stripmd(x::Vector) = string(map(stripmd, x)...)

stripmd(x::Markdown.BlockQuote) = "$(stripmd(x.content))"
stripmd(x::Markdown.Admonition) = "$(stripmd(x.content))"
stripmd(x::Markdown.Bold) = "$(stripmd(x.text))"
stripmd(x::Markdown.Code) = "$(stripmd(x.code))"
stripmd(x::Markdown.Header) = stripmd(x.text)
stripmd(x::Markdown.HorizontalRule) = " "
stripmd(x::Markdown.Image) = "$(stripmd(x.alt)) $(x.url)"
stripmd(x::Markdown.Italic) = "$(stripmd(x.text))"
stripmd(x::Markdown.LaTeX) = "$(x.formula)"
stripmd(x::Markdown.LineBreak) = " "
stripmd(x::Markdown.Link) = "$(stripmd(x.text)) $(x.url)"
stripmd(x::Markdown.List) = join(map(stripmd, x.items), " ")
stripmd(x::Markdown.MD) = join(map(stripmd, x.content), " ")
stripmd(x::Markdown.Paragraph) = stripmd(x.content)
stripmd(x::Markdown.Footnote) = "$(stripmd(x.id)) $(stripmd(x.text))"
stripmd(x::Markdown.Table) =
    join([join(map(stripmd, r), " ") for r in x.rows], " ")

"""
    apropos([io::IO=stdout], pattern::Union{AbstractString,Regex})

Search available docstrings for entries containing `pattern`.

When `pattern` is a string, case is ignored. Results are printed to `io`.

`apropos` can be called from the help mode in the REPL by wrapping the query in double quotes:
```
help?> "pattern"
```
"""
apropos(string) = apropos(stdout, string)
apropos(io::IO, string) = apropos(io, Regex("\\Q$string", "i"))

function apropos(io::IO, needle::Regex)
    for mod in modules
        # Module doc might be in README.md instead of the META dict
        docsearch(doc(mod), needle) && println(io, mod)
        for (k, v) in meta(mod)
            docsearch(v, needle) && println(io, k)
        end
    end
end

max_stars_repo_name: JuliaLabs/julia

max_issues_repo_name: JuliaLabs/julia

max_forks_repo_name: JuliaLabs/julia


text:
function DirectoryCrawler(arg0::File)
    return DirectoryCrawler((File,), arg0)
end

function feed(obj::DirectoryCrawler, arg0::Pattern, arg1::DataLoader, arg2::DataProvidersManager)
    return jcall(obj, "feed", jboolean, (Pattern, DataLoader, DataProvidersManager), arg0, arg1, arg2)
end