text:
################################################################################
##### Base.Array
################################################################################
function _collect(::Type{T}, arr::Array{T}) where {T}
    arr
end
function _collect(::Type{T}, arr::AbstractArray) where {T}
    collect(T, arr)
end
_collect(arr::Array) = arr
_collect(arr) = collect(arr)

function from_namedtuple(Arr::Type{<: Array{T}}, nt::NamedTuple) where {T}
    _collect(T, nt.values)::Arr
end

function from_namedtuple(Arr::Type{<: Array}, nt::NamedTuple)
    _collect(nt.values)::Arr
end

roottype(::Type{T}) where {T <: Array} = Array


text:
export CnoidalWaveSerreGreenNaghdi
using Elliptic

"""
    CnoidalWaveSerreGreenNaghdi(param; P)

Compute the Serre-Green-Naghdi cnoidal wave with prescribed `h₀<h₁<h₂`.
`h_1` is the minimum, `h_2` is the maximum of the wave.
As `h₀ -> h₁`, the cnoidal wave converges towards the solitary wave.
See for instance Gavrilyuk, Nkonga, Shyue and Truskinovsky, doi:10.1088/1361-6544/ab95ac

# Arguments
- `param :: NamedTuple`: parameters of the problem containing `h₀<h₁<h₂` and dimensionless parameters `ϵ` and `μ`, and number of collocation points `N`.
- `P :: Int`: (keyword, optional, default = 2) the number of periods of the cnoidal wave in the constructed mesh.

# Return values
`(η,u,v,mesh,param)` with
- `η :: Vector{Float64}`: surface deformation;
- `u :: Vector{Float64}`: layer-averaged velocity;
- `v :: Vector{Float64}`: tangential velocity;
- `mesh :: Mesh`: mesh collocation points;
- `param :: NamedTuple`: useful parameters
"""
function CnoidalWaveSerreGreenNaghdi(
                param :: NamedTuple;
                P = 2 :: Int)

        ϵ = param.ϵ
        μ = param.μ

        h₀=param.h₀
        h₁=param.h₁
        h₂=param.h₂
        c = sqrt(h₀*h₁*h₂)
        @info string("The velocity is c=",c)
        m = sqrt((h₂-h₁)/(h₂-h₀))
        κ = sqrt(3*(h₂-h₀))/(2*c)/sqrt(μ)
        λ = Elliptic.K(m^2)/κ
        @info string("The period is 2*λ=",2*λ)
        mesh = Mesh((L=P*λ,N=param.N))
        formula = h₁ .-1 .+ (h₂-h₁)*(Jacobi.cn.(κ*mesh.x,m^2).^2)

        a₀ = h₀
        a₁ = h₂ - h₀
        formula2 = a₀ .-1 .+ a₁*(Jacobi.dn.(κ*mesh.x,m^2).^2)
        H₀= a₀+a₁*Elliptic.E(m^2)/Elliptic.K(m^2)
        u2 = c*(1 ./ H₀ .- 1 ./(1 .+ formula2))
        param = (h₀=h₀,h₁=h₁,h₂=h₂,a₀=a₀,a₁=a₁,H₀=H₀,c=c,λ=λ,m=m,κ=κ)

        η = formula/ϵ
        h = 1 .+ ϵ*η
        u = c*η./h
	k = mesh.k
        Dx=  1im * k
	F₀ = sqrt(μ)*Dx
        DxF(v) = real.(ifft(F₀ .* fft(v)))
	v = u - 1/3 ./h .* (DxF(h.^3 .*DxF(u)))
        #
        # h2 = 1 .+ ϵ*formula2
        # v2 = u2 - 1/3 ./h2 .* (DxF(h2.^3 .*DxF(u2)))

        return (η,u,v,mesh,param)

end

ext: jl


text:
using PlantBiophysics
using Documenter
using PlantBiophysics
using DataFrames
using CSV
using Plots

DocMeta.setdocmeta!(PlantBiophysics, :DocTestSetup, :(using PlantBiophysics,Plots,DataFrames,CSV); recursive = true)

makedocs(;
    modules = [PlantBiophysics],
    authors = "remi.vezy <VEZY@users.noreply.github.com> and contributors",
    repo = "https://github.com/VEZY/PlantBiophysics.jl/blob/{commit}{path}#L{line}",
    sitename = "PlantBiophysics.jl",
    format = Documenter.HTML(;
        prettyurls = get(ENV, "CI", "false") == "true",
        canonical = "https://VEZY.github.io/PlantBiophysics.jl",
        assets = String[],
    ),
    pages = [
        "Home" => "index.md",
        "Getting started" => [
            "TL;DR" => "./getting_started/get_started.md",
            "Parameter fitting" => "./getting_started/first_fit.md",
        ],
        "Parameter fitting" => "./fitting/parameter_fitting.md",
        "Simulation" => [
            "Simple Simulation" => "./simulation/first_simulation.md",
            "Several time steps" => "./simulation/several_simulation.md",
            "Several objects" => "./simulation/several_objects_simulation.md",
            "Uncertainty propagation" => "./simulation/uncertainty_propagation.md",
        ],
        "Concepts" => Any[
            "Package design" => "./concepts/package_design.md",
            "Implement a model" => "./concepts/implement_a_model.md",
        ],
        "Generic models" => Any[
            "Photosynthesis" => "./models/photosynthesis.md",
            "Stomatal conductance" => "./models/gs.md",
            "Energy balance" => "./models/energy_balance.md",
            "Light interception" => "./models/light.md",
        ],
        "Components and methods" => Any[
            "LeafModels" => "./components/leafmodels.md",
        ],
        "API" => "functions.md"
    ],
)

deploydocs(;
    repo = "github.com/VEZY/PlantBiophysics.jl.git",
)


text:
module TestEstspec

using QuantEcon
using Base.Test
using FactCheck

# set up
srand(42)
x_20 = rand(20)
x_21 = rand(21)

facts("Testing estspec") do

    context("testing output sizes of periodogram and ar_periodogram") do
    # test shapes of periodogram and ar_periodogram functions
        for x in Any[x_20, x_21]
            w, I_w = periodogram(x)
            n_w, n_Iw, n_x = length(w), length(I_w), length(x)

            @fact n_w --> floor(Int, n_x / 2 + 1)
            @fact n_Iw --> floor(Int, n_x / 2 + 1)

            w, I_w = ar_periodogram(x)
            n_w, n_Iw, n_x = length(w), length(I_w), length(x)

            # when x is even we get 10 elements back
            @fact n_w --> (iseven(n_x) ? floor(Int, n_x / 2) : floor(Int, n_x / 2 + 1))
            @fact n_Iw --> (iseven(n_x) ? floor(Int, n_x / 2) : floor(Int, n_x / 2 + 1))
        end
    end  # context

    context("testing `smooth` function options") do
        # window length must be between 3 and length(x)
        @fact_throws smooth(x_20, window_len=25)
        @fact_throws smooth(x_20, window_len=2)

        # test hanning is default
        out = smooth(x_20, window="foobar")
        @fact out --> smooth(x_20, window="hanning")

        # test window_len must be odd
        @fact length(smooth(x_20, window_len=6)) --> length(smooth(x_20, window_len=7))

    end  # context

end  # facts
end  # module


text:
# FFT test

Fs = 1000;            # Sampling frequency
T = 1/Fs;             # Sampling period

FFTW.set_num_threads(Sys.CPU_CORES)

function prepare_input(input_size)
	t = (0:input_size-1)*T;        # Time vector

	S = 0.7*sin.(2pi*50t) + sin.(2pi*120t);
	return S + 2*randn(size(t));
end

function test_function(input_data)
	return fft(input_data)
end


text:
"""
$(TYPEDSIGNATURES)

```jldoctest
julia> random_design!((Uniform(2, 3), DiscreteUniform(-1, 5), Uniform(5, 10)), 10)
10×3 Matrix{Float64}:
 2.04922  -1.0  8.21161
 2.59117   3.0  5.40944
 2.77148  -1.0  9.62061
 2.25659   4.0  5.79254
 2.64968   2.0  6.22902
 2.31523   5.0  6.00256
 2.86526   2.0  9.21818
 2.44753   0.0  8.80749
 2.08284  -1.0  8.47297
 2.81201  -1.0  6.11722

```
"""
function random_design!(distributions::Tuple, n::Int)
    return hcat([rand(distributions[d], n) for d in 1:size(distributions, 1)]...)
end


text:
module ANaiveJuliaTutorial

using IJulia, Pkg

export tutorial, notebooks

function tutorial()
    cmd = IJulia.find_jupyter_subcommand("notebook")
    push!(cmd.exec, joinpath(@__DIR__, "..", "notebooks", "tutorial.ipynb"))
    return IJulia.launch(cmd, joinpath(@__DIR__, "..", "notebooks"), false)
end

function notebooks()
    return IJulia.notebook(dir=joinpath(@__DIR__, "..", "notebooks"))
end

REQUIRE = [
    "PyCall",
    "IJulia",
    "Revise",
    "Interact",
]

function __init__()
    for each in REQUIRE
        if each in keys(Pkg.installed())
            continue
        else
            Pkg.add(each)
        end
    end
end

end # module


text:
# Decision #
# ========================== #
@enum DecisionState NotTaken Taken Known

mutable struct Decision{T}
    state::DecisionState
    value::T
end
state(decision::Decision) = decision.state
decision_value(decision::Decision) = state(decision) == NotTaken ? NaN : decision.value

function Decision(value, ::Type{T}) where T
    return Decision(Taken, T(value))
end

function KnownDecision(value, ::Type{T}) where T
    return Decision(Known, T(value))
end

function Decision(info::JuMP.VariableInfo, ::Type{T}) where T
    if info.has_fix
        return Decision(Taken, T(info.fixed_value))
    end
    return Decision(NotTaken, T(NaN))
end

function KnownDecision(info::JuMP.VariableInfo, ::Type{T}) where T
    value = if info.has_fix
        # Value fixed at construction
        T(info.fixed_value)
    else
        # Value is known, but not yet set. Pick a feasible value for now
        value = if info.has_lb
            T(info.lower_bound)
        elseif info.has_ub
            T(info.upper_bound)
        else
            zero(T)
        end
    end
    return Decision(Known, value)
end

Base.copy(decision::Decision) = Decision(decision.state, decision.value)

# Decisions #
# ========================== #
struct IgnoreDecisions end

struct Decisions
    decisions::OrderedDict{MOI.VariableIndex, Decision{Float64}}

    function Decisions()
        return new(OrderedDict{MOI.VariableIndex, Decision{Float64}}())
    end
end

has_decision(decisions::Decisions, index::MOI.VariableIndex) = haskey(decisions.decisions, index)
decision(decisions::Decisions, index::MOI.VariableIndex) = decisions.decisions[index]
decision_value(decisions::Decisions, index::MOI.VariableIndex) = decision_value(decision(decisions, index))

function set_decision!(decisions::Decisions, index::MOI.VariableIndex, decision::Decision)
    decisions.decisions[index] = decision
    return nothing
end

function remove_decision!(::IgnoreDecisions, ::MOI.VariableIndex)
    return nothing
end

function remove_decision!(decisions::Decisions, index::MOI.VariableIndex)
    if !haskey(decisions.decisions, index)
        return nothing
    end
    delete!(decisions.decisions, index)
    return nothing
end

function clear!(decisions::Decisions)
    empty!(decisions.decisions)
    return nothing
end

function all_decisions(decisions::Decisions)
    return filter(decisions.decisions) do (index, decision)
        state(decision) != Known
    end |> keys |> collect
end

function all_known_decisions(decisions::Decisions)
    return filter(decisions.decisions) do (index, decision)
        state(decision) == Known
    end |> keys |> collect
end

function num_decisions(decisions::Decisions)
    return count(decisions.decisions) do (index, decision)
        return state(decision) != Known
    end
end

function num_known_decisions(decisions::Decisions)
    return count(decisions.decisions) do (index, decision)
        state(decision) == Known
    end
end

function untake_decisions!(decisions::Decisions)
    need_update = false
    # Update states and values
    for decision in all_decisions(decisions)
        if decision.state != NotTaken
            decision.state = NotTaken
            need_update |= true
        end
    end
    return need_update
end

function take_decisions!(decisions::Decisions, x::AbstractVector)
    # Check decision length
    num_decisions(decisions) == length(x) || error("Given decision of length $(length(x)) not compatible with number of defined decision variables $(num_decisions(decisions)).")
    # Update states and values (assume x given in sorted order)
    for (decision, val) in zip(all_decisions(decisions), x)
        decision.state = Taken
        decision.value = val
    end
    return nothing
end

function update_known_decisions!(decisions::Decisions, x::AbstractVector)
    # Check decision length
    num_known_decisions(decisions) == length(x) || error("Given decision of length $(length(x)) not compatible with number of defined known decision variables $(num_known_decisions(decisions)).")
    # Update values (assume x given in sorted order)
    for (key, val) in zip(all_known_decisions(decisions), x)
        decisions.decisions[key].value = val
    end
    return nothing
end

is_decision_type(::DataType) = false

include("variable_interface.jl")
include("expressions/expressions.jl")
include("functions/functions.jl")
include("sets.jl")
include("modifications.jl")
include("bridges/bridges.jl")
include("macros.jl")
include("updates.jl")
include("moi_overrides.jl")


text:
__precompile__()

module ZStd

export decompress, decompress!

if isfile(joinpath(dirname(@__FILE__), "..", "deps", "deps.jl"))
    include(joinpath(dirname(@__FILE__), "..", "deps", "deps.jl"))
else
    error("ZStd.jl is not properly installed. Please run Pkg.build(\"ZStd\") " *
          "and restart Julia.")
end

struct ZStdError <: Exception
    msg::String
end

Base.showerror(io::IO, ex::ZStdError) = print(io, "ZStd: " * ex.msg)

# check whether the strides of A correspond to contiguous data
# shamelessly stolen from https://github.com/stevengj/Blosc.jl
iscontiguous(::Array) = true
iscontiguous(::Vector) = true
iscontiguous(A::DenseVector) = stride(A,1) == 1
function iscontiguous(A::DenseArray)
    p = sortperm([strides(A)...])
    s = 1
    for k = 1:ndims(A)
        if stride(A,p[k]) != s
            return false
        end
        s *= size(A,p[k])
    end
    return true
end

# Determine whether the input represents a zstd error, yes => throw it, no => return it
function check_zstd_error(code::Csize_t)
    iserr = Bool(ccall((:ZSTD_isError, libzstd), Cuint, (Csize_t, ), code))
    if iserr
        msg = unsafe_string(ccall((:ZSTD_getErrorName, libzstd), Ptr{Cchar}, (Csize_t, ), code))
        throw(ZStdError(msg))
    end
    return code # input is not an error
end

"""
    ZStd.MAX_COMPRESSION

An integer representing the maximum compression level available.
"""
const MAX_COMPRESSION = Int(ccall((:ZSTD_maxCLevel, libzstd), Cint, ()))

"""
    maxcompressedsize(srcsize)

Get the maximum compressed size in the worst-case scenario for a given input size.
"""
function maxcompressedsize(srcsize::Csize_t)
    return ccall((:ZSTD_compressBound, libzstd), Csize_t, (Csize_t, ), srcsize)
end

maxcompressedsize(srcsize::Int) = Int(maxcompressedsize(Csize_t(srcsize)))

"""
    ZStd.ZSTD_VERSION

The version of Zstandard in use.
"""
const ZSTD_VERSION = let
    ver = Int(ccall((:ZSTD_versionNumber, libzstd), Cuint, ()))
    str = join(match(r"(\d+)(\d{2})(\d{2})$", string(ver)).captures, ".")
    VersionNumber(str)
end

# Simple API
function ZSTD_compress(dst::Ptr, dstCapacity::Csize_t, src::Ptr, compressedSize::Csize_t, compressionLevel::Cint)
    return ccall((:ZSTD_decompress, libzstd), Csize_t,
      (Ptr{Void}, Csize_t, Ptr{Void}, Csize_t, Cint),
      dst, dstCapacity, src, compressedSize, compressionLevel)
end

function ZSTD_decompress(dst::Ptr, dstCapacity::Csize_t, src::Ptr, compressedSize::Csize_t)
    return ccall((:ZSTD_decompress, libzstd), Csize_t,
      (Ptr{Void}, Csize_t, Ptr{Void}, Csize_t),
      dst, dstCapacity, src, compressedSize)
end

const ZSTD_CONTENTSIZE_UNKNOWN = typemax(Culonglong)
const ZSTD_CONTENTSIZE_ERROR = typemax(Culonglong) - 1
function getFrameContentSize(src::Ptr, srcSize::Csize_t)
    return ccall((:ZSTD_getFrameContentSize, libzstd), Culonglong,
      (Ptr{Void}, Csize_t),
      src, srcSize)
end

"""
    decompress!(dst::DenseVector{T}, src::DenseVector{UInt8})
Like `decompress`, but uses a pre-allocated destination buffer `dst`,
which is resized as needed to store the decompressed data from `src`.
"""
function decompress!{T}(dst::DenseVector{T}, src::DenseVector{UInt8})
    if !iscontiguous(dst) || !iscontiguous(src)
        throw(ArgumentError("Source and Destination must be contiguous arrays"))
    end
    if !isbits(T)
        throw(ArgumentError("Destination must be a DenseVector of `isbits` element types"))
    end

    sT = sizeof(T)
    uncompressed = getFrameContentSize(pointer(src), Csize_t(sizeof(src)))
    if uncompressed == 0
        return resize!(dst, 0)
    elseif uncompressed == ZSTD_CONTENTSIZE_ERROR
        error("Error while reading frame content size.")
    elseif uncompressed == ZSTD_CONTENTSIZE_UNKNOWN
        # Should use streaming decompression
        uncompressed = sT * div(min(1000000000, 100 * sizeof(src)), sT)
        warn("Can't determine uncompressed size - setting buffer to $(div(uncompressed, 1000000)) MB")
    end
    if mod(uncompressed, sT) > 0
        error("uncompressed data is not a multiple of sizeof($T)")
    end

    elCnt = div(uncompressed, sT)
    resize!(dst, elCnt)
    uncompressed = check_zstd_error(ZSTD_decompress(pointer(dst), Csize_t(sizeof(dst)), pointer(src), Csize_t(sizeof(src))))
    return resize!(dst, uncompressed)
end

"""
    decompress(T::Type, src::DenseVector{UInt8})
Return the compressed buffer `src` as an array of element type `T`.
"""
decompress{T}(::Type{T}, src::DenseVector{UInt8}) = decompress!(Vector{T}(0), src)

end # module


text:
using BinDeps

const defdir = joinpath(dirname(@__FILE__), "..", "datasets")

function get_usp_data(dir)
    mkpath(dir)

    @info """Downloading usp-stream-data.zip ...
    You will need the password for this file and it can be found at https://sites.google.com/view/uspdsrepository
    """

    path = download("http://sites.labic.icmc.usp.br/vsouza/repository/usp-stream-data.zip")
    run(unpack_cmd(path, dir, ".zip", ""))
end

function dataset_airlines(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/airlines.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_chess(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/chess.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_covtype(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/covtype.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_elec(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/elec.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_gassensor(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/gassensor.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_kddcup99(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/kddcup99.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_keystroke(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/keystroke.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_luxembourg(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/luxembourg.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_NOAA(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/NOAA.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_outdoor(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/outdoor.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_ozone(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/ozone.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_power_lsn(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/power-lsn.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_poker_lsn(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/poker-lsn.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_powersupply(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/powersupply.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_rialto(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/rialto.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_sensorstream(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/sensorstream.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_spam_assassin(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/SpamAssassin.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_1(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-abrupt_balanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_2(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-abrupt_imbalanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_3(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-gradual_balanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_4(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-gradual_imbalanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_5(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental_balanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_6(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental_imbalanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_7(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental-abrupt_balanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_8(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental-abrupt_imbalanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_9(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental-reoccurring_balanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_10(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-incremental-reoccurring_imbalanced_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end

function dataset_insect_11(batch::Int)::EasyStream.BatchStream
    filename = "$(defdir)/real/repository/INSECTS-out-of-control_norm.arff"

    isfile(filename) || get_usp_data(defdir)

    data = ARFFFiles.load(DataFrame,filename)

    conn = EasyStream.TablesConnector(data)

    stream = EasyStream.BatchStream(conn, batch = batch)

    return stream
end


text:
push!(LOAD_PATH, "./src")

using Test
using YaDB

function feed_db(;pkey=:id)
  Users = if pkey == :id
    Table([:name => String, :num_friends => Int])
  else
    Table([:name => String, :num_friends => Int]; pkey=pkey => Int)
  end

  insert(Users, [[0, "Hero", 0],
      [1, "Dunn", 2],
      [2, "Sue", 3],
      [3, "Chi", 3],
      [4, "Thor", 3],
      [5, "Clive", 2],
      [6, "Hicks", 3],
      [7, "Devin", 2],
      [8, "Kate", 2]
  ])
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  insert(Users, [:name => "PasMas", :num_friends => 5])

  Users
end

function feed_db_nv()  ## With null values
  Users = Table([:name => U_SN, :num_friends => U_IN])

  insert(Users, [[0, "Hero", 0],
      [1, "Dunn", 2],
      [2, "Sue", 3],
      [3, "Chi", 3],
      [4, "Thor", nothing],
      [5, "Clive", 2],
      [6, "Hicks", 3],
      [7, "Devin", nothing],
      [8, "Kate", 2]
  ])
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  insert(Users, [:name => "PasMas", :num_friends => 5])

  Users
end

# ======================================================================

# using Pkg; Pkg.activate("MLJ_env", shared=true); push!(LOAD_PATH, "./src"); using YaDB;
# Users = Table([:name => U_SN, :num_friends => U_IN]; pkey=(:user_id => Int))

# insert(Users, Dict(:name => "Ayumi", :num_friends => 5))

# # OK insert/1
# insert(Users, [:name => "PasMas", :num_friends => 3, id(Users) => 3])
# insert(Users, [id(Users) => 17, :name => "Foo", :num_friends => nothing])
# insert(Users, [id(Users) => 77, :name => "FooFoo", :num_friends => nothing])

# # The type below is ambiguous, no way to tell that num_friends is Int or Nothing
# insert(Users, [:name => "FooBar", :num_friends => nothing])
# ======================================================================

@testset "Create Table/1" begin
  Users = Table([:name => String, :num_friends => Int]; pkey=(:user_id => Int))

  @test length(Users) == 0
  @test sort(Users.columns) == [:name, :num_friends, :user_id]
  @test id(Users) == :user_id
end

@testset "Create Table - default pkey" begin
  Users = Table([:name => String, :num_friends => Int])

  @test length(Users) == 0
  @test sort(Users.columns) == [:id, :name, :num_friends]
  @test id(Users) == :id
end

@testset "Insert rows" begin
  Users = Table([:name => String, :num_friends => Int]; pkey=(:user_id => Int))

  ## using Dict + pkey value
  insert(Users, [id(Users) => 1, :name => "Dunn", :num_friends => 2])
  @test length(Users) == 1
  @test Users.rows[1][:name] == "Dunn"

  ## using only Dict => pkey value auto. gen.
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  @assert length(Users) == 2

  ## re-insert (with same pkey) is a NO-OP
  insert(Users, [id(Users) => 1, :name => "Dunn", :num_friends => 2])
  @assert length(Users) == 2

  ## mass insertion
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", 3],
     [7, "Clive", 2],
     [8, "Devin", 2],
     [9, "Kate", 2 ]
  ]
  insert(Users, insert_v)
  @test length(Users) == 2 + length(insert_v)
end

@testset "Insert rows / NULL value as nothing" begin
  Users = Table([:name => U_SN, :num_friends => U_IN]; pkey=(:user_id => Int))

  ## using Dict + pkey value
  insert(Users, [id(Users) => 1, :name => "Dunn", :num_friends => 2])
  @test length(Users) == 1
  @test Users.rows[1][:name] == "Dunn"

  ## using only Dict => pkey value auto. gen.
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  @assert length(Users) == 2

  ## re-insert (with same pkey) is a NO-OP
  insert(Users, [id(Users) => 1, :name => "Dunn", :num_friends => 2])
  @assert length(Users) == 2

  ## insert with 'null' value => encoded as nothing, with pkey
  insert(Users, [id(Users) => 17, :name => "Foo", :num_friends => nothing])
  @assert length(Users) == 3

  ## insert with 'null' value => encoded as nothing, with pkey
  insert(Users, [:name => "Bar", :num_friends => nothing])
  @assert length(Users) == 4

  ## mass insertion
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", 3],
     [7, "Clive", 2],
     [8, "Devin", 2],
     [9, "Kate", 2 ],
     [10, "Tom", nothing]
  ]
  insert(Users, insert_v)
  @test length(Users) == 4 + length(insert_v)
end

@testset "Update rows" begin
  Users = Table([:name => String, :num_friends => Int]; pkey=(:id => Int))
  insert(Users, [:id => 1, :name => "Dunn", :num_friends => 2])
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  @test length(Users) == 2

  update(Users, Dict{Symbol, Any}(:num_friends => 7), row -> row[:name] == "Ayumi")
  @test length(Users) == 2
  @test Users.rows[2][:num_friends] == 7 && Users.rows[2][:name] == "Ayumi"

  ## several updates: users with 3 friends have now 4 friends
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", 3],
     [7, "Clive", 2],
     [8, "Devin", 2],
     [9, "Kate", 2 ]
  ]
  insert(Users, insert_v)
  @test filter(r -> r[:num_friends] == 3, Users.rows) |> length == 2

  update(Users, Dict{Symbol, Any}(:num_friends => 4), row -> row[:num_friends] == 3)
  @test filter(r -> r[:num_friends] == 3, Users.rows) |> length == 0
  @test filter(r -> r[:num_friends] == 4, Users.rows) |> length == 2
end

@testset "Update rows / NULL value as nothing" begin
  Users = Table([:name => U_SN, :num_friends => U_IN]; pkey=(:user_id => Int))

  insert(Users, [id(Users) => 1, :name => "Dunn", :num_friends => 2])
  insert(Users, Dict(:name => "Ayumi", :num_friends => 5))
  @test length(Users) == 2

  update(Users, Dict{Symbol, Any}(:num_friends => 7), row -> row[:name] == "Ayumi")
  @test length(Users) == 2
  @test Users.rows[2][:num_friends] == 7 && Users.rows[2][:name] == "Ayumi"

  ## several updates: users with 3 friends have now 4 friends
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", 3],
     [7, "Clive", 2],
     [8, "Devin", nothing],
     [9, "Kate", nothing ]
  ]
  insert(Users, insert_v)
  @test filter(r -> r[:num_friends] == 3, Users.rows) |> length == 2

  update(Users, Dict{Symbol, Any}(:num_friends => 4), row -> row[:num_friends] == 3)
  @test filter(r -> r[:num_friends] == 3, Users.rows) |> length == 0
  @test filter(r -> r[:num_friends] == 4, Users.rows) |> length == 2

  update(Users, Dict{Symbol, Any}(:num_friends => 1), row -> row[:num_friends] === nothing)
  @test filter(r -> r[:num_friends] === nothing, Users.rows) |> length == 0
  @test filter(r -> r[:num_friends] == 1, Users.rows) |> length == 2
end

@testset "Delete rows" begin
  Users = Table([:name => String, :num_friends => Int]; pkey=(:id => Int))
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", 3],
     [7, "Clive", 2],
     [8, "Devin", 2],
     [9, "Kate", 2 ]
  ]
  insert(Users, insert_v)
  @test length(Users) == length(insert_v)

  delete(Users, row -> row[:num_friends] == 3)
  @test length(Users) == length(insert_v) - 2

  delete(Users)
  @test length(Users) == 0
end

@testset "Delete rows / NULL value as nothing" begin
  Users = Table([:name => U_SN, :num_friends => U_IN]; pkey=(:user_id => Int))
  insert_v = [
     [5, "Chi", 3],
     [6, "Thor", nothing],
     [7, "Clive", 2],
     [8, "Devin", nothing],
     [9, "Kate", 2 ]
  ]
  insert(Users, insert_v)
  @test length(Users) == length(insert_v)

  delete(Users, row -> row[:num_friends] === nothing)
  @test length(Users) == length(insert_v) - 2

  delete(Users)
  @test length(Users) == 0
end

@testset "Select / where / limit" begin
  Users = feed_db()
  n = length(Users)
  user_names = select(Users, keep_cols=[:name])

  @test length(user_names) == n
  @test user_names.columns == [:name]

  dunn_ids = where(Users, row -> row[:name] == "Dunn") |>
    u -> select(u, keep_cols=[:id])
  @test length(dunn_ids) == 1
  @test dunn_ids.rows[1][:id] == 1

  amp_name_ids = where(Users, row -> row[:name][1] ∈ ['A', 'P']) |>
    u -> select(u, keep_cols=[:id, :name])
  @test length(amp_name_ids) == 2
  @test amp_name_ids.rows[1][:name] ∈ ["Ayumi", "PasMas"]
  @test amp_name_ids.rows[2][:name] ∈ ["Ayumi", "PasMas"]

  ext_user = name_lengths₂ = select(Users;
    add_cols=Dict{Symbol, Function}(
      :name_ini => row -> string(row[:name][1]),
      :len_name => row -> length(row)
    )
  )
  @test sort(ext_user.columns) == Symbol[:id, :len_name, :name, :name_ini, :num_friends]
end

@testset "Select / where / limit / NULL value as nothing" begin
  Users = feed_db_nv()
  n = length(Users)
  user_names = select(Users, keep_cols=[:name])

  @test length(user_names) == n
  @test user_names.columns == [:name]

  dunn_ids = where(Users, row -> row[:name] == "Dunn") |>
    u -> select(u, keep_cols=[:id])
  @test length(dunn_ids) == 1
  @test dunn_ids.rows[1][:id] == 1

  amp_name_ids = where(Users, row -> row[:name][1] ∈ ['A', 'P']) |>
    u -> select(u, keep_cols=[:id, :name])
  @test length(amp_name_ids) == 2
  @test amp_name_ids.rows[1][:name] ∈ ["Ayumi", "PasMas"]
  @test amp_name_ids.rows[2][:name] ∈ ["Ayumi", "PasMas"]

  ext_user = name_lengths₂ = select(Users;
    add_cols=Dict{Symbol, Function}(
      :name_ini => row -> string(row[:name][1]),
      :len_name => row -> length(row)
    )
  )
  @test sort(ext_user.columns) == Symbol[:id, :len_name, :name, :name_ini, :num_friends]
end

@testset "Group by/1" begin
  Users = feed_db()

  """
  -- find number of users and smallest user_id for each possible name length:
  SELECT LENGTH(name) as name_length, MIN(user_id) AS min_user_id,
    COUNT(*) AS num_users
  FROM users
  GROUP BY LENGTH(name);
    """
  name_len_fn = row -> length(row[:name])
  min_user_id = rows -> minimum(row[:id] for row ∈ rows)
  num_rows = rows -> length(rows)

  stats_by_len = select(Users, add_cols=D_SF(:name_length => name_len_fn)) |>
    u -> group_by(u,
        group_by_cols=[:name_length],
        agg=D_SF(:min_user_id => min_user_id, :num_users => num_rows))

  @test sort(stats_by_len.columns) == [:min_user_id, :name_length, :num_users]
  @test length(stats_by_len.rows) == 4
end

@testset "Group by/1 / NULL value as nothing " begin
  Users = feed_db_nv()

  """
  -- find number of users and smallest user_id for each possible name length:
  SELECT LENGTH(name) as name_length, MIN(user_id) AS min_user_id,
    COUNT(*) AS num_users
  FROM users
  GROUP BY LENGTH(name);
    """
  name_len_fn = row -> length(row[:name])
  min_user_id = rows -> minimum(row[:id] for row ∈ rows)
  num_rows = rows -> length(rows)

  stats_by_len = select(Users, add_cols=D_SF(:name_length => name_len_fn)) |>
    u -> group_by(u,
        group_by_cols=[:name_length],
        agg=D_SF(:min_user_id => min_user_id, :num_users => num_rows))

  @test sort(stats_by_len.columns) == [:min_user_id, :name_length, :num_users]
  @test length(stats_by_len.rows) == 4
end

@testset "Group by/2" begin
  Users = feed_db()

  """
  -- average number of friends for users whose names start with specific letters
  -- but see only the results for letters whose corresponding average is greater
  -- than 1
  SELECT SUBSTR(name, 1, 1) AS first_letter,
    AVG(num_friends) AS avg_num_friends
  FROM users
  GROUP BY SUBSTR(name, 1, 1)
  HAVING AVG(num_friends) > 1;
  """
  first_letter_fn = row -> row[:name] !== nothing ? string(row[:name][1]) : ""
  avg_num_friends_fn =
    rows -> sum([row[:num_friends] for row in rows]) / length(rows)
  enough_friends_fn = rows -> avg_num_friends_fn(rows) > 1.

  avg_friends_by_letter =
    select(Users, add_cols=D_SF(:first_letter => first_letter_fn)) |>
    u -> group_by(u, group_by_cols=[:first_letter],
        agg=D_SF(:avg_num_friends => avg_num_friends_fn),
                  having=enough_friends_fn)

  @test sort(avg_friends_by_letter.columns) == [:avg_num_friends, :first_letter]
  @test length(avg_friends_by_letter.rows) == 8
end

@testset "Order by" begin
  Users = feed_db()
  first_letter_fn = row -> row[:name] !== nothing ? string(row[:name][1]) : ""
  avg_num_friends_fn =
    rows -> sum([row[:num_friends] for row in rows]) / length(rows)
  enough_friends_fn = rows -> avg_num_friends_fn(rows) > 1.

  avg_friends_by_letter =
    select(Users, add_cols=D_SF(:first_letter => first_letter_fn)) |>
    u -> group_by(u, group_by_cols=[:first_letter],
        agg=D_SF(:avg_num_friends => avg_num_friends_fn),
                  having=enough_friends_fn)
  #
  friendliest_letters₂ = avg_friends_by_letter |>
    u -> order_by(u, row -> -row[:avg_num_friends]) |>
    u -> limit(u, 3)
  #

  @test sort(friendliest_letters₂.columns) == [:avg_num_friends, :first_letter]
  @test friendliest_letters₂.rows[1][:first_letter] == "A"
  @test length(friendliest_letters₂.rows) == 3
end

# => MethodError: no method matching zero(::Type{Any})
# @testset "Order by / NULL value as nothing" begin
#   Users = feed_db_nv()

#   first_letter_fn = row -> row[:name] !== nothing ? string(row[:name][1]) : ""

#   avg_num_friends_fn =
#     rows -> sum([row[:num_friends] for row in rows if row[:num_friends] !== nothing]) / length(rows)
#   # nothing is like 0

#   enough_friends_fn = rows -> avg_num_friends_fn(rows) > 1.

#   avg_friends_by_letter =
#     select(Users, add_cols=D_SF(:first_letter => first_letter_fn)) |>
#     u -> group_by(u, group_by_cols=[:first_letter],
#         agg=D_SF(:avg_num_friends => avg_num_friends_fn)),
#                   having=enough_friends_fn)

#   friendliest_letters₂ = avg_friends_by_letter |>
#     u -> order_by(u, row -> -row[:avg_num_friends]) |>
#     u -> limit(u, 3)
#   #

#   @test sort(friendliest_letters₂.columns) == [:avg_num_friends, :first_letter]
#   @test friendliest_letters₂.rows[1][:first_letter] == "A"
#   @test length(friendliest_letters₂.rows) == 3
# end

@testset "Join" begin
  Users = feed_db(pkey=:user_id)

  User_Interests = Table([:user_id => Int, :interest => String]) ## Default pkey
  insert(User_Interests, [
      [1, 0, "SQL"],
      [2, 0, "NoSQL"],
      [3, 2, "SQL"],
      [4, 2, "MySQL"],
      [5, 7, "PostgreSQL"],
      [6, 7, "SQL"]
  ])

  sql_users = join(Users, User_Interests) |>
    u -> where(u, r -> r[:interest] == "SQL") |>
    u -> select(u, keep_cols=[:name])

  @test length(sql_users.rows) == 3

  sql_users₂ = join(Users, User_Interests; left_join=true) |>
    u -> where(u, r -> r[:interest] !== nothing && occursin(r"SQL\z", r[:interest])) |>
    u -> select(u, keep_cols=[:name])

  @test length(sql_users₂.rows) == 6
end

max_stars_repo_path: Data Science From Scratch/test/ya_db_test.jl

max_stars_repo_name: pascal-p/julia-notebooks

max_issues_repo_name: pascal-p/julia-notebooks

max_forks_repo_name: pascal-p/julia-notebooks


text:
function CheckValveCoefsOver(Q_base::Dict{String, Array{Float64,1}})
    aCheckValve = Dict{String, Array{Float64,1}}()
    bCheckValve = Dict{String, Array{Float64,1}}()
    for (name, Q_base_current) in Q_base
        Q_power_current = Array{Float64}(length(Q_base_current))
        aCheckValve_current = Array{Float64}(length(Q_base_current)-1)
        bCheckValve_current = Array{Float64}(length(Q_base_current)-1)
        for i = 1:length(Q_base_current)
            Q_power_current[i] = Q_base_current[i]^1.852
        end
        for i = 1:length(Q_base_current)-1
            bCheckValve_current[i] = (Q_power_current[i+1] - Q_power_current[i])/(Q_base_current[i+1] - Q_base_current[i])
            aCheckValve_current[i] = Q_power_current[i] - bCheckValve_current[i] * Q_base_current[i]
        end
        aCheckValve[name] = aCheckValve_current
        bCheckValve[name] = bCheckValve_current
    end
    return(aCheckValve, bCheckValve)
end

function CheckValveCoefsUnder(Q_base::Dict{String, Array{Float64,1}})
    aCheckValve = Dict{String, Array{Float64,1}}()
    bCheckValve = Dict{String, Array{Float64,1}}()
    for (name, Q_base_current) in Q_base
        Q_power_current = Array{Float64}(length(Q_base_current))
        aCheckValve_current = Array{Float64}(length(Q_base_current))
        bCheckValve_current = Array{Float64}(length(Q_base_current))
        for i = 1:length(Q_base_current)
            Q_power_current[i] = Q_base_current[i]^1.852
        end
        for i = 1:length(Q_base_current)
            bCheckValve_current[i] = 1.852*Q_base_current[i] ^0.852
            aCheckValve_current[i] = Q_power_current[i] - bCheckValve_current[i]*Q_base_current[i]
        end
        aCheckValve[name] = aCheckValve_current
        bCheckValve[name] = bCheckValve_current
    end
    return(aCheckValve, bCheckValve)
end

function Q_base_checkvalve(wn_dict::Dict{Any,Any}, Qmax::Dict{String, Float64}, n::Int64, Q_lb::Float64, logspace_ratio::Float64, dH_critical::Float64, dense_coef::Float64)
    pipes = wn_dict["pipes"]
    Q_base = Dict{String, Array{Float64,1}}()
    for (ix, pipe) in pipes
        if pipe["cv"] == true
            name = pipe["name"]
            Q_bar = 5#Qmax[name]
            if Q_bar <= Q_lb
                Q_base[name] = [0, Q_lb]
            else
                HW_multiplier = (1/pipe["roughness"])^1.852 *10.67 *pipe["length"]*pipe["diameter"]^4.87
                if Q_bar ^1.852*HW_multiplier >= dH_critical
                    density = logspace_ratio*dense_coef
                else
                    density = logspace_ratio
                end
                num_log = floor(Int64,log10(Q_bar/Q_lb)*density+1)
                Q_base[name] = [0; logspace(log10(Q_lb), log10(Q_bar), num_log +1)]
            end
            length(Q_base[name]) > n ? error("Please Choose another n > $length(Q_base[name])") : nothing
        end
    end
    return(Q_base)
end


text:
function CPP_solve(data, graph)
    """Given data, solve a CPP problem with integer programming."""

    Er = graph["req_edges"]
    Enr = graph["noreq_edges"]
    E = vcat(Er, Enr)

    depot = data[1, :depots][1]

    card_Er = nrow(Er)
    card_Enr = nrow(Enr)
    card_E = nrow(E)

    trav_cost = E[:, :trav_cost]
    serv_cost = E[:, :serv_cost]
    demand = E[:, :demand]

    ### Model definition ###
    model = Model(with_optimizer(CPLEX.Optimizer, CPX_PARAM_SCRIND=0))

    ### Variables ###
    X_tot = vcat([1 for edge in 1:card_Er], [0 for edge in 1:card_Enr])
    @variable(model, Y_tot[1:card_E], Int)
    @variable(model, K[1:data[1,:nodes]], Int)

    ### Objective ###
    @objective(model, Min, sum(X_tot.*serv_cost+Y_tot.*trav_cost))

    ### Constraint ###
    # Positivity constraint
    @constraint(model, positivity_Y[edge=1:card_E], Y_tot[edge] >= 0)
    @constraint(model, positivity_K[i=1:data[1,:nodes]], K[i] >= 0)

    # Connectivity constrain
    @constraint(model, connectivity[i=1:data[1,:nodes]],
        sum([X_tot[edge]+Y_tot[edge] for edge in 1:card_E if (E[edge,:start_node]==i)!=(E[edge,:end_node]==i)]) == 2*K[i]
        )

    ### Solve problem ###

    optimize!(model)
    stop = now()

    solving_time = stop-start

    ### Format output ###

    tours = []

    cost = sum(X_tot.*serv_cost) + sum(value.(Y_tot).*trav_cost)

    X = Dict(
        "req_arcs" => [],
        "req_edges" => [1 for edge in 1:card_Er],
        "req_edges_swap" => []
        )

    Y = Dict(
        "req_arcs" => [],
        "noreq_arcs" => [],
        "req_edges" => [value.(Y_tot[arc]) for arc in 1:card_Er],
        "noreq_edges" => [value.(Y_tot[arc]) for arc in (1+card_Er):card_E],
        "req_edges_swap" => [],
        "noreq_edges_swap" => []
        )

    tour = Dict(
        "depot" => depot,
        "cost" => cost,
        "capacity" => sum(X_tot.*demand),
        "X" => X,
        "Y" => Y
        )

    push!(tours, tour)

    return(tours, solving_time)
end

max_stars_repo_name: TheoGuyard/Groomer-Optimization-Problem

max_issues_repo_name: TheoGuyard/Groomer-Optimization-Problem

max_forks_repo_name: TheoGuyard/Groomer-Optimization-Problem


text:
function generate_test_value{T<:Signed}(::Type{T}, size)
  convert(T, rand(-size:size))
end

function generate_test_value{T<:Unsigned}(::Type{T}, size)
  convert(T, rand(0:size))
end

function generate_test_value{T<:AbstractFloat}(::Type{T}, size)
  convert(T, (rand(T) - .5) * size)
end

function generate_test_value{T<:String}(::Type{T}, size)
  convert(T, randstring(size))
end

function generate_test_value{T,d}(::Type{Array{T,d}}, size)
  dims = []
  array_size = size
  for n = 1:d
    a = rand(1:array_size)
    array_size = max(1,div(array_size,a))
    append!(dims,a)
  end
  reshape([generate_test_value(T,size) for x in 1:prod(dims)], dims...)
end

function generate_test_value{A}(::Type{Tuple{A}}, size)
  (generate_test_value(A, size),)
end

function generate_test_value{A,B}(::Type{Tuple{A,B}}, size)
  (generate_test_value(A, size),generate_test_value(B, size))
end

function generate_test_value{A,B,C}(::Type{Tuple{A,B,C}}, size)
  (generate_test_value(A, size), generate_test_value(B, size),
   generate_test_value(C, size))
end

function generate_test_value{A,B,C,D}(::Type{Tuple{A,B,C,D}}, size)
  (generate_test_value(A, size), generate_test_value(B, size),
   generate_test_value(C, size), generate_test_value(D, size))
end

## in more complex situations, types do not encode the full parametricity of a parent
## in this case we need

function generate_test_vector{A}(parent::A, size)
  vsize = rand(1:size)
  [generate_test_value(parent,size) for x in 1:vsize]
end

function generate_test_matrix{A}(parent::A, size)
  rsize = rand(1:size)
  csize = rand(1:max(1,div(size,rsize)))
  dims = [rsize, csize]
  reshape([generate_test_value(T,size) for x in 1:prod(dims)], dims...)
end

function generate_test_tuple{A}(parentA::A, size)
  (generate_test_vector(parentA,size),)
end
function generate_test_tuple{A,B}(parentA::A, parentB::B, size)
  (generate_test_vector(parentA,size),generate_test_vector(parentB,size))
end

function generate_test_tuple{A,B,C}(parentA::A, parentB::B, parentC::C, size)
  (generate_test_vector(parentA,size),generate_test_vector(parentB,size),
   generate_test_vector(parentC,size))
end

function generate_test_tuple{A,B,C,D}(parentA::A, parentB::B, parentC::C, parentD::D, size)
  (generate_test_vector(parentA,size),generate_test_vector(parentB,size),
   generate_test_vector(parentC,size),generate_test_vector(parentD,size))
end


text:
include("Quantities.jl")
module Physical
using Quantities
export as, asbase, BaseUnit, DerivedUnit, Prefix
export Uncertain

# looks in Physical/data and loads files in alpabetical order
# stores load order in Physica.loaded_files for debugging
function load_unit_system(loaddir::AbstractString)
	Quantities.reset_unit_system()
	Quantities.PUnits.reset_prefix_system()
	datapath = normpath(dirname(Base.source_path()), "../data/$(loaddir)")
	global loaded_files = sort(readdir(datapath))
	for filename in loaded_files
		include(joinpath(datapath,filename))
	end
end
include(normpath(dirname(Base.source_path()), "../data/what_to_load.jl"))

end # end module


text:
# Initialize an empty PairwiseListMatrix (PLM) for a pairwise measure in column or sequence pairs.
# These functions take care of the labels.

"Returns the column labels: use the column mapping (column number in the input MSA file) if it’s available, otherwise it uses the actual column numbers."
columnlabels(msa::AnnotatedMultipleSequenceAlignment) = haskey(getannotfile(msa), "ColMap") ? getcolumnmapping(msa) : collect(1:ncolumns(msa))
columnlabels(msa::AbstractMatrix{Residue}) = collect(1:ncolumns(msa))

"Returns the sequence labels: use the sequence names if they are available, otherwise it uses the actual sequence numbers."
sequencelabels(msa::Union{AnnotatedMultipleSequenceAlignment, MultipleSequenceAlignment}) = ASCIIString[ name for name in names(msa) ]
sequencelabels(msa::AbstractMatrix{Residue}) = ASCIIString[ string(number) for number in collect(1:nsequences(msa)) ]

"""
Initialize an empty `PairwiseListMatrix` for a pairwise measure in column pairs.
This function uses the column mapping (column number in the input MSA file)
if it’s available, otherwise it uses the actual column numbers.
You can use the positional argument to indicate the number `Type` (default: `Float64`),
if the `PairwiseListMatrix` should store the diagonal values on the list (default: `false`) and
a default value for the diagonal (default: `NaN`).
"""
function sequencepairsmatrix{T}(msa::AbstractMatrix{Residue}, ::Type{T}, diagonal::Bool, diagonalvalue::T)
    PairwiseListMatrix(T, nsequences(msa), sequencelabels(msa), diagonal, diagonalvalue)
end

sequencepairsmatrix(msa::AbstractMatrix{Residue}) = sequencepairsmatrix(msa, Float64, false, NaN)

"""
Initialize an empty `PairwiseListMatrix` for a pairwise measure in sequence pairs.
This function uses the sequence names if they are available, otherwise it uses the actual sequence numbers.
You can use the positional argument to indicate the number `Type` (default: `Float64`),
if the `PairwiseListMatrix` should store the diagonal values on the list (default: `false`) and
a default value for the diagonal (default: `NaN`).
"""
function columnpairsmatrix{T}(msa::AbstractMatrix{Residue}, ::Type{T}, diagonal::Bool, diagonalvalue::T)
    PairwiseListMatrix(T, ncolumns(msa), columnlabels(msa), diagonal, diagonalvalue)
end

columnpairsmatrix(msa::AbstractMatrix{Residue}) = columnpairsmatrix(msa, Float64, false, NaN)


text:
# QuantileLoss

using LaTeXStrings, LossFunctions, Measures, Latexify

Plots.plot(QuantileLoss(.75), c=:olive, linewidth=2)
Plots.plot!(QuantileLoss(.5), c=:orangered4, linewidth=2.5)
Plots.plot!(QuantileLoss(.25), c=:springgreen4, linewidth=2.5)

max_stars_repo_name: Ellon-M/visualizations

max_issues_repo_name: Ellon-M/visualizations

max_forks_repo_name: Ellon-M/visualizations


text:
@testset "Integration tests" begin
    @testset "Integration tests with synthetic data" begin
        in_temporary_directory() do
            rxnrel = "RXNREL.RRF"
            rxnsat = "RXNSAT.RRF"

            open(rxnrel, "w") do io
                generate_synthetic_rxnrel(io)
            end
            open(rxnsat, "w") do io
                generate_synthetic_rxnsat(io)
            end

            graph_built = build_graph(;
                showprogress = false,
                rxnrel,
                rxnsat,
            )
            my_filename_for_serialization = "my_filename.serialized"
            rm(my_filename_for_serialization; force = true, recursive = true)
            Serialization.serialize(my_filename_for_serialization, graph_built)
            graph_loaded_from_file = Serialization.deserialize(my_filename_for_serialization)
            for graph in [graph_built, graph_loaded_from_file]
                @test graph isa PharmGraph
                @test available_systems(graph) == ["ATC1", "ATC2", "ATC3", "ATC4", "ATC5", "NDC", "RXCUI"]
                @test haskey(graph, PharmClass("ATC5", "A01BC23"); normalization = true)
                @test haskey(graph, PharmClass("ATC5", "A01BC23"); normalization = false)
                @test graph[PharmClass("ATC5", "A01BC23")] == PharmClass("ATC5", "A01BC23")
                @test getindex(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == PharmClass("ATC5", "A01BC23")
                @test getindex(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == PharmClass("ATC5", "A01BC23")
                @test equivalent(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == [PharmClass("ATC5", "A01BC23")]
                @test equivalent(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == [PharmClass("ATC5", "A01BC23")]
                @test equivalent(graph, PharmClass("NDC", "12345678901")) == [PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test equivalent(graph, PharmClass("RXCUI", "1234567")) == [PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test parents(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == PharmClass[PharmClass("ATC1", "A"), PharmClass("ATC2", "A01"), PharmClass("ATC3", "A01B"), PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23")]
                @test parents(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == PharmClass[PharmClass("ATC1", "A"), PharmClass("ATC2", "A01"), PharmClass("ATC3", "A01B"), PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23")]
                @test parents(graph, PharmClass("NDC", "12345678901")) == PharmClass[PharmClass("ATC1", "A"), PharmClass("ATC2", "A01"), PharmClass("ATC3", "A01B"), PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23"), PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test parents(graph, PharmClass("RXCUI", "1234567")) == PharmClass[PharmClass("ATC1", "A"), PharmClass("ATC2", "A01"), PharmClass("ATC3", "A01B"), PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23"), PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test children(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == [PharmClass("ATC5", "A01BC23"), PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test children(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == [PharmClass("ATC5", "A01BC23"), PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test children(graph, PharmClass("NDC", "12345678901")) == [PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test children(graph, PharmClass("RXCUI", "1234567")) == [PharmClass("NDC", "12345678901"), PharmClass("RXCUI", "1234567")]
                @test all_neighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == PharmClass[PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23"), PharmClass("RXCUI", "1234567")]
                @test all_neighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == PharmClass[PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23"), PharmClass("RXCUI", "1234567")]
                @test inneighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == PharmClass[PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23")]
                @test inneighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == PharmClass[PharmClass("ATC4", "A01BC"), PharmClass("ATC5", "A01BC23")]
                @test outneighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = true) == PharmClass[PharmClass("ATC5", "A01BC23"), PharmClass("RXCUI", "1234567")]
                @test outneighbors(graph, PharmClass("ATC5", "A01BC23"); normalization = false) == PharmClass[PharmClass("ATC5", "A01BC23"), PharmClass("RXCUI", "1234567")]
                a = sort(unique(all_neighbors(graph, PharmClass("ATC5", "A01BC23"))))
                b = sort(unique(inneighbors(graph, PharmClass("ATC5", "A01BC23"))))
                c = sort(unique(outneighbors(graph, PharmClass("ATC5", "A01BC23"))))
                @test a == sort(unique(vcat(b, c)))
                @test !issubset(a, b)
                @test issubset(b, a)
                @test issubset(c, a)
                @test !issubset(c, b)
                @test length(graph) == 9
                count = 0
                for node in graph
                    count += 1
                end
                @test count == length(graph)
                @test eltype(graph) == PharmClass
            end
        end
    end
end


text:
"""
    graph_from_object(osm_data_object::Union{XMLDocument,Dict};
                      network_type::Symbol=:drive,
                      weight_type::Symbol=:time,
                      graph_type::Symbol=:static,
                      precompute_dijkstra_states::Bool=false,
                      largest_connected_component::Bool=true
                      )::OSMGraph

Creates an `OSMGraph` object from download OpenStreetMap network data, use with `download_osm_network`.

# Arguments
- `osm_data_object::Symbol`: OpenStreetMap network data parsed as either XML or Dictionary object depending on the download method.
- `network_type::Symbol=:drive`: Network type filter, pick from `:drive`, `:drive_service`, `:walk`, `:bike`, `:all`, `:all_private`, `:none`, `:rail`, must match the network type used to download `osm_data_object`.
- `weight_type::Symbol=:time`: Weight type for graph edges, pick from `:distance` (km), `:time` (hours), `:lane_efficiency` (time scaled by number of lanes).
- `graph_type::Symbol=:static`: Type of `LightGraphs.AbstractGraph`, pick from `:static` (StaticDiGraph), `:light` (DiGraph), `:simple_weighted` (SimpleWeightedDiGraph), `:meta` (MetaDiGraph).
- `precompute_dijkstra_states::Bool=false`: Set true to precompute dijkstra parent states for every source node in the graph, *NOTE* this may take a while and may not be possible for graphs with large amount of nodes due to memory limits.
- `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network.

# Return
- `OSMGraph`: Container for storing OpenStreetMap node, way, relation and graph related obejcts.
"""
function graph_from_object(osm_data_object::Union{XMLDocument,Dict};
                           network_type::Symbol=:drive,
                           weight_type::Symbol=:time,
                           graph_type::Symbol=:static,
                           precompute_dijkstra_states::Bool=false,
                           largest_connected_component::Bool=true
                           )::OSMGraph
    g = init_graph_from_object(osm_data_object, network_type)
    add_node_and_edge_mappings!(g)
    add_weights!(g, weight_type)
    add_graph!(g, graph_type)
    # Finding connected components can only be done after LightGraph object has been constructed
    largest_connected_component && trim_to_largest_connected_component!(g, g.graph, weight_type, graph_type) # Pass in graph to make type stable
    add_node_tags!(g)
    !(network_type in [:bike, :walk]) && add_indexed_restrictions!(g)

    if precompute_dijkstra_states
        add_dijkstra_states!(g)
    else
        U = DEFAULT_OSM_INDEX_TYPE
        g.dijkstra_states = Vector{Vector{U}}(undef, length(g.nodes))
    end

    add_kdtree!(g)
    @info "Created OSMGraph object with kwargs: network_type=$network_type, weight_type=$weight_type, graph_type=$graph_type, precompute_dijkstra_states=$precompute_dijkstra_states, largest_connected_component=$largest_connected_component"
    return g
end

"""
    graph_from_object(file_path::String;
                      network_type::Symbol=:drive,
                      weight_type::Symbol=:time,
                      graph_type::Symbol=:static,
                      precompute_dijkstra_states::Bool=false,
                      largest_connected_component::Bool=true
                      )::OSMGraph

Creates an `OSMGraph` object from a downloaded OpenStreetMap network data file, the extention must be either `.json`, `.osm` or `.xml`.

# Arguments
- `file_path::String`: OpenStreetMap network data file location.
- `network_type::Symbol=:drive`: Network type filter, pick from `:drive`, `:drive_service`, `:walk`, `:bike`, `:all`, `:all_private`, `:none`, `:rail`, must match the network type used to download `osm_data_object`.
- `weight_type::Symbol=:time`: Weight type for graph edges, pick from `:distance` (km), `:time` (hours), `:lane_efficiency` (time scaled by number of lanes).
- `graph_type::Symbol=:static`: Type of `LightGraphs.AbstractGraph`, pick from `:static` (StaticDiGraph), `:light` (DiGraph), `:simple_weighted` (SimpleWeightedDiGraph), `:meta` (MetaDiGraph).
- `precompute_dijkstra_states::Bool=false`: Set true to precompute dijkstra parent states for every source node in the graph, *NOTE* this may take a while and may not be possible for graphs with large amount of nodes due to memory limits.
- `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network.

# Return
- `OSMGraph`: Container for storing OpenStreetMap node, way, relation and graph related obejcts.
"""
function graph_from_file(file_path::String;
                         network_type::Symbol=:drive,
                         weight_type::Symbol=:time,
                         graph_type::Symbol=:static,
                         precompute_dijkstra_states::Bool=false,
                         largest_connected_component::Bool=true
                         )::OSMGraph

    !isfile(file_path) && throw(ArgumentError("File $file_path does not exist"))
    deserializer = file_deserializer(file_path)
    obj = deserializer(file_path)
    return graph_from_object(obj,
                             network_type=network_type,
                             weight_type=weight_type,
                             graph_type=graph_type,
                             precompute_dijkstra_states=precompute_dijkstra_states,
                             largest_connected_component=largest_connected_component)
end

"""
    graph_from_download(download_method::Symbol;
                        network_type::Symbol=:drive,
                        metadata::Bool=false,
                        download_format::Symbol=:json,
                        save_to_file_location::Union{String,Nothing}=nothing,
                        weight_type::Symbol=:time,
                        graph_type::Symbol=:static,
                        precompute_dijkstra_states::Bool=false,
                        largest_connected_component::Bool=true,
                        download_kwargs...
                        )::OSMGraph

Downloads OpenStreetMap network data and creates an `OSMGraph` object.

# Arguments
- `download_method::Symbol`: Download method, choose from `:place_name`, `:bbox` or `:point`.
- `network_type::Symbol=:drive`: Network type filter, pick from `:drive`, `:drive_service`, `:walk`, `:bike`, `:all`, `:all_private`, `:none`, `:rail`.
- `metadata::Bool=false`: Set true to return metadata.
- `download_format::Symbol=:json`: Download format, either `:osm`, `:xml` or `json`.
- `save_to_file_location::Union{String,Nothing}=nothing`: Specify a file location to save downloaded data to disk.
- `weight_type::Symbol=:time`: Weight type for graph edges, pick from `:distance` (km), `:time` (hours), `:lane_efficiency` (time scaled by number of lanes).
- `graph_type::Symbol=:static`: Type of `LightGraphs.AbstractGraph`, pick from `:static` (StaticDiGraph), `:light` (DiGraph), `:simple_weighted` (SimpleWeightedDiGraph), `:meta` (MetaDiGraph).
- `precompute_dijkstra_states::Bool=false`: Set true to precompute dijkstra parent states for every source node in the graph, *NOTE* this may take a while and may not be possible for graphs with large amount of nodes due to memory limits.
- `largest_connected_component::Bool=true`: Set true to keep only the largest connected components in the network.

# Required Kwargs for each Download Method

*`download_method=:place_name`*
- `place_name::String`: Any place name string used as a search argument to the Nominatim API.

*`download_method=:bbox`*
- `minlat::AbstractFloat`: Bottom left bounding box latitude coordinate.
- `minlon::AbstractFloat`: Bottom left bounding box longitude coordinate.
- `maxlat::AbstractFloat`: Top right bounding box latitude coordinate.
- `maxlon::AbstractFloat`: Top right bounding box longitude coordinate.

*`download_method=:point`*
- `point::GeoLocation`: Centroid point to draw the bounding box around.
- `radius::Number`: Distance (km) from centroid point to each bounding box corner.

*`download_method=:polygon`*
- `polygon::AbstractVector`: Vector of longitude-latitude pairs.

# Network Types
- `:drive`: Motorways excluding private and service ways.
- `:drive_service`: Motorways including private and service ways.
- `:walk`: Walkways only.
- `:bike`: Cycleways only.
- `:all`: All motorways, walkways and cycleways excluding private ways.
- `:all_private`: All motorways, walkways and cycleways including private ways.
- `:none`: No network filters.
- `:rail`: Railways excluding proposed and platform.

# Return
- `OSMGraph`: Container for storing OpenStreetMap node, way, relation and graph related obejcts.
"""
function graph_from_download(download_method::Symbol;
                             network_type::Symbol=:drive,
                             metadata::Bool=false,
                             download_format::Symbol=:json,
                             save_to_file_location::Union{String,Nothing}=nothing,
                             weight_type::Symbol=:time,
                             graph_type::Symbol=:static,
                             precompute_dijkstra_states::Bool=false,
                             largest_connected_component::Bool=true,
                             download_kwargs...
                             )::OSMGraph
    obj = download_osm_network(download_method,
                               network_type=network_type,
                               metadata=metadata,
                               download_format=download_format,
                               save_to_file_location=save_to_file_location;
                               download_kwargs...)
    return graph_from_object(obj,
                             network_type=network_type,
                             weight_type=weight_type,
                             graph_type=graph_type,
                             precompute_dijkstra_states=precompute_dijkstra_states,
                             largest_connected_component=largest_connected_component)
end

"""
Adds mappings between nodes, edges and highways to `OSMGraph`.
"""
function add_node_and_edge_mappings!(g::OSMGraph{U,T,W}) where {U <: Integer,T <: Integer,W <: Real}
    for (way_id, way) in g.highways
        @inbounds for (i, node_id) in enumerate(way.nodes)
            if haskey(g.node_to_highway, node_id)
                push!(g.node_to_highway[node_id], way_id)
            else
                g.node_to_highway[node_id] = [way_id]
            end

            if i < length(way.nodes)
                if !way.tags["reverseway"]::Bool
                    o = way.nodes[i] # origin
                    d = way.nodes[i + 1] # destination
                else
                    o = way.nodes[i + 1] # origin and destination reversed
                    d = way.nodes[i]
                end

                g.edge_to_highway[[o, d]] = way_id

                if !way.tags["oneway"]::Bool
                    g.edge_to_highway[[d, o]] = way_id
                end
            end
        end
    end

    @assert(length(g.nodes) == length(g.node_to_highway), "Data quality issue: number of graph nodes ($(length(g.nodes::Dict{T,Node{T}}))) not equal to set of nodes extracted from highways ($(length(g.node_to_highway)))")
    g.node_to_index = OrderedDict{T,U}(n => i for (i, n) in enumerate(collect(keys(g.nodes))))
    g.index_to_node = OrderedDict{U,T}(i => n for (n, i) in g.node_to_index)
end

"""
Adds maxspeed and lanes tags to every `OSMGraph` node.
"""
function add_node_tags!(g::OSMGraph)
    # Custom mean used to minimise allocations
    @inline function _roundedmean(hwys, subkey, T)
        total = zero(T)
        for id in hwys
            total += g.highways[id].tags[subkey]::T
        end
        return round(T, total/length(hwys))
    end

    M = DEFAULT_OSM_MAXSPEED_TYPE
    L = DEFAULT_OSM_LANES_TYPE

    for (id, data) in g.nodes
        highways = g.node_to_highway[id]
        tags_dict = g.nodes[id].tags::Dict{String, Any}
        tags_dict["maxspeed"] = _roundedmean(highways, "maxspeed", M)
        tags_dict["lanes"] = _roundedmean(highways, "lanes", L)
        push!(g.node_coordinates, [data.location.lat, data.location.lon])
    end
end

"""
Finds the adjacent node id on a given way.
"""
function adjacent_node(g::OSMGraph, node::T, way::T)::Union{T,Vector{<:T}} where T <: Integer
    way_nodes = g.highways[way].nodes
    if node == way_nodes[1]
        return way_nodes[2]
    elseif node == way_nodes[end]
        return way_nodes[length(way_nodes) - 1]
    else
        idx = findfirst(isequal(node), way_nodes)
        is_oneway = g.highways[way].tags["oneway"]
        is_reverseway = g.highways[way].tags["reverseway"]
        if is_oneway && !is_reverseway
            return way_nodes[idx + 1]
        elseif is_oneway && is_reverseway
            return way_nodes[idx - 1]
        else
            # via_node is in the middle of a non-oneway highway, this is only the possible when the
            # restriction is "only_straight_on", meaning vehicles can neither turn left nor right.
            return [way_nodes[idx - 1], way_nodes[idx + 1]]
        end
    end
end

"""
Adds restrictions linked lists to `OSMGraph`.

# Example
`[from_way_node_index, ...via_way_node_indices..., to_way_node_index]`
"""
function add_indexed_restrictions!(g::OSMGraph{U,T,W}) where {U <: Integer,T <: Integer,W <: Real}
    g.indexed_restrictions = DefaultDict{U,Vector{MutableLinkedList{U}}}(Vector{MutableLinkedList{U}})

    for (id, r) in g.restrictions
        if r.type == "via_node"
            if r.is_exclusion
                # no_left_turn, no_right_turn, no_u_turn, no_straight_on
                restricted_to_ways = [r.to_way]
            elseif r.is_exclusive
                # only_right_turn, only_left_turn, only_straight_on
                # Multiple to_ways, e.g. if only_right_turn, then left turn and straight on are restricted
                all_ways = g.node_to_highway[r.via_node]
                permitted_to_way = r.to_way
                restricted_to_ways = [w for w in all_ways if w != r.from_way && w != permitted_to_way]
            else
                continue
            end

            from_node = adjacent_node(g, r.via_node, r.from_way)::T
            for to_way in restricted_to_ways
                to_node_temp = adjacent_node(g, r.via_node, to_way)
                to_node = isa(to_node_temp, Integer) ? [to_node_temp] : to_node_temp

                for tn in to_node
                    # only_straight_on restrictions may have multiple to_nodes
                    indices = [g.node_to_index[n] for n in [tn, r.via_node::T, from_node]]
                    push!(g.indexed_restrictions[g.node_to_index[r.via_node]], MutableLinkedList{U}(indices...))
                end
            end

        elseif r.type == "via_way"
            via_way_nodes_list = [g.highways[w].nodes for w in r.via_way::Vector{T}]
            via_way_nodes = join_arrays_on_common_trailing_elements(via_way_nodes_list...)::Vector{T}

            from_way_nodes = g.highways[r.from_way].nodes
            from_via_intersection_node = first_common_trailing_element(from_way_nodes, via_way_nodes)
            from_node = adjacent_node(g, from_via_intersection_node, r.from_way)::T

            to_way_nodes = g.highways[r.to_way].nodes
            to_via_intersection_node = first_common_trailing_element(to_way_nodes, via_way_nodes)
            to_node = adjacent_node(g, to_via_intersection_node, r.to_way)::T

            if to_via_intersection_node == via_way_nodes[end]
                # Ordering matters, see doc string, but
                # we don't want to reorder the array in the Way object
                via_way_nodes = reverse(via_way_nodes)
            end

            indices = [g.node_to_index[n] for n in [to_node, via_way_nodes..., from_node]]
            push!(g.indexed_restrictions[g.node_to_index[to_via_intersection_node]], MutableLinkedList{U}(indices...))
        end
    end
end

"""
Adds edge weights to `OSMGraph`.
"""
function add_weights!(g::OSMGraph, weight_type::Symbol=:distance)
    n_edges = length(g.edge_to_highway)
    o_indices = Vector{Int}(undef, n_edges) # edge origin node indices
    d_indices = Vector{Int}(undef, n_edges) # edge destination node indices
    weights = Vector{Float64}(undef, n_edges)

    W = DEFAULT_OSM_EDGE_WEIGHT_TYPE

    @inbounds for (i, edge) in enumerate(keys(g.edge_to_highway))
        o_loc = g.nodes[edge[1]].location
        d_loc = g.nodes[edge[2]].location

        o_indices[i] = g.node_to_index[edge[1]]
        d_indices[i] = g.node_to_index[edge[2]]

        dist = distance(o_loc, d_loc, :haversine)
        if weight_type == :time || weight_type == :lane_efficiency
            highway = g.edge_to_highway[edge]
            maxspeed = g.highways[highway].tags["maxspeed"]::DEFAULT_OSM_MAXSPEED_TYPE
            if weight_type == :time
                weight = dist / maxspeed
            else
                lanes = g.highways[highway].tags["lanes"]::DEFAULT_OSM_LANES_TYPE
                lane_efficiency = get(LANE_EFFICIENCY, lanes, 1.0)
                weight = dist / (maxspeed * lane_efficiency)
            end
        else
            # Distance
            weight = dist
        end
        weights[i] = max(weight, eps(W))
    end

    n = length(g.nodes)
    g.weights = sparse(o_indices, d_indices, weights, n, n)
    g.weight_type = weight_type
    return g
end

"""
Adds a LightGraphs.AbstractGraph object to `OSMGraph`.
"""
function add_graph!(g::OSMGraph, graph_type::Symbol=:static)
    if graph_type == :light
        g.graph = DiGraph(g.weights)
    elseif graph_type == :static
        g.graph = StaticDiGraph(DiGraph(g.weights))
    elseif graph_type == :simple_weighted
        g.graph = SimpleWeightedDiGraph(g.weights)
    elseif graph_type == :meta
        g.graph = MetaDiGraph(DiGraph(g.weights))
        for (o, d, w) in zip(findnz(transpose(g.weights))...)
            set_prop!(g.graph, o, d, :weight, w)
        end
    else
        throw(ErrorException("Graph type $graph_type not implemented"))
    end
end

"""
Trims graph object to the largest connected component.
"""
function trim_to_largest_connected_component!(g::OSMGraph{U, T, W}, graph, weight_type::Symbol=:time, graph_type::Symbol=:static) where {U, T, W}
    cc = weakly_connected_components(graph)
    sort!(cc, by=x -> length(x), rev=true)
    indices_to_delete = flatten(cc[2:end])
    nodes_to_delete = [g.index_to_node[i] for i in indices_to_delete]
    highways_to_delete = Set(flatten([g.node_to_highway[n] for n in nodes_to_delete]))

    delete_from_dict!(g.nodes, nodes_to_delete, :on_key)
    delete_from_dict!(g.node_to_index, nodes_to_delete, :on_key)
    delete_from_dict!(g.node_to_highway, nodes_to_delete, :on_key)
    delete_from_dict!(g.index_to_node, indices_to_delete, :on_key)
    delete_from_dict!(g.highways, highways_to_delete, :on_key)
    delete_from_dict!(g.edge_to_highway, highways_to_delete, :on_value)

    for (id, r) in g.restrictions
        if r.via_node !== nothing
            !isempty(intersect([r.via_node::T], nodes_to_delete)) && delete!(g.restrictions, id)
        elseif r.via_way !== nothing
            ways = Set([r.from_way, r.to_way, r.via_way::Vector{T}...])
            !isempty(intersect(ways, highways_to_delete)) && delete!(g.restrictions, id)
        end
    end

    # Trim then rebuild graph so indices are in order
    add_node_and_edge_mappings!(g)
    add_weights!(g, weight_type)
    add_graph!(g, graph_type)
    return g
end

"""
Adds precomputed dijkstra states for every source node in `OSMGraph`, not recommended for graphs with greater than 50k nodes.
"""
function add_dijkstra_states!(g::OSMGraph{U,T,W}) where {U <: Integer,T <: Integer,W <: Real}
    srcs = collect(vertices(g.graph)) # source vertices (origin node indices)

    n = length(srcs)
    @info "Precomputing $n Dijkstra States, this might take a while..."
    n > 50000 && @warn "Precomputing and caching all $n Dijkstra States may not be possible due to memory limits"
    g.dijkstra_states = Vector{Vector{U}}(undef, n)

    Threads.@threads for src in srcs
        g.dijkstra_states[src] = dijkstra(g.graph, src, distmx=g.weights, restrictions=g.indexed_restrictions)
    end
end

"""
Adds KDTree to `OSMGraph` for finding nearest neighbours.
"""
function add_kdtree!(g::OSMGraph)
    node_locations = [node.location for (id, node) in g.nodes]  # node locations must have the same order as node indices
    cartesian_locations = to_cartesian(node_locations)
    g.kdtree = KDTree(cartesian_locations)
end


text:
function Cartpole(method=:none)

    opts = SolverOptions(
        cost_tolerance_intermediate=1e-2,
        penalty_scaling=10.,
        penalty_initial=1.0
    )

    model = Dynamics.Cartpole()
    n,m = size(model)
    N = 101
    tf = 5.
    dt = tf/(N-1)

    Q = 1.0e-2*Diagonal(@SVector ones(n))
    Qf = 100.0*Diagonal(@SVector ones(n))
    R = 1.0e-1*Diagonal(@SVector ones(m))
    x0 = @SVector zeros(n)
    xf = @SVector [0, pi, 0, 0]
    obj = LQRObjective(Q,R,Qf,xf,N)

    u_bnd = 3.0
    bnd = BoundConstraint(n,m, u_min=-u_bnd, u_max=u_bnd)
    goal = GoalConstraint(xf)
    con_bnd = ConstraintVals(bnd, 1:N-1)
    con_goal = ConstraintVals(goal, N:N)
    conSet = ConstraintSet(n,m,[con_bnd, con_goal], N)

    X0 = [@SVector fill(NaN,n) for k = 1:N]
    u0 = @SVector fill(0.01,m)
    U0 = [u0 for k = 1:N-1]
    Z = Traj(X0,U0,dt*ones(N))
    prob = Problem{RK3}(model, obj, conSet, x0, xf, Z, N, 0.0, tf)
    rollout!(prob)

    return prob, opts
end


text: # add extra loggers here

max_stars_repo_name: anirudhmurali/hello-julia-genie

max_issues_repo_name: anirudhmurali/hello-julia-genie

max_forks_repo_name: anirudhmurali/hello-julia-genie


text:
using DataDeps

register(DataDep(
  "trains",
  "http://archive.ics.uci.edu/ml/datasets/Trains",
  "http://archive.ics.uci.edu/ml/machine-learning-databases/trains/trains-transformed.data",
  "d536c86b5d3b913e5a494dce1ea6f09a2b756b29831ce9156c02fe7ba25a556e",
  post_fetch_method=(path -> begin
    # The missing values are `-`. Replace with `?`
    write(path, replace(read(path, String), "-" => "?"))

    UCIData.process_dataset(path,
        target_index=33,
        feature_indices=1:32,
        categoric_indices=[4:5; 7; 9:10; 12; 14:15; 17; 19:20; 22:32],
        delim=' ',
        footerskip=1,
        ignorerepeated=true,
    )
  end),
))

max_stars_repo_name: JackDunnNZ/uci-data

max_issues_repo_name: JackDunnNZ/uci-data

max_forks_repo_name: JackDunnNZ/uci-data


text:
using Plots
using DelimitedFiles

data = readdlm("./gr.dat")

r = data[:,1]
g = data[:,2]

plot(r,g,label="",linewidth=2)
plot!(xlabel="r / Angstroms",ylabel="MDDF")

plot!(size=(300,300))

savefig("./gr.pdf")


text:
module Flux

# Zero Flux Given

using Base: tail
using Zygote, MacroTools, Juno, Reexport, Statistics, Random
using MacroTools: @forward
@reexport using NNlib
using Zygote: Params, @adjoint, gradient, pullback
export gradient

export Chain, Dense, Maxout, RNN, LSTM, GRU, Conv, CrossCor, ConvTranspose, MaxPool, MeanPool,
       DepthwiseConv, Dropout, AlphaDropout, LayerNorm, BatchNorm, InstanceNorm, GroupNorm,
       SkipConnection, params, fmap, cpu, gpu, f32, f64

include("optimise/Optimise.jl")
using .Optimise
using .Optimise: @epochs
export SGD, Descent, ADAM, Momentum, Nesterov, RMSProp,
  ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM,
  ADAMW, RADAM, InvDecay, ExpDecay, WeightDecay

allow_cuda() = parse(Bool, get(ENV, "FLUX_USE_CUDA", "true"))
const consider_cuda = allow_cuda()

using CUDAapi
const use_cuda = consider_cuda && has_cuda()
if use_cuda
  try
    using CuArrays
  catch
    @error "CUDA is installed, but CuArrays.jl fails to load. Please fix the issue, or load Flux with FLUX_USE_CUDA=false."
    rethrow()
  end
end

include("utils.jl")
include("onehot.jl")
include("functor.jl")

include("layers/stateless.jl")
include("layers/basic.jl")
include("layers/conv.jl")
include("layers/recurrent.jl")
include("layers/normalise.jl")

include("data/Data.jl")

include("deprecations.jl")

if use_cuda
  include("cuda/cuda.jl")
end

function __init__()
  # check if the GPU usage conditions that are baked in the precompilation image
  # match the current situation, and force a recompilation if not.
  if (allow_cuda() != consider_cuda) || (consider_cuda && has_cuda() != use_cuda)
      cachefile = if VERSION >= v"1.3-"
          Base.compilecache_path(Base.PkgId(Flux))
      else
          abspath(DEPOT_PATH[1], Base.cache_file_entry(Base.PkgId(Flux)))
      end
      rm(cachefile)
      error("Your set-up changed, and Flux.jl needs to be reconfigured. Please load the package again.")
  end
end

end # module


text:
using CxxWrap

module JERI
  using CxxWrap

  @wrapmodule joinpath(@__DIR__,"../libjeri.so") :define_jeri

  function __init__()
    @initcxx
  end

  export initialize, finalize
  export Atom, create_atom
  export Shell, create_shell
  export BasisSet
  export ShellPair, precompute_shell_pair_data

  export OEIEngine
  export compute_overlap_block, compute_overlap_grad_block
  export compute_kinetic_block, compute_kinetic_grad_block
  export compute_nuc_attr_block, compute_nuc_attr_grad_block

  export PropEngine, compute_dipole_block

  export TEIEngine, compute_eri_block
end


text:
using Turing, ContinuousBenchmarks

include(joinpath(ContinuousBenchmarks.STAN_DATA_DIR, "normal-loc.data.jl"))
include(joinpath(ContinuousBenchmarks.STAN_MODELS_DIR, "normal-loc.model.jl"))

nlchain = sample(nlmodel(nldata[:y]), HMC(1000, 0.05, 3))
print(nlchain[:mu])


text:
# module Ecto.MirrorTypes

import Base: isless, !, ==, string

isless(n::Int, field::MirrorField) = MirrorPredicate(vcat(n, " < ", field))
isless(field::MirrorField, n::Int) = MirrorPredicate(vcat(field, " < ", n))
!(pred::MirrorPredicate) = MirrorPredicate(vcat("!(", pred.body, ")"))
==(field::MirrorField, n::Int) = MirrorPredicate(vcat(field, "==", n))
==(n::Int, field::MirrorField) = ==(field, n)

string(field::MirrorField) = string(field.typ, '.', field.name)

function construct(pred::MirrorPredicate, alias::Dict{Type,Symbol})
    join(map(x-> isa(x, MirrorField) ? construct(x, alias) : x, pred.body))
end

function construct(fld::MirrorField, alias::Dict{Type,Symbol})
    string(alias[fld.typ], '.', fld.name)
end

function build_alias(vec::Vector{Type})::Dict{Type,Symbol}
    vecdict = Dict{Char,Vector{Type}}()
    for t in vec
        ch = lowercase(string(t.name.name))[1]
        if !haskey(vecdict,ch)
            vecdict[ch] = Vector{Type}()
        end
        push!(vecdict[ch], t)
    end
    typedict = Dict{Type,Symbol}()
    for (k,v) in vecdict
        if 1 == length(v)
            typedict[first(v)] = Symbol(k)
        else
            for (idx,t) in enumerate(v)
                typedict[t] = Symbol(k,idx)
            end
        end
    end
    typedict
end

# Ref: julia/base/strings/io.jl join
function join_with(vec::Vector, delim)
    i = start(vec)
    is_done = done(vec,i)
    result = Vector()
    while !is_done
        el, i = next(vec,i)
        is_done = done(vec,i)
        result = vcat(result, el)
        if !is_done
            push!(result, delim)
        end
    end
    result
end


text:
export marginalrule

@marginalrule MvNormalMeanCovariance(:out_μ_Σ) (m_out::MultivariateNormalDistributionsFamily, m_μ::PointMass, m_Σ::PointMass) = begin
    return (out = prod(ProdAnalytical(), MvNormalMeanCovariance(mean(m_μ), mean(m_Σ)), m_out), μ = m_μ, Σ = m_Σ)
end

@marginalrule MvNormalMeanCovariance(:out_μ_Σ) (m_out::PointMass, m_μ::MultivariateNormalDistributionsFamily, m_Σ::PointMass) = begin
    return (out = m_out, μ = prod(ProdAnalytical(), m_μ, MvNormalMeanCovariance(mean(m_out), mean(m_Σ))), Σ = m_Σ)
end

@marginalrule MvNormalMeanCovariance(:out_μ_Σ) (m_out::MultivariateNormalDistributionsFamily, m_μ::MultivariateNormalDistributionsFamily, m_Σ::PointMass) = begin
    xi_out, W_out = weightedmean_precision(m_out)
    xi_m, W_m = weightedmean_precision(m_μ)

    W_bar = cholinv(mean(m_Σ))

    xi = [ xi_out; xi_m ]

    T = promote_type(eltype(W_bar), eltype(W_out), eltype(W_m))
    d = length(xi_out)
    W = Matrix{T}(undef, (2*d, 2*d))
    @inbounds for k2 in 1:d
        for k1 in 1:d
            tmp1 = W_bar[k1,k2]
            tmp2 = -tmp1
            k1d = k1+d
            k2d = k2+d
            W[k1,k2] = tmp1 + W_out[k1,k2]
            W[k1d,k2] = tmp2
            W[k1,k2d] = tmp2
            W[k1d,k2d] = tmp1 + W_m[k1,k2]
        end
    end

    # naive:
    # xi = [ xi_out; xi_m ]
    # W  = [ W_out + W_bar -W_bar; -W_bar W_m + W_bar ]

    return (out_μ = MvNormalWeightedMeanPrecision(xi, W), Σ = m_Σ)
end

@marginalrule MvNormalMeanCovariance(:out_μ) (m_out::MultivariateNormalDistributionsFamily, m_μ::MultivariateNormalDistributionsFamily, q_Σ::Any) = begin
    xi_out, W_out = weightedmean_precision(m_out)
    xi_m, W_m = weightedmean_precision(m_μ)

    W_bar = cholinv(mean(q_Σ))

    xi = [ xi_out; xi_m ]

    T = promote_type(eltype(W_bar), eltype(W_out), eltype(W_m))
    d = length(xi_out)
    W = Matrix{T}(undef, (2*d, 2*d))
    @inbounds for k2 in 1:d
        for k1 in 1:d
            tmp1 = W_bar[k1,k2]
            tmp2 = -tmp1
            k1d = k1+d
            k2d = k2+d
            W[k1,k2] = tmp1 + W_out[k1,k2]
            W[k1d,k2] = tmp2
            W[k1,k2d] = tmp2
            W[k1d,k2d] = tmp1 + W_m[k1,k2]
        end
    end

    # naive:
    # xi = [ xi_out; xi_m ]
    # W  = [ W_out + W_bar -W_bar; -W_bar W_m + W_bar ]

    return MvNormalWeightedMeanPrecision(xi, W)
end

@marginalrule MvNormalMeanCovariance(:out_μ) (m_out::PointMass, m_μ::MultivariateNormalDistributionsFamily, q_Σ::Any) = begin
    return (out = m_out, μ = prod(ProdAnalytical(), MvNormalMeanCovariance(mean(m_out), mean(q_Σ)), m_μ))
end

@marginalrule MvNormalMeanCovariance(:out_μ) (m_out::MultivariateNormalDistributionsFamily, m_μ::PointMass, q_Σ::Any) = begin
    return (out = prod(ProdAnalytical(), MvNormalMeanCovariance(mean(m_μ), mean(q_Σ)), m_out), μ = m_μ)
end


text:
# SPDX-License-Identifier: X11
# 2020-11-20
# Trick or Treat (200pt)

function main()
  N,K = parse.(Int, (split ∘ readline)())
  snacks = zeros(Int, N)
  for i ∈ 1:K
    readline()
    for x ∈ parse.(Int, (split ∘ readline)())
      snacks[x] += 1
    end
  end
  (println ∘ length ∘ filter)(iszero, snacks)
end

main()

max_stars_repo_name: jthvai/competitive-programming

max_issues_repo_name: jthvai/competitive-programming

max_forks_repo_name: jthvai/competitive-programming


text:
@with_kw struct actM1{A,B,C} <: RTGmodel
    model::A = actM1_model()
    protein_lookup = get_protein_lookup(model)
    p::C = init_p(model; idx_hill_coefs=[])
    u::B = init_u(model, protein_lookup)
end

"""
Activation Model:
"""
function actM1_model()
    actM1 = @reaction_network begin
        # NLSact: Nuclear Localization Signal (NLS) activation
        (1), BmhMks → BmhMks
        (k2n + k1*(BmhMks)/(k1m+BmhMks)), Rtg13_a_c → Rtg13_i_c
        (k2, k2n), Rtg3_i_c ↔ Rtg3_a_c
        # Rtg1Binding: Rtg13 formation
        (k3, kn3), Rtg1_c + Rtg3_a_c ↔ Rtg13_a_c
        (k3, kn3), Rtg1_c + Rtg3_i_c ↔ Rtg13_i_c
        # Translocation
        (k4, kn4), Rtg13_a_c ↔ Rtg13_a_n
        (k4, kn4), Rtg3_a_c ↔ Rtg3_a_n
    end k1 k1m k2 k2n k3 kn3 k4 kn4
    return actM1
end