text:
#///////////////////////////////////////
#// File Name: scene_loader_test.jl
#// Author: Haruki Nishimura (hnishimura@stanford.edu)
#// Date Created: 2021/01/28
#// Description: Test code for src/scene_loader.jl
#///////////////////////////////////////

using Distributions
using LinearAlgebra
using PyCall
using Random
using RobotOS

@testset "Trajectron Scene Loader Test" begin
    conf_file_name = "config.json"
    test_data_name = "eth_test.pkl";
    test_scene_id = 0;
    start_time_idx = 2;
    incl_robot_node = false;

    param = TrajectronSceneParameter(conf_file_name, test_data_name,
                                     test_scene_id, start_time_idx,
                                     incl_robot_node);
    scene_loader = TrajectronSceneLoader(param, verbose=false);

    ado_positions = fetch_ado_positions!(scene_loader);

    ado_positions_converted = convert_nodes_to_str(ado_positions);
    @test typeof(collect(keys(ado_positions_converted))[1]) == String
    @test typeof(collect(values(ado_positions_converted))[1]) == Vector{Float64}
end

@testset "Trajectron Scene Loader Test (with Robot)" begin
    conf_file_name = "config.json"
    test_data_name = "eth_test.pkl";
    test_scene_id = 0;
    start_time_idx = 2;
    incl_robot_node = true;

    param = TrajectronSceneParameter(conf_file_name, test_data_name,
                                     test_scene_id, start_time_idx,
                                     incl_robot_node);
    scene_loader = TrajectronSceneLoader(param, verbose=false);
    online_nodes_list_py = scene_loader.online_scene.nodes
    online_nodes_list = []
    for node in online_nodes_list_py
        push!(online_nodes_list, pybuiltin("str")(node))
    end

    dto = 0.4;
    sim_horizon = 10.0;
    ado_trajectory = get_trajectory_for_ado(scene_loader, Time(0.0),
                                            "PEDESTRIAN/4", sim_horizon);

    @test to_sec(minimum(ado_trajectory)[1]) ≈ 2.0;
    @test all(maximum(ado_trajectory)[2] - minimum(ado_trajectory)[2] .≈ [13.25, 0.36]);
    @test to_sec(maximum(ado_trajectory)[1]) ≈ 7.2;

    ado_positions = fetch_ado_positions!(scene_loader);

    ado_positions_converted = convert_nodes_to_str(ado_positions);
    @test typeof(collect(keys(ado_positions_converted))[1]) == String
    @test typeof(collect(values(ado_positions_converted))[1]) == Vector{Float64}
    @test !in("PEDESTRIAN/ROBOT", keys(ado_positions_converted))

    scene_loader.curr_time_idx -= 1;
    ado_states_full = fetch_ado_positions!(scene_loader, return_full_state=true);
    ado_states_full_converted = convert_nodes_to_str(ado_states_full);
    @test keys(ado_positions_converted) == keys(ado_states_full_converted);
    for key in keys(ado_states_full_converted)
        @test length(ado_states_full_converted[key]) == 6;
        @test ado_states_full_converted[key][1:2] == ado_positions_converted[key];
    end

    scene_loader_removed = TrajectronSceneLoader(param, verbose=false,
                                                 ado_id_removed="PEDESTRIAN/1")
    online_nodes_list_removed_py = scene_loader_removed.online_scene.nodes
    online_nodes_list_removed = []
    for node in online_nodes_list_removed_py
        push!(online_nodes_list_removed, pybuiltin("str")(node))
    end
    @test setdiff(online_nodes_list, online_nodes_list_removed) == ["PEDESTRIAN/1"]
end

@testset "Synthetic Scene Loader Test" begin
    rng = MersenneTwister(123);

    param = SyntheticSceneParameter(rng);
    ado_pos_dict = Dict("Pedestrian/1" => [1.0, 1.0],
                        "Pedestrian/2" => [2.0, 1.0],
                        "Pedestrian/3" => [3.0, 1.0]);
    loader = SyntheticSceneLoader(param, ado_pos_dict);
    @test loader.ado_pos_dict == ado_pos_dict;

    num_samples = 100;
    dto = 0.4;
    prediction_steps = 12;
    deterministic = false;
    predictor_param = GaussianPredictorParameter(prediction_steps, num_samples, deterministic, rng);
    ado_vel_dict = Dict("Pedestrian/1" => MvNormal([0.0, 0.0], Diagonal([1.0, 1.0])),
                        "Pedestrian/2" => MvNormal([0.0, 0.0], Diagonal([1.0, 1.0])),
                        "Pedestrian/3" => MvNormal([0.0, 0.0], Diagonal([1.0, 1.0])));
    predictor = GaussianPredictor(predictor_param, dto, ado_vel_dict);

    prediction_dict = sample_future_ado_positions!(predictor, loader.ado_pos_dict);
    ado_positions = fetch_ado_positions!(loader, prediction_dict);
    @test typeof(ado_positions) == Dict{String, Vector{Float64}}
end


text:
using Requires

abstract type Abstractfermion# <: AbstractVector{ComplexF64}
end

abstract type AbstractFermionfields{NC,Dim}  <: Abstractfermion
end

abstract type Adjoint_fermion{T} <: Abstractfermion
end

struct Adjoint_fermionfields{T} <: Adjoint_fermion{T}
    parent::T
end

function Base.adjoint(F::T) where T <: Abstractfermion
    Adjoint_fermionfields{T}(F)
end

function Base.setindex!(x::Adjoint_fermionfields{T},v,i)  where T <: Abstractfermion
    error("type $(typeof(x)) has no setindex method. This type is read only.")
end

function Base.getindex(x::Adjoint_fermionfields{T},i)  where T <: Abstractfermion
    @inbounds return conj(x.parent[i])
end

function Base.size(x::Adjoint_fermionfields{T})  where T <: Abstractfermion
    return size(x.parent)
end

abstract type Shifted_fermionfields{NC,Dim} <: Abstractfermion
end

const default_boundaryconditions = (nothing,[1,-1],nothing,[1,1,1,-1])

include("./AbstractFermions_4D.jl")
include("./AbstractFermions_5D.jl")
include("./AbstractFermions_2D.jl")
include("./AbstractFermions_3D.jl")

function __init__()
    @require MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" begin
        include("./WilsonFermion/WilsonFermion_4D_wing_mpi.jl")
        include("./WilsonFermion/WilsonFermion_4D_nowing_mpi.jl")
        include("./StaggeredFermion/StaggeredFermion_4D_nowing_mpi.jl")
        include("./DomainwallFermion/DomainwallFermion_5d_wing_mpi.jl")
        include("./DomainwallFermion/DomainwallFermion_5d_mpi.jl")
    end

end

function Initialize_pseudofermion_fields(u::AbstractGaugefields{NC,Dim},Dirac_operator::String;L5=2,nowing = false) where {NC,Dim}
    mpi = u.mpi
    if mpi
        if Dim == 4
            if Dirac_operator == "staggered"
                if nowing
                    x = StaggeredFermion_4D_nowing_mpi(u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs)
                else
                    error("Dirac_operator  = $Dirac_operator witn nowing = $nowing is not supported")
                end
                #x = Initialize_StaggeredFermion(u)
            elseif Dirac_operator == "Wilson"
                if nowing
                    x = WilsonFermion_4D_nowing_mpi(u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs)
                    #error("Dirac_operator  = $Dirac_operator with nowing = $nowing is not supported")
                else
                    x = WilsonFermion_4D_mpi(u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs)
                end
                #x = Initialize_WilsonFermion(u)
            elseif Dirac_operator == "Domainwall"
                #@warn "Domainwall fermion is not well tested!!"
                if nowing
                    x = DomainwallFermion_5D_mpi(L5,u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs,nowing=nowing)
                else
                    x = DomainwallFermion_5D_wing_mpi(L5,u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs)
                end
            elseif Dirac_operator == "MobiusDomainwall"
                @warn "MobiusDomainwall fermion is not well tested!!"
                if nowing
                    x = MobiusDomainwallFermion_5D_mpi(L5,u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs,nowing=nowing)
                else
                    x = MobiusDomainwallFermion_5D_wing_mpi(L5,u.NC,u.NX,u.NY,u.NZ,u.NT,u.PEs)
                end
            else
                error("Dirac_operator = $Dirac_operator is not supported")
            end
        else
            error("Dim = $Dim is not supported")
        end

        #error("mpi = $mpi is not supported")
    else
        if Dim == 4
            if Dirac_operator == "staggered"
                x = Initialize_StaggeredFermion(u,nowing = nowing)
            elseif Dirac_operator == "Wilson"
                x = Initialize_WilsonFermion(u,nowing = nowing)
            elseif Dirac_operator == "Domainwall"
                #@warn "Domainwall fermion is not well tested!!"
                x = Initialize_DomainwallFermion(u,L5,nowing=nowing)
            elseif Dirac_operator == "MobiusDomainwall"
                @warn "MobiusDomainwall fermion is not well tested!!"
                x = Initialize_MobiusDomainwallFermion(u,L5,nowing=nowing)
            else
                error("Dirac_operator = $Dirac_operator is not supported")
            end
        elseif Dim == 2
            if Dirac_operator == "staggered"
                x = Initialize_StaggeredFermion(u)
            elseif Dirac_operator == "Wilson"
                x = Initialize_WilsonFermion(u)
            elseif Dirac_operator == "Domainwall"
                #@warn "Domainwall fermion is not well tested!!"
                x = Initialize_DomainwallFermion(u,L5)
            elseif Dirac_operator == "MobiusDomainwall"
                @warn "MobiusDomainwall fermion is not well tested!!"
                x = Initialize_MobiusDomainwallFermion(u,L5)
            else
                error("Dirac_operator = $Dirac_operator is not supported")
            end
        else
            error("Dim = $Dim is not supported")
        end
    end

    return x
end

function Initialize_pseudofermion_fields(u::AbstractGaugefields{NC,Dim},parameters) where {NC,Dim}
    mpi = u.mpi
    if mpi
        error("mpi = $mpi is not supported")
    else
        if Dim == 4
            if Dirac_operator == "staggered"
                x = Initialize_StaggeredFermion(u)
            elseif parameters["Dirac_operator"] == "Wilson"
                x = WilsonFermion_4D_wing(parameters,u.NC,u.NX,u.NY,u.NZ,u.NT)
                #x = Initialize_WilsonFermion(u)
            else
                error("Dirac_operator = $Dirac_operator is not supported")
            end
        elseif Dim == 2
                if Dirac_operator == "staggered"
                    x = Initialize_StaggeredFermion(u)
                else
                    error("Dirac_operator = $Dirac_operator is not supported")
                end
        else
            error("Dim = $Dim is not supported")
        end
    end

    return x
end

function convert_to_normalvector(F::T) where T <: AbstractFermionfields
    error("convert_to_normalvector is not implemented in type $(typeof(F)) ")
end

function clear_fermion!(F::T) where T <: AbstractFermionfields
    error("clear_fermion! is not implemented in type $(typeof(F)) ")
end

function apply_γ5!(F::T) where T <: AbstractFermionfields
    error("apply_γ5! is not implemented in type $(typeof(F)) ")
end

function Base.similar(F::T) where T <: AbstractFermionfields
    error("Base.similar is not implemented in type $(typeof(F)) ")
end

function Base.length(F::T) where T <: AbstractFermionfields
    error("Base.length(F) is not implemented in type $(typeof(F)) ")
end

function substitute_fermion!(a::T1,b::T2) where {T1 <: AbstractFermionfields, T2  <: AbstractFermionfields}
    error("substitute_fermion!(a,b) is not implemented in type $(typeof(a)) and type $(typeof(b)) ")
end

function gauss_distribution_fermion!(F::T) where T <: AbstractFermionfields
    error("gauss_distribution_fermi! is not implemented in type $(typeof(F)) ")
end

function set_wing_fermion!(F::Vector{<: AbstractFermionfields{NC,Dim}},boundarycondition) where {NC,Dim}
    for μ=1:Dim
        set_wing_fermion!(F[μ],boundarycondition)
    end
end

function set_wing_fermion!(F::T,boundarycondition) where T <: AbstractFermionfields
    error("set_wing_fermion!(F,boundarycondition) is not implemented in type $(typeof(F)) ")
end

function set_wing_fermion!(F::AbstractFermionfields{NC,Dim}) where {NC,Dim}
    set_wing_fermion!(F,default_boundaryconditions[Dim])
end

function Dx!(temp::T,U,x,temps)  where T <: AbstractFermionfields
    error("Dx!(temp,U,x,temps) is not implemented in type $(typeof(temp)) ")
end

function LinearAlgebra.axpby!(a::Number, X::T1, b::Number, Y::T2) where {T1 <: Abstractfermion, T2 <: Abstractfermion}
    error("LinearAlgebra.axpby(! (Y <- a*X + b*Y ) is not implemented in type X:$T1,Y:$T2")
end

function LinearAlgebra.dot(a::T1,b::T2) where {T1 <: Abstractfermion, T2 <: Abstractfermion}
    error("LinearAlgebra.dot is not implemented in type X:$T1,Y:$T2")
end

function get_origin(a::T1) where T1 <: AbstractFermionfields
    error("get_origin is not implemented in type $(typeof(a)) ")
end

function initialize_Adjoint_fermion(x::T1) where T1 <: AbstractFermionfields
    error("initialize_Adjoint_fermion is not implemented in type $(typeof(x)) ")
end

ext: jl


text:
function db_create_schema(dbdata)
    cols = colnames(dbdata)
    coltypes = map(eltype, columns(dbdata))
    pkeys = db_get_primary_keys(dbdata)
    schema = [(column=col,
               coltype=getproperty(coltypes, col),
               pkey=in(col, pkeys))
              for col in cols]
end

# Primary key getter
db_get_primary_keys(dbdata::IndexedTable) = colnames(dbdata)[dbdata.pkey]

db_get_primary_keys(dbdata::AbstractNDSparse) = colnames(dbdata.index)

# Sorted row iterator
db_sorted_row_iterator(dbdata; id_key=DEFAULT_DB_ID_KEY, rev=false) =
    sort(rows(dbdata); by=row->getproperty(row, id_key), rev=rev)

# Concatenate fields of dbentry (which is a named tuple) into a vector of strings
dbentry2text(dbentry, fields) = begin
    concatenated = [field2text(dbentry, field) for field in fields]
    filter!(!isempty, concatenated)
    return concatenated
end

dbentry2text(dbentry, ::Nothing) = dbentry2text(dbentry, Symbol[])

field2text(nt, prop) = begin
    if hasproperty(nt, prop)
        return make_a_string(getproperty(nt, prop))
    else
        return ""
    end
end

make_a_string(value) = string(value)

make_a_string(value::AbstractVector) = join(string.(value), " ")

# Checks that the id_key exists in dbdata and that its elements are Int's
function db_check_id_key(dbdata, id_key=nothing)
    if id_key != nothing
        if !in(id_key, colnames(dbdata)) &&
            throw(ErrorException("$id_key must be a column in the loaded data"))
        elseif !(eltype(getproperty(columns(dbdata), id_key)) <: Int)
            throw(ErrorException("$id_key elements must be of Int type"))
        end
    end
end

# Selects an entry in dbdata based on the value of id from a column
# selected by id_key
function db_select_entry(dbdata, id; id_key=DEFAULT_DB_ID_KEY)
    __first(dbdata::AbstractNDSparse) = first(rows(dbdata))
	__first(dbdata) = first(dbdata)
    cols = colnames(dbdata)
    if id_key in cols
        entry = filter(isequal(id), dbdata, select=id_key)
    else
        entry = filter(x -> false, dbdata, select=cols[1])  # empty entry
    end
    !isempty(entry) && (return __first(entry))
    return entry
end

# Selects all id_key's i.e. linear ids that correspond to certain values from a column
function db_select_idxs_from_values(dbdata, values, values_key; id_key=DEFAULT_DB_ID_KEY)
    collect(rows(filter(in(values), dbdata, select=values_key), id_key))
end

# Transforms a dbentry to a string using only fields; fields of length > max_length are trimmed
function dbentry2printable(dbentry, fields; max_length=50, separator=" - ")
    join(map(str->chop_to_length(str, max_length), dbentry2text(dbentry, fields)), separator)
end

dbentry2printable(::Nothing, fields; kwargs...) = ""

# Primitives to push/pop from IndexedTable/NDSparse
function db_check_entry_for_pushing(dbdata, entry, id_key, expected_value)
    if id_key != nothing
        !hasproperty(entry, id_key) &&
            throw(ErrorException("$id_key must be a column in the loaded data"))
        getproperty(entry, id_key) != expected_value &&
            throw(ErrorException("$id_key==$expected_value condition not fulfilled"))
        field_problem = false
        entry_fields = map(typeof, entry)
        dbdata_fields = map(eltype, columns(dbdata))
        for field in colnames(dbdata)
            if !hasproperty(entry_fields, field) || (getproperty(entry_fields, field) != getproperty(dbdata_fields, field))
                field_problem = true
            end
        end
        field_problem && throw(ErrorException("Missing entry field or wrong type."))
    end
end

db_push!(dbdata, entry; id_key=nothing) = begin
    db_check_id_key(dbdata, id_key)
    db_check_entry_for_pushing(dbdata, entry, id_key, length(dbdata) + 1)
    push!(rows(dbdata), entry)
    nothing
end

db_pushfirst!(dbdata, entry; id_key=nothing) = begin
    db_check_id_key(dbdata, id_key)
    db_check_entry_for_pushing(dbdata, entry, id_key, 1)
    cols = columns(dbdata)
    for col in colnames(dbdata)
        pushfirst!(getproperty(cols, col), getproperty(entry, col))
    end
    db_id_key_recreate!(dbdata, id_key)
    nothing
end

db_pop!(dbdata; id_key=nothing) = map(pop!, columns(dbdata))

db_popfirst!(dbdata; id_key=nothing) = begin
    db_check_id_key(dbdata, id_key)
    cols = columns(dbdata)
    popped = map(popfirst!, cols)
    db_id_key_recreate!(dbdata, id_key)
    return popped
end

db_deleteat!(dbdata, idxs; id_key=nothing) = begin
    db_check_id_key(dbdata, id_key)
    cols = columns(dbdata)
    map(x->deleteat!(x, idxs), cols)
    db_id_key_recreate!(dbdata, id_key)
end

db_id_key_recreate!(dbdata, id_key=nothing) = begin
    if id_key != nothing
        getproperty(columns(dbdata), id_key)[:].= 1:length(dbdata)
    end
    nothing
end

db_id_key_shift!(dbdata, id_key=nothing, by=0) = begin
    if id_key != nothing
        getproperty(columns(dbdata), id_key)[:].+= by
    end
    nothing
end

function db_drop_columns(dbdata::JuliaDB.IndexedTables.AbstractIndexedTable, to_drop)
    try
        return select(dbdata, Tuple(col for col in colnames(dbdata) if !in(col, to_drop)))
    catch
        @debug "Could not drop $(to_drop), returning original data."
        return dbdata
    end
end

function db_drop_columns(dbdata::AbstractNDSparse, to_drop)
    nchunks(dbdata::JuliaDB.DNDSparse) = length(dbdata.chunks)
    nchunks(dbdata::NDSparse) = nothing
    try
        ## simple oneliner: ndsparse(db_drop_columns(table(dbdata), to_drop))
        cols = columns(dbdata)
        indexcols = (col for col in colnames(dbdata.index) if !in(col, to_drop))
        datacols = (col for col in colnames(dbdata.data) if !in(col, to_drop))
        indexes = (getproperty(cols, col) for col in indexcols)
        data = (getproperty(cols, col) for col in datacols)
        return ndsparse(NamedTuple{Tuple(indexcols)}(indexes),
                        NamedTuple{Tuple(datacols)}(data);
                        chunks=nchunks(dbdata))
    catch
        @debug "Could not drop $(to_drop), returning original data."
        return dbdata
    end
end


text:
global x = parse(Int64, readline());
global i = 0;
while x > 1
    global x = trunc(Int, ifelse(iseven(x), x / 2, 3 * x + 1));
    global i += 1;
    println(join(Int64[i, x], " ==> "));
end

max_stars_repo_name: MediocreBoris/gitgud

max_issues_repo_name: MediocreBoris/gitgud

max_forks_repo_name: MediocreBoris/gitgud


text:
module SpikeTools
export raster, paddedbins, boxcarsmooth

function raster(spike_times::Vector{Float64}, start_times, r)
    firstr = first(r)
    lastr = last(r)
    n = length(r) - 1
    rasters = zeros(Int32, length(r) - 1, length(start_times))

    n = length(spike_times)
    j = 1
    for i = 1:length(start_times)
        start_time = start_times[i]
        a = start_time + firstr
        b = start_time + lastr
        j = searchsortedfirst(spike_times, a, Base.Order.Forward)
        if j <= n
            j += spike_times[j] == a
            while j <= n && spike_times[j] < b
                rasters[searchsortedlast(r, spike_times[j] - start_time, Base.Order.Forward), i] += 1
                j += 1
            end
        end
    end

    rasters
end

function paddedbins(lims::Tuple{Real, Real}, step::Real, expand_by::Real)
    expand_by = div(expand_by, step)
    extract_times = (lims[1]-expand_by*step:step:lims[2]+expand_by*step)/1000
    extract_range = expand_by+1:length(extract_times)-expand_by-1
    times = extract_times[extract_range]*1000+step/2
    (extract_times, extract_range, times)
end

function boxcarsmooth{T<:Real}(rast, fwhm, outtype::Type{T}=Float64)
    convrast = zeros(T, size(rast))
    for j = 1:size(rast, 2)
        halffwhm = fwhm÷2
        v = 0
        for i = 1:2halffwhm
            v += rast[i, j]
        end
        convrast[halffwhm+1, j] = v
        for i = halffwhm+2:size(rast, 1)-halffwhm
            v -= rast[i-halffwhm-1, j]
            v += rast[i+halffwhm-1, j]
            convrast[i, j] = v
        end
    end
    convrast
end
end

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_licenses: MIT Unlicense


text:
module TestStringAnalysis

using Test
using SparseArrays
using LinearAlgebra
using StringAnalysis
using Languages
using DataStructures

println("Running tests:")
include("autohash.jl")
include("tokenizer.jl")
include("ngramizer.jl")
include("document.jl")
include("metadata.jl")
include("corpus.jl")
include("preprocessing.jl")
include("dtm.jl")
include("coom.jl")
include("stemmer.jl")
include("stats.jl")
include("lda.jl")
include("lsa.jl")
include("rp.jl")
include("show.jl")

end


text:
immutable Levy <: ContinuousUnivariateDistribution
    location::Float64
    scale::Float64
    function Levy(l::Real, s::Real)
        s >= zero(s) || error("scale must be non-negative")
        new(float64(l), float64(s))
    end
end

Levy(location::Real) = Levy(location, 1.0)
Levy() = Levy(0.0, 1.0)

@distr_support Levy d.location Inf

function entropy(d::Levy)
    c = d.scale
    (1.0 - 3.0 * digamma(1.0) + log(16.0 * pi * c * c)) / 2.0
end

mean(d::Levy) = Inf
var(d::Levy) = Inf
skewness(d::Levy) = NaN
kurtosis(d::Levy) = NaN

mode(d::Levy) = d.scale / 3.0 + d.location

function median(d::Levy)
    m, c = d.location, d.scale
    m + c / (2.0 * erfcinv(0.5)^2)
end

function pdf(d::Levy, x::Float64)
    m, c = d.location, d.scale
    (sqrt(c)/sqrt2π) * exp(-(c / (2.0 * (x - m)))) / (x - m)^1.5
end

cdf(d::Levy, x::Float64) = erfc(sqrt(d.scale / (2.0 * (x - d.location))))

quantile(d::Levy, p::Float64) = d.location + d.scale / (2.erfcinv(p)^2)

mgf(d::Levy, t::Real) = t == zero(t) ? 1.0 : NaN

function cf(d::Levy, t::Real)
    m, c = d.location, d.scale
    exp(im * m * t - sqrt(-2.0 * im * c * t))
end

rand(d::Levy) = d.location + 1 / rand(Normal(0.0, 1.0 / sqrt(d.scale)))^2


text:
##
## Implementation of the algorithm from
## A GENERALIZATION OF THE MULTISHIFT QR ALGORITHM by RAF VANDEBRIL AND DAVID S. WATKINS.
## https://doi.org/10.1137/11085219X
## pdf file found at http://www.sci.wsu.edu/math/faculty/watkins/pdfiles/vw12_SIMAX.pdf
##
## Q is neither descending or ascending, but rather twisted
## the unitary transformation is not limited to 1 or 2 rotators, but rather m
##
## TODO:
##
## * Clean up allocations
##
## This algorithm encompasses those in CSS and RDS, but those are more efficient space wise
## This is more general and perhaps of interest as building blocks for experimentation

##################################################

##
## Twisted refers to the QFactorization
##

## unlike faster case, here instead of checking for parity in the diagonal rotators, we move -1 terms into  D
function deflate(QF::QFactorizationTwisted{T, S, Vt, Pvt}, k, ctr) where {T,S <: Real, Vt, Pvt}
    c,s = vals(QF.Q[k])
    i = idx(QF.Q[k])

    QF.Q[k] = Rotator(one(T), zero(T), i) # ± 1, not just 1
    if sign(c) < 0
        ## move into D term so no dflip concerns, as with RDS
        Asc = ascending_part(QF.Q, i)
        Des = descending_part(QF.Q, i)

        m, M = i, i+1
        for (i,U) in enumerate(Asc)
            c,s = vals(U)
            idx(U) < ctr.start_index && break
            m -= 1
            j = idx(U)
            Asc[i] = Rotator(-c, s, j)
        end
        for (i, U) in enumerate(Des)
            c,s = vals(U)
            idx(U) > ctr.stop_index && break
            M += 1
            j = idx(U)
            Des[i] = Rotator(-c, s, j)
        end

        QF.D.x[m] *= -one(T)
        QF.D.x[M] *= -one(T)
    end
end

function Base.Matrix(QF::QFactorizationTwisted{T, S}) where {T, S}

    n = length(QF) + 1
    M = diagm(0 => ones(S, n))
    D = Matrix(QF.D) * M
    lmul!(Vector(QF.Q), D)
    D

end

##################################################

## We use `bulge_step!` for more general usage;
function bulge_step(QF::QFactorizationTwisted{T,S}, RF, storage, ctr, m)  where {T,S}

    create_bulge(QF, RF, storage, ctr, m)

    δ, Δ = ctr.start_index, ctr.stop_index
    n = ctr.stop_index - ctr.zero_index + 1

    Asc = copy(storage.VU[1:m])  # stored as V,U ## XXX copy! Could eliminate this
    Ms = QF.Q
    D = QF.D
    RF = RF

    bulge_step!(n, m, view(Ms, δ:Δ), D, RF, Asc)

    return nothing
end

## Can enter here for more general usage
## n is the size of matrix, not the number of rotators
function bulge_step!(n, m, Ms::TwistedChain, D, RF::AbstractRFactorization, Asc, choice = :left)

    ## XXX
    ## We *can* save memory if we
    ## * didn't copy VU to Asc
    ## * put Asc, Des in storage and didn't adjust sizes in step_knit
    ## * used the limb defined in create storage
    ## This would require some reworking of step_knit!

    Des = reverse(adjoint.(Asc))

    # psd[k] = ps[m+k] with padding soecufued by choice
    ps = copy(Ms.pv)
    psd = ps[m+1:end]
    if isa(choice, Symbol)
        append!(psd, repeat([choice], min(m, length(Ms.pv))))
    else
        append!(psd, choice)
    end

    ## Must put AscendingChain into place
    ## py passing through RF and D <--
    passthrough!(RF, AscendingChain(Asc))
    passthrough!(D, AscendingChain(Asc))

    limb, limb_side = step_0!(m, ps, psd, Ms, Des, Asc, D, RF)

    for k in 1:(n-m-2)
        limb_side = step_k!(k, n, m, psd, limb_side, limb, Des, Asc, Ms, D, RF)
    end

    # use new choice
    if n-m-1 > 0
        limb_side = step_k!(n-m-1, n, m, psd, limb_side, limb, Des, Asc, Ms, D, RF)
    end

    # now knit in  limb, Des, Asc
    step_knit!(n, m, psd, limb_side, limb, Des,  Asc, Ms, D, RF)

    Ms.pv[:] = psd

    return nothing
end

############################################################
##
## Implement thee different steps for one pass of the bulge chasing algorithm

function step_0!(m,  ps, psd, Ms::TwistedChain, Des, Asc, D, RF) where {T}

    has_limb = ifelse(m > 1, true, false)

    # grab limb, but keep order
    # limb is a copy, not a view
    limb = TwistedChain(copy(Ms.x[1:(m-1)]), copy(Ms.pv[1:(m-2)]))
    limb_side = :nothing

    if has_limb
        if ps[m-1] == :left
            passthrough!(DescendingChain(Des), limb)  # pass limb through left
            limb_side = :left
        else
            passthrough!(limb, AscendingChain(Asc))   # pass limb to right
            limb_side = :right
        end
    end

    U = Ms.x[m]
    psm = m <= length(ps) ? ps[m] : :right # doesn't matter in this case

    if psm == :left
        Des[end], Di = fuse(Des[end], U)  # fuse with descending; aka Ms[m]

        ## need to passthrough Ms too now
        i = idx(Di)
        dMs = descending_part(Ms, i) # stop_index

        if limb_side == :right
            passthrough_phase!(Di, dMs,  (AscendingChain(Asc), limb), D)
        else
            passthrough_phase!(Di, dMs,  (AscendingChain(Asc), ), D)
        end

    else

        AA = popfirst!(Asc)
        AA, Di = fuse(U, AA)

        if limb_side == :right
            passthrough_phase!(Di, (AscendingChain(Asc), limb), D)
        else
            passthrough_phase!(Di, (AscendingChain(Asc), ), D)
        end
        pushfirst!(Asc, AA)

    end

    limb, limb_side

end

function step_k!(k, n, m, psd, limb_side, limb, Des, Asc, Ms, D, RF) where {T}

    phatk = psd[k]

    ## 4 cases based on:
    ## phatk/limb_side
    ## A bit redundandant, but easier to verify

    U = Ms.x[m+k]

    if phatk == :left && limb_side == :left

        push!(Des, U)  ## Augment Descending

        passthrough!(DescendingChain(Des), AscendingChain(Asc)) ## translate ascending

        passthrough!(limb, AscendingChain(Asc))

        ## similarity transform Asc to right side
        passthrough!(RF, AscendingChain(Asc)) # <-- pass Asc through RF
        passthrough!(D, AscendingChain(Asc))

        U = popfirst!(Des)

        if m > 1
            limb_side = :left
        end

    elseif phatk == :left && (limb_side == :right || limb_side == :nothing)

        push!(Des, U)  ## Augment Descending
        passthrough!(DescendingChain(Des), AscendingChain(Asc)) ## translate ascending
        passthrough!(DescendingChain(Des), limb)

        ## similarity transform Asc to right side
        passthrough!(RF, AscendingChain(Asc)) # <-- pass Asc through RF
        passthrough!(D, AscendingChain(Asc))

        U = popfirst!(Des)

        if m > 1
            limb_side = :left
        end

    elseif phatk == :right && limb_side == :left

        pushfirst!(Asc, U)

        passthrough!(DescendingChain(Des), AscendingChain(Asc))

        passthrough!(limb, AscendingChain(Asc))

        ## similarity transform descending to left side
        passthrough!(DescendingChain(Des), D)
        passthrough!(DescendingChain(Des), RF) #--> pass Des through Rf;  similarity transform

        U = pop!(Asc)

        if m > 1
            limb_side = :right
        end

    elseif phatk == :right && (limb_side == :right || limb_side == :nothing)

        pushfirst!(Asc, U)
        passthrough!(DescendingChain(Des), AscendingChain(Asc))

        passthrough!(DescendingChain(Des), limb)

        ## similarity transform descending to left side
        passthrough!(DescendingChain(Des), D)
        inds = idx.(Des)
        passthrough!(DescendingChain(Des), RF) #--> pass Des through Rf;  similarity transform
        @assert all(idx.(Des) .== inds)
        U = pop!(Asc) # k = idx(U)

        if m > 1
            limb_side = :right
        end

    end

    Ms.x[k] = U

    return limb_side

end

## steps k=n-m to  n-2
function step_knit!(n, m, psd, limb_side, limb, Des,  Asc, Ms, D, RF) where {T}

    # make bottom
    U = pop!(Des)
    V = popfirst!(Asc)
    bottom, Di = fuse(U,V)

    ## Decoupled too! Decoupled on right? as we augmented Ascending
    ## have a complicted check, as m can be big
    psdm = length(psd)-m+1 > 0 ? psd[end-m+1] : :left

    if psdm == :right
        Decoupled = view(Ms, 1:n-m)
        if limb_side == :right
            passthrough_phase!(Di, (AscendingChain(Asc), Decoupled, limb), D)
        else
            passthrough_phase!(Di, (AscendingChain(Asc),  Decoupled), D)
        end
    else
        if limb_side == :right
            passthrough_phase!(Di, (AscendingChain(Asc), limb), D)
        else
            passthrough_phase!(Di, (AscendingChain(Asc),), D)
        end
    end

    ## Final Steps, knit in
    ## setps k=n-m to  n-2

    for k in (n-m):(n-2)
        phatk = k > 1 ? psd[k-1] : :left
        if k > (n-m) && k > 2 && psd[k-2] != phatk
            ## need to reposition V structure to other side
            if phatk == :left
                passthrough!(RF, DescendingChain(Des))
                passthrough!(D, DescendingChain(Des))
                bottom = passthrough!(RF, bottom)
                bottom = passthrough!(D, bottom)
                passthrough!(RF, AscendingChain(Asc))
                passthrough!(D, AscendingChain(Asc))
            else
                passthrough!(AscendingChain(Asc),D)
                passthrough!(AscendingChain(Asc),RF)
                bottom = passthrough!(bottom, D)
                ind = idx(bottom)
                bottom = passthrough!(bottom, RF)
                @assert idx(bottom) == ind
                passthrough!(DescendingChain(Des),D)
                passthrough!(DescendingChain(Des),RF)
            end
            # flip limb side
            if limb_side == :right
                limb_side = :left
            elseif limb_side == :left
                limb_side = :right
            end
        end

        ## lengthen one side of V
        if phatk == :left
            Des = push!(Des, bottom)
        else
            Asc = pushfirst!(Asc, bottom)
        end
        passthrough!(DescendingChain(Des), AscendingChain(Asc))

        # remove bottom rotator from limb; keep track of if it is right or left
        lpk = length(limb.pv) > 0 ? limb.pv[end] : :nothing
        L, lpk =  pop!(limb)
        ## we have
        ## 4 cases in terms of limb_side and lpk:
        ## :l, :l -> fuse (Asc, Des), translate
        ## :l, :r -> translate, fuse (Asc, limb, Des)
        ## :r, :l -> translate, fuse (limb)
        ## :r, :r -> fuse (.), translate
        if limb_side == :left || limb_side == :nothing
            if lpk == :left || lpk == :nothing

                # fuse then translate
                Asc[1], Di = fuse(L, Asc[1])
                if length(Asc) > 1
                    AA = Asc[2:end]
                    passthrough_phase!(Di,  (AscendingChain(AA), DescendingChain(Des)), D)
                    Asc[2:end] = AA
                else
                    passthrough_phase!(Di, (DescendingChain(Des), ), D)
                end

                ## translate
                length(limb) > 0 && passthrough!(limb, AscendingChain(Asc))

            elseif lpk == :right

                # translate then fuse
                length(limb) > 0 && passthrough!(limb, AscendingChain(Asc))

                # then fuse
                Asc[1], Di = fuse(L, Asc[1])
                if length(Asc) > 1
                    AA = Asc[2:end]
                    passthrough_phase!(Di, (AscendingChain(AA), limb, DescendingChain(Des)), D)
                    Asc[2:end] = AA
                else
                    passthrough_phase!(Di, (limb, DescendingChain(Des)), D)
                end

            end

        else # limb on right

            if  lpk == :left

                # translate then fuse
                length(limb) > 0 && passthrough!(DescendingChain(Des), limb)

                Des[end], Di = fuse(Des[end], L)
                passthrough_phase!(Di, (), D)
            else
                # fuse  then translate
                Des[end], Di = fuse(Des[end], L)
                passthrough_phase!(Di, limb, (), D)

                length(limb) > 0 && passthrough!(DescendingChain(Des), limb)
            end
        end

        if phatk == :left
            # similarity to move Asc to right side
            passthrough!(RF, AscendingChain(Asc))  # <--
            passthrough!(D, AscendingChain(Asc))

            limb_side = :left
        else
            # similarity to move Des to left side
            passthrough!(DescendingChain(Des), D)
            inds = idx.(Des)
            passthrough!(DescendingChain(Des), RF)
            @assert all(idx.(Des) .== inds)

            limb_side = :right

        end

        ## pop off top, and add to Decopuled
        ## we have idx(U) ==   k; so psd[k-1] = phatk indicates direction to  add
        if phatk == :left
            U  = popfirst!(Des)
            Ms.x[k] = U
        else
            U = pop!(Asc)
            Ms.x[k] = U
        end

        U = pop!(Des)
        V = popfirst!(Asc)
        bottom, Di = fuse(U,V)

        if phatk == :right
            Decoupled = view(Ms, 1:k)
            if limb_side == :right
                passthrough_phase!(Di, (AscendingChain(Asc), Decoupled, limb), D)
            else
                passthrough_phase!(Di, (AscendingChain(Asc), Decoupled), D)
            end
        else
            if limb_side == :right
                passthrough_phase!(Di, (AscendingChain(Asc), limb), D)
            else
                passthrough_phase!(Di, (AscendingChain(Asc),), D)
            end
        end

    end

    # just the bottom left, but may be on wrong side if m > 1
    if psd[end] == :left
        if m > 1 && (length(psd) > 1 && psd[end-1] != psd[end])
            bottom = passthrough!(RF, bottom)  # move to other side
            bottom = passthrough!(D, bottom)
        end

        Ms.x[end] = bottom

    else
        if m > 1 && (length(psd) > 1 && psd[end-1] != psd[end])
            bottom = passthrough!(bottom, D)   # move bottom to other side
            bottom = passthrough!(bottom, RF)
        end
        @assert length(Ms.x) == idx(bottom)
        Mx.x[end] = bottom
    end

    return nothing

end


text:
using Makie
using GeometryBasics
using PyCall
using FractalTools
using StaticArrays
using Random

# Construct interpolation data
hexagon = GeometryBasics.Ngon(@SVector [Point(BigFloat(cos(θ)), BigFloat(sin(θ))) for θ in 0 : π / 3 : 2π - π / 3])
f(x, y) = x^2 + y^2
pnts3d = getdata(f, hexagon, 100)

# Construct interpolant
itp = interpolate(pnts3d)

function erf(xd, yd)
    fval = f(xd, yd)
    ival = itp(xd, yd)
    abs(fval - ival) / abs(fval) * 100
end

# Calculate error values
pnts2d = project(pnts3d)
ivals = map(pnt -> Point(pnt[1], pnt[2], itp(pnt...)), pnts2d)
fvals = map(pnt -> Point(pnt[1], pnt[2], f(pnt...)), pnts2d)
evals = map(pnt -> Point(pnt[1], pnt[2], erf(pnt...)), pnts2d)

# Construct GUI
fig         = Figure()
lb          = Label(fig, "Show Interpolant")
tog         = Toggle(fig, active=false)
ax1         = LScene(fig, scenekw=(camema=cam3d!, raw=false), tellwidth=false)
fig[1, 1]   = vgrid!(ax1, hgrid!(lb, tog, tellwidth=false), tellwidth=false)
ax2         = fig[1, 2] = LScene(fig, scenekw=(camema=cam3d!, raw=false), tellwidth=false)

        trisurf!(ax1, fvals, meshcolor3=last.(fvals), colormap=:viridis)
plt =   trisurf!(ax1, ivals, meshcolor3=last.(ivals), colormap=:heat)
        wireframe!(ax1, project(triangulate(pnts3d)[2]))
        trisurf!(ax2, evals, meshcolor3=last.(evals), colormap=:viridis)

on(tog.active) do val
    plt.visible[] = val
end

tog.active[] = false

display(fig)

# # Record video
# fps = 24
# record(fig.scene, "errorplot.mp4"; framerate = fps) do io
#     for i = 1 : 10 * fps
#         sleep(1/fps)
#         recordframe!(io)
#     end
# end


text:
# Channel get function lookups
const getchanfun = IdDict(
    AI => DAQmx.GetDevAIPhysicalChans,
    AO => DAQmx.GetDevAOPhysicalChans,
    DI => DAQmx.GetDevDILines,
    DO => DAQmx.GetDevDOLines,
    CI => DAQmx.GetDevCIPhysicalChans,
    CO => DAQmx.GetDevCOPhysicalChans)

function DAQDevice(name::String)
    x = DAQDevice(name, nothing)
    chanobjs = lschan(x; asobjects=true)
    zipped = zip(keys(getchanfun),chanobjs)
    x.channels = LittleDict([kv for kv in zipped])
    return x
end

function DAQTask{T}(name::String) where T <: AbstractIO
    handleptr = Ref{TaskHandle}()
    DAQmx.CreateTask(name, handleptr) |> catch_error
    conditions = Base.AsyncCondition[]
    task = DAQTask{T}(name, handleptr[], nothing, nothing, conditions)
    finalizer(task) do task
        isrunning(task) && stop(task)
        clear(task)
    end
    return task
end

DAQTask{T}() where T <: AbstractIO = DAQTask{T}("")


text:
abstract type ALBERT_V1 <: PretrainedTokenizer end
abstract type ALBERT_V2 <: PretrainedTokenizer end

const vectors_albertversion1 = [
    ("albert_base_v1_30k-clean.vocab",
    "albert base version1 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_base_v1_30k-clean.vocab"),
    ("albert_large_v1_30k-clean.vocab",
    " albert large version1 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_large_v1_30k-clean.vocab"),
    ("albert_xlarge_v1_30k-clean.vocab",
    "albert xlarge version1 of size ~800kb download",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_xlarge_v1_30k-clean.vocab"),
    ("albert_xxlarge_v1_30k-clean.vocab",
    "albert xxlarge version1 of size ~800kb download",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_xxlarge_v1_30k-clean.vocab")
]

const vectors_albertversion2 = [
    ("albert_base_v2_30k-clean.vocab",
    "albert base version2 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_base_v2_30k-clean.vocab"),
    ("albert_large_v2_30k-clean.vocab",
    " albert large version2 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_large_v2_30k-clean.vocab"),
    ("albert_xlarge_v2_30k-clean.vocab",
    "albert xlarge version2 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_xlarge_v2_30k-clean.vocab"),
    ("albert_xxlarge_v2_30k-clean.vocab",
    "albert xxlarge version2 of size ~800kb download.",
    "1de4ad94a1b98f5f5f2c75af0f52bc85714d67b8578aa8f7650521bb123335c0",
    "https://raw.githubusercontent.com/tejasvaidhyadev/ALBERT.jl/master/src/Vocabs/albert_xxlarge_v2_30k-clean.vocab")
]

function init_vocab_datadeps()
    for (depname, description, sha, link) in vectors_albertversion1
        register(DataDep(depname,
            """
            sentencepiece albert vocabulary file by google research .
            Website: https://github.com/google-research/albert
            Author: Google Research
            Licence: Apache License 2.0
            $description
            """,
            link,
            sha
                ))
        append!(tokenizer_files(ALBERT_V1), ["$depname"])
    end

    for (depname, description, sha, link) in vectors_albertversion2
        register(DataDep(depname,
            """
            sentencepiece albert vocabulary file by google research .
            Website: https://github.com/google-research/albert
            Author: Google Research
            Licence: Apache License 2.0
            $description
            """,
            link,
            sha
                 ))
        append!(tokenizer_files(ALBERT_V2), ["$depname"])
    end
end


text:
# MIT License
#
# Copyright (c) 2020 Joshua E Gentry

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
using Test
using JSON

# -------------------------------------------------------------------------------------------------
# Test download_languages()
@testset "download_languages" begin
    local dir = mktempdir(;cleanup = false)
        @test download_languages("eng"; target=dir) == true
        @test isfile(joinpath(dir, "eng.traineddata")) == true
        @test filesize(joinpath(dir, "eng.traineddata")) > 0
        @test isfile(joinpath(dir, ".languages")) == true
        @test filesize(joinpath(dir, ".languages")) > 0
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test download_languages("fra+heb+hin"; target=dir) == true
        @test isfile(joinpath(dir, "fra.traineddata")) == true
        @test filesize(joinpath(dir, "fra.traineddata")) > 0
        @test filesize(joinpath(dir, "heb.traineddata")) > 0
        @test isfile(joinpath(dir, "heb.traineddata")) == true
        @test filesize(joinpath(dir, "hin.traineddata")) > 0
        @test isfile(joinpath(dir, "hin.traineddata")) == true
        @test isfile(joinpath(dir, ".languages")) == true
        @test filesize(joinpath(dir, ".languages")) > 0
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test download_languages("kat"; target=dir) == true
        local timestamp = mtime(joinpath(dir, "kat.traineddata"))
        @test download_languages("kat"; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "kat.traineddata"))
        @test download_languages("kat"; target=dir, force=true) == true
        @test timestamp != mtime(joinpath(dir, "kat.traineddata"))
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test download_languages("eng"; target=dir, baseUrl="https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/") == true
        @test isfile(joinpath(dir, "eng.traineddata")) == true
        @test filesize(joinpath(dir, "eng.traineddata")) > 0
        @test isfile(joinpath(dir, ".languages")) == true
        @test filesize(joinpath(dir, ".languages")) > 0
    rm(dir;recursive = true)
end

# -------------------------------------------------------------------------------------------------
# Test update_languages()
@testset "update_languages" begin
    local dir = mktempdir(;cleanup = false)
        @test update_languages("eng"; target=dir) == true
        @test isfile(joinpath(dir, "eng.traineddata")) == true
        @test filesize(joinpath(dir, "eng.traineddata")) > 0
        @test isfile(joinpath(dir, ".languages")) == true
        @test filesize(joinpath(dir, ".languages")) > 0
        local timestamp = mtime(joinpath(dir, "eng.traineddata"))
        local lastcheck = JSON.parse(read(joinpath(dir, ".languages"), String))["lastcheck"]
        @test update_languages("eng"; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "eng.traineddata"))
        @test lastcheck == JSON.parse(read(joinpath(dir, ".languages"), String))["lastcheck"]
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test update_languages("fra+heb+hin"; target=dir) == true
        @test isfile(joinpath(dir, "fra.traineddata")) == true
        @test filesize(joinpath(dir, "fra.traineddata")) > 0
        @test filesize(joinpath(dir, "heb.traineddata")) > 0
        @test isfile(joinpath(dir, "heb.traineddata")) == true
        @test filesize(joinpath(dir, "hin.traineddata")) > 0
        @test isfile(joinpath(dir, "hin.traineddata")) == true
        @test isfile(joinpath(dir, ".languages")) == true
        @test filesize(joinpath(dir, ".languages")) > 0
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test update_languages("kat"; target=dir) == true
        local timestamp = mtime(joinpath(dir, "kat.traineddata"))
        @test update_languages("kat"; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "kat.traineddata"))
    rm(dir;recursive = true)
end

# -------------------------------------------------------------------------------------------------
# Test download_pdf_font()
@testset "download_pdf_font" begin
    local dir = mktempdir(;cleanup = false)
        @test download_pdf_font(; target=dir) == true
        @test isfile(joinpath(dir, "pdf.ttf")) == true
        @test filesize(joinpath(dir, "pdf.ttf")) > 0
        @test isfile(joinpath(dir, ".pdffont")) == true
        @test filesize(joinpath(dir, ".pdffont")) > 0
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test download_pdf_font(; target=dir) == true
        local timestamp = mtime(joinpath(dir, "pdf.ttf"))
        @test download_pdf_font(; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "pdf.ttf"))
        @test download_pdf_font(; target=dir, force=true) == true
        @test timestamp != mtime(joinpath(dir, "pdf.ttf"))
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test download_pdf_font(; target=dir, baseUrl="https://raw.githubusercontent.com/tesseract-ocr/tessconfigs/master/") == true
        @test isfile(joinpath(dir, "pdf.ttf")) == true
        @test filesize(joinpath(dir, "pdf.ttf")) > 0
        @test isfile(joinpath(dir, ".pdffont")) == true
        @test filesize(joinpath(dir, ".pdffont")) > 0
    rm(dir;recursive = true)
end

# -------------------------------------------------------------------------------------------------
# Test update_pdf_font()
@testset "update_pdf_font" begin
    local dir = mktempdir(;cleanup = false)
        @test update_pdf_font(; target=dir) == true
        @test isfile(joinpath(dir, "pdf.ttf")) == true
        @test filesize(joinpath(dir, "pdf.ttf")) > 0
        @test isfile(joinpath(dir, ".pdffont")) == true
        @test filesize(joinpath(dir, ".pdffont")) > 0
        local timestamp = mtime(joinpath(dir, "pdf.ttf"))
        local lastcheck = JSON.parse(read(joinpath(dir, ".pdffont"), String))["lastcheck"]
        @test update_pdf_font(; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "pdf.ttf"))
        @test lastcheck == JSON.parse(read(joinpath(dir, ".pdffont"), String))["lastcheck"]
    rm(dir;recursive = true)

    local dir = mktempdir(;cleanup = false)
        @test update_pdf_font(; target=dir) == true
        local timestamp = mtime(joinpath(dir, "pdf.ttf"))
        @test update_pdf_font(; target=dir) == true
        @test timestamp == mtime(joinpath(dir, "pdf.ttf"))
    rm(dir;recursive = true)
end


text:
using Test
using Alert

# without a backend, `alert` should error
if get(ENV,"CI","false") == "true"
    @test_throws ProcessFailedException alert()
else
    @test alert() !== nothing
end


text:
"""
    simple_parse_tsp(filename; verbose = true)

Try to parse the ".tsp" file given by `filename`. Very simple implementation
just to be able to test the optimization; may break on other files. Returns a
graph of cities and edges for use in `get_optimal_tour`.
"""
function simple_parse_tsp(filename; verbose = false)
    cities = Vector{Tuple{Float64, Float64}}()
    section = :Meta
    for line in readlines(filename)
        line = strip(line)
        line == "EOF" && break
        if section == :Meta && verbose
            println(line)
        end
        if section == :NODE_COORD_SECTION
            nums = split(line)
            @assert length(nums) == 3
            x = parse(Float64, nums[2])
            y = parse(Float64, nums[3])
            push!(cities, (x, y))
        end
        # change section type
        if line == "NODE_COORD_SECTION"
            section = :NODE_COORD_SECTION
        end
    end

    N = length(cities)
    g = MetaGraph(N)
    for i in 1:N
        for j in i+1:N
            add_edge!(g, i, j, :weight, euclidean_distance(cities[i], cities[j]))
        end
    end

    return g
end

"""
    get_edges_cost(g, edges)

Return the sum of the cost of the edges
"""
function get_edges_cost(g::AbstractGraph, edges)
    edges_cost = 0.0
    for edge in edges
        edges_cost += get_prop(g, edge.src, edge.dst, :weight)
    end
    return edges_cost
end

"""
    get_edges_cost(cost, edges)

Return the sum of the cost of the edges
"""
function get_edges_cost(cost, edges)
    edges_cost = 0.0
    for edge in edges
        edges_cost += cost[edge.src, edge.dst]
    end
    return edges_cost
end

function get_tour_cost(g, tour)
    cost = 0.0
    for i in 1:length(tour)-1
        src = tour[i]
        dst = tour[i+1]
        cost += get_prop(g, src, dst, :weight)
    end
    cost += get_prop(g, tour[end], tour[1], :weight)
    return cost
end

"""
    euclidean_distance(point1, point2)

The usual Euclidean distance measure.
Copied from https://github.com/ericphanson/TravelingSalesmanExact.jl
"""
euclidean_distance(point1, point2) = sqrt((point1[1] - point2[1])^2 + (point1[2] - point2[2])^2)

function fix_edge!(root, edge)
    cost = get_prop(root.g, edge..., :weight)
    set_prop!(root.g, edge..., :weight, 0.0)
    root.cost[edge[1], edge[2]] = 0.0
    root.cost[edge[2], edge[1]] = 0.0
    return cost
end

function disallow_edge!(root, edge)
    rem_edge!(root.g, edge...)
    root.cost[edge[1], edge[2]] = Inf
    root.cost[edge[2], edge[1]] = Inf
end


text:
using Base: promote_eltype
using Base.Broadcast: check_broadcast_shape, broadcast_shape
using Base.Cartesian

if VERSION >= v"0.5.0-dev+5189"
    _to_shape(dims::Base.DimsOrInds) = map(_to_shape, dims)
    _to_shape(r::Base.OneTo) = Int(last(r))
else
    _to_shape(x) = x
end

if VERSION < v"0.5.0-dev+5434"
    function gen_nullcheck(narrays::Int, nd::Int)
        e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 ))
        for k = 2:narrays
            isnull = Symbol("isnull_$k")
            j_d_k = Symbol("j_d_$k")
            e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) ))
            e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k)
        end
        return e_nullcheck
    end

    function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool)
        F = Expr(:quote, f)
        e_nullcheck = gen_nullcheck(narrays, nd)
        if lift
            return quote
                # set up aliases to facilitate subsequent Base.Cartesian magic
                B_isnull = B.isnull
                @nexprs $narrays k->(values_k = A_k.values)
                @nexprs $narrays k->(isnull_k = A_k.isnull)
                # check size
                @assert ndims(B) == $nd
                @ncall $narrays check_broadcast_shape size(B) k->A_k
                # main loops
                @nloops($nd, i, B,
                    d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre
                    begin # body
                        if $e_nullcheck
                            @inbounds (@nref $nd B_isnull i) = true
                        else
                            @nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k)
                            @inbounds (@nref $nd B i) = (@ncall $narrays $F v)
                        end
                    end
                )
            end
        else
            return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f)
        end
    end

    function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool)
        As = [Symbol("A_"*string(i)) for i = 1:narrays]
        body = gen_broadcast_body(nd, narrays, f, lift)
        @eval let
            local _F_
            function _F_(B, $(As...))
                $body
            end
            _F_
        end
    end

    function Base.broadcast!(f, X::NullableArray; lift::Bool=false)
        broadcast!(f, X, X; lift=lift)
    end

    @eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}()
        @doc """
        `broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`
        This method implements the same behavior as that of `broadcast!` when called on
        regular `Array` arguments. It also includes the `lift` keyword argument, which
        when set to true will lift `f` over the entries of the `As`.

        Lifting is disabled by default. Note that this method's signature specifies
        the destination `B` array as well as the source `As` arrays as all
        `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting
        of both `Array`s and `NullableArray`s will fall back to the implementation
        of `broadcast!` in `base/broadcast.jl`.
        """ ->
        function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
            nd = ndims(B)
            narrays = length(As)

            cache_f    = Base.@get! cache      f       Dict{Bool, Dict{Int, Dict{Int, Any}}}()
            cache_lift = Base.@get! cache_f    lift    Dict{Int, Dict{Int, Any}}()
            cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}()
            func       = Base.@get! cache_f_na nd      gen_broadcast_function(nd, narrays, f, lift)

            func(B, As...)
            return B
        end
    end  # let cache
else
    using Base.Broadcast: newindexer, map_newindexer, newindex

    function _nullcheck(nargs)
        nullcheck = :(isnull_1[I_1])
        for i in 2:nargs
            sym_isnull = Symbol("isnull_$i")
            sym_idx = Symbol("I_$i")
            nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck)
        end
        # if 0 argument arrays, treat nullcheck as though it returns false
        nargs >= 1 ? nullcheck : :(false)
    end

    @generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f,
    Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false)
        nullcheck = _nullcheck(nargs)
        quote
            T = eltype(Z)
            $(Expr(:meta, :noinline))
            # destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast)
            @nexprs $nargs i->(keep_i = keeps[i])
            @nexprs $nargs i->(Idefault_i = Idefaults[i])
            if !lift
                # destructure the keeps and As tuples
                @nexprs $nargs i->(X_i = Xs[i])
                @simd for I in CartesianRange(indices(Z))
                    # reverse-broadcast the indices
                    @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
                    # extract array values
                    @nexprs $nargs i->(@inbounds val_i = X_i[I_i])
                    # call the function and store the result
                    @inbounds Z[I] = @ncall $nargs f val
                end
            else
                # destructure the indexmaps and Xs tuples
                @nexprs $nargs i->(values_i = Xs[i].values)
                @nexprs $nargs i->(isnull_i = Xs[i].isnull)
                @simd for I in CartesianRange(indices(Z))
                    # reverse-broadcast the indices
                    @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
                    if $nullcheck
                        # if any args are null, store null
                        @inbounds Z.isnull[I] = true
                    else
                        # extract array values
                        @nexprs $nargs i->(@inbounds val_i = values_i[I_i])
                        # call the function and store the result
                        @inbounds Z[I] = @ncall $nargs f val
                    end
                end
            end
        end
    end

    @doc """
    `broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`

    This method implements the same behavior as that of `broadcast!` when called
    on regular `Array` arguments. It also includes the `lift` keyword argument,
    which when set to true will lift `f` over the entries of the `As`.

    Lifting is disabled by default. Note that this method's signature specifies
    the destination `B` array as well as the source `As` arrays as all
    `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of
    both `Array`s and `NullableArray`s will fall back to the implementation of
    `broadcast!` in `base/broadcast.jl`.
    """ ->
    # Required to solve dispatch ambiguity between
    #   broadcast!(f, X::AbstractArray, x::Number...)
    #   broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...)
    @inline Base.broadcast!(f, Z::NullableArray; lift=false) =
        broadcast!(f, Z, Z; lift=lift)

    @inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...;
                                     lift=false)
        nargs = length(Xs)
        shape = indices(Z)
        check_broadcast_shape(shape, Xs...)
        keeps, Idefaults = map_newindexer(shape, Xs)
        Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift)
        return Z
    end
end

@doc """
`broadcast(f, As::NullableArray...;lift::Bool=false)`

This method implements the same behavior as that of `broadcast` when called on
regular `Array` arguments. It also includes the `lift` keyword argument, which
when set to true will lift `f` over the entries of the `As`.

Lifting is disabled by default. Note that this method's signature specifies the
source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on
arguments consisting of both `Array`s and `NullableArray`s will fall back to the
implementation of `broadcast` in `base/broadcast.jl`.
""" ->
@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false)
    return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)),
                                       _to_shape(broadcast_shape(Xs...))),
                      Xs...; lift=lift)
end

# broadcasted ops
for (op, scalar_op) in (
    (:(@compat Base.:(.==)), :(==)),
    (:(@compat Base.:.!=), :!=),
    (:(@compat Base.:.<), :<),
    (:(@compat Base.:.>), :>),
    (:(@compat Base.:.<=), :<=),
    (:(@compat Base.:.>=), :>=)
)
    @eval begin
        ($op)(X::NullableArray, Y::NullableArray) = broadcast($scalar_op, X, Y)
    end
end


text:
# a point in the circuit with a given voltage and current
@connector function Port(; name)
    sts = @variables p(t)=1.0 q(t)=1.0
    ODESystem(Equation[], t, sts, []; name)
end

# Fluid ground port, where pressure is 0
function FluidGround(; name)
    @named g = Port()
    eqs = [g.p ~ 0.]
    compose(ODESystem(eqs, t, [], []; name), g)
end

# abstraction for all 2-pin components
function OnePort(; name)
    # two pins
    @named i = Port() # Inlet
    @named o = Port() # Outlet
    # component has a pressure across it, and flow through
    sts = @variables p(t)=1.0 q(t)=1.0
    eqs = [
        p ~ i.p - o.p    # KVL TODO: rename
        0. ~ i.q + o.q   # Conservation of mass
        q ~ i.q          # Flow through component is Flow through +ve port
    ]
    compose(ODESystem(eqs, t, sts, []; name), i, o)
end

function ModelingToolkit.connect(::Type{Port}, ps...)
    eqs = [
        0. ~ sum(port->port.q, ps)    # KCL
    ]
    # KVL
    for i in 1:length(ps)-1
        push!(eqs, ps[i].p ~ ps[i+1].p)
    end
    return eqs
end

connect_star(comps...) = connect([comp.n for comp in comps]...)

function connect_series(comps...)
    eqs = Equation[]
    for i in 1:length(comps)-1
        eqs = vcat(eqs, connect(comps[i].i, comps[i+1].o))
    end
    return eqs
end

function connect_parallel(comps...)
    return [
        connect((comp.i for comp in comps)...)
        connect((comp.o for comp in comps)...)
    ]
end


text:
#   This file is part of Reduce.jl. It is licensed under the MIT license
#   Copyright (C) 2017 Michael Reed

oldwdir = pwd()
wdir = @__DIR__
include("svn.jl")
if isfile("ver")
    global ver = NaN
    open("ver","r") do f
        global ver = parse(Int,read(f,String))
    end
    if ver ≠ ρ
        if Sys.islinux()
            run(`rm -rf $(joinpath(wdir,"usr"))`)
        elseif Sys.isapple()
            run(`rm -rf $(joinpath(wdir,"csl"))`)
        elseif Sys.iswindows()
            run(`cmd /C DEL /F /S /Q /A "$(joinpath(wdir,"reduce.exe"))"`)
        end
        run(Sys.windows() ? `cmd /C DEL /F /S /Q /A "ver"` : `rm ver`)
    end
end

function writever(rv)
    open("ver","w") do f
        write(f,"$rv")
    end
end

if !Sys.iswindows()
    try
        try
            process = _spawn(`$rpsl`)
            kill(process)
        catch
            if Sys.islinux()
                cmd = `$(joinpath(wdir,"usr","bin"))/$rpsl`
            elseif Sys.isapple()
                cmd = `$(joinpath(wdir,"psl"))/$rpsl`
            else
                cmd = `$(joinpath(wdir,"Reduce-svn$(rsvn[ρ])-src","bin"))/$rpsl`
            end
            process = _spawn(cmd)
            kill(process)
            !isfile("ver") && writever(0)
        end
    catch
        http = "https://sourceforge.net/projects/reduce-algebra/files/snapshot_"
        rtg = "reduce.tar.gz"
        dl = "/download"
        cd(wdir)
        println("Building Reduce.jl with CSL binaries ... ")
        if Sys.islinux()
            src = "/reduce-csl_"
            if occursin("64",read(`uname -m`,String))
                download(http*date[ρ]*"/linux64"*src*rsvn[ρ]*"_amd64.tgz"*dl,joinpath(wdir,rtg))
            else
                download(http*date[ρ]*"/linux32"*src*rsvn[ρ]*"_i386.tgz"*dl,joinpath(wdir,rtg))
            end
            run(`rm -rf $(joinpath(wdir,"usr"))`)
            run(`tar -xvf $(rtg)`)
            run(`rm $(rtg)`)
            writever(ρ)
        elseif Sys.isapple()
            snap = "Reduce-snapshot"
            download(http*date[ρ]*"/macintosh/"*snap*"_"*rsvn[ρ]*".dmg"*dl,joinpath(wdir,"$(snap)_$(date[ρ]).dmg"))
            run(`hdiutil attach $(wdir)/$(snap)_$(date[ρ]).dmg`)
            run(`rm -rf $(joinpath(wdir,"csl"))`)
            run(`cp -r /Volumes/$(snap)/csl $(wdir)/csl`)
            run(`hdiutil unmount /Volumes/$(snap)`)
            run(`rm $(snap)_$(date[ρ]).dmg`)
            writever(ρ)
        else
            download(http*date[ρ]*"/Reduce-svn$(rsvn[ρ])-src.tar.gz"*dl,joinpath(wdir,rtg))
            run(`tar -xvf $(rtg)`)
            run(`rm $(rtg)`)
            cd(joinpath("$wdir","Reduce-svn$(rsvn[ρ])-src"))
            run(`./configure --with-psl`)
            run(`make`)
            writever(ρ)
        end
        println("DONE")
    end
else
    try
        #cmd = `"$(wdir)\psl\bpsl.exe" -td 16000000 -f "$(wdir)\red\reduce.img"`
        cmd = `"$(wdir)\reduce.exe" --nogui`
        process = _spawn(cmd)
        kill(process)
    catch
        cd(wdir)
        println("Building Reduce.jl with CSL binaries ...")
        cab = "wincsl.cab"
        http = "https://ayera.dl.sourceforge.net/project/reduce-algebra/snapshot_"
        download(http*date[ρ]*"/windows/wincsl_$(rsvn[ρ]).cab",joinpath(wdir,cab))
        open("redextract.bat","w") do f
                write(f,"expand $cab \"$(wdir)\" -F:*\n")
                #write(f,"reg Query \"HKLM\\Hardware\\Description\\System\\CentralProcessor\\0\" | find /i \"x86\" > NUL && set OSQ=32BIT || set OSQ=64BIT\n")
                #write(f,"echo %OSQ% > osbit.txt")
        end
        run(`$(wdir)\\redextract.bat`)
        writever(ρ)
        println("DONE")
    end
end

cd(oldwdir)


text:
using Nextion
using Test

@testset "hello_nextion" begin

    function _execute_command(nexSerial, cmd)
        println(cmd)
        send(nexSerial, cmd)
    end

    function main()
        portname = "/dev/ttyUSB0"
        #baudrate = 9600
        #nexSerial = NexSerial(portname, baudrate)
        nexSerial = NexSerial(portname)
        _execute_command(nexSerial, "page 0")
        sleep(2)
        _execute_command(nexSerial, "page 1")
        sleep(2)
        _execute_command(nexSerial, "t0.txt=\"Hello\"")
        close(nexSerial)
    end

    main()
    @test true

end


text:
using Multigraphs
using Base.Test

# write your own tests here
@test 1 == 1

g = Multigraph()
v1 = add_vertex!(g, "hi")
v2 = add_vertex!(g, "bye")
add_edge!(g, v1, v2, "dog")

# loop through vertices
for v in values(g.vertices)
    println(v)
    @test v.value in ["hi", "bye"]
end

# loop through edges
for e in values(g.edges)
    println(e)
    @test e.value == "dog"
end

# get out degree
for v in values(g.vertices)
    println("outdegree of vertex ",v.index," is: ", out_degree(g,v))
    if v.value == "hi"
        @test out_degree(g,v) == 1
    elseif v.value == "bye"
	@test out_degree(g,v) == 0
    end
end

# get in degree
for v in values(g.vertices)
    println("indegree of vertex ",v.index," is: ", in_degree(g,v))
    if v.value == "hi"
        @test in_degree(g,v) == 0
    elseif v.value == "bye"
        @test in_degree(g,v) == 1
    end
end

# iterate over out edges
for v in values(g.vertices)
    for e_ind in g.out_edges[v.index]
        print("Edge ",e_ind,": ",g.edges[e_ind])
        println(" connects ", v," and ", g.edges[e_ind].v2)
        @test v == g.edges[e_ind].v1
        @test g.edges[e_ind].v2.value == "bye"
    end
end

# iterate over in edges
for v in values(g.vertices)
    for e_ind in g.in_edges[v.index]
        print("Edge ",e_ind,": ",g.edges[e_ind])
        println(" connects ", g.edges[e_ind].v1," and ", v)
        @test v == g.edges[e_ind].v2
        @test g.edges[e_ind].v1.value == "hi"
    end
end


text:
module SentinelArrays

using Dates, Random

export SentinelArray, SentinelMatrix, SentinelVector, SentinelCollisionError

const RNG = [MersenneTwister()]

function __init__()
    Threads.resize_nthreads!(RNG)
    return
end

"""
    SentinelArray(A::AbstractArray, sentinel, value)
    SentinelVector{T}(undef, len, sentinel, value)
    SentinelMatrix{T}(undef, dims, sentinel, value)
    SentinelArray{T, N}(undef, dims, sentinel, value)

Construct a `SentinelArray` by either wrapping an existing `AbstractArray` or using the standard `undef` pattern, which first constructs an `Array` to be wrapped by `SentinelArray`.

A `SentinelArray{T}` wraps an `AbstractArray` of type `T`, and accepts a `sentinel` and `value` argument. If the wrapped array has an element equal to `sentinel`, `value` is returned instead.
This allows, for example, simulating a `Vector{Union{Float64, Missing}}` by doing something like `SentinelVector{Float64}(undef, len, NaN, missing)`. The resulting `SentinelArray` will have
`eltype` equal to `Union{T, typeof(value)}`.

For isbits types, a random/reasonable sentinel will be attempted if none provided. For non-isbits, `#undef` is used as sentinel. The default value, if not provided, is `missing`.

A `SentinelArray` allows setting an element to `value`, which will result in the `sentinel` being stored in the wrapped array.

For isbits element types, if a `setindex!` operation is attempted with the `SentinelArray`'s current sentinel value, another sentinel will be attempted to be chosen that doesn't already exist in the wrapped array.
If another sentinel can't be found (i.e. the values already exist in the wrapped array), an error will be thrown.

**CAVEATS**:
  * For float and date/time types, reasonable sentinels exist that don't have meaningful *real* values, so it's recommended to use the default sentinels
  * For `Integer` types, all bit patterns are valid, so a random value is chosen; if usage will involve extremely high cardinality relative to the type range, consider using `Vector{Union{T, Missing}}` instead
  * `SentinelArray`s are not allowed to have plain `Bool` elements since there isn't a possible sentinel
  * A `SentinelArray` *IS NOT* thread-safe when usage relies on `setindex!` and potential sentinel collision (due to the check to automatically cycle the sentinel to another value); this will normally only affect small `Integer` element types where the potential for sentinel collision is highest
  * Access to a `SentinelArray`'s raw data is possible via `parent(A)` or `pointer(A)`, but note that the raw data will *NOT* respect the sentinel=>value semantics; i.e. the raw data may have entries of the sentinel value.

Use-cases:
  * It is generally recommended to use `Vector{Union{T, typeof(value}}` instead of `SentinelArray` if possible
  * Currently, a non-copying operation like `convert(Vector{T}, A::Vector{Union{T, Missing}})` isn't possible, even though you may know there aren't `missing` values; a `SentinelArray` allows this operation by just "unwrapping" the storage array, like `parent(A)`
  * It's also currently not possible to use `Vector{Union{T, Missing}}` with `Mmap.mmap`; with `SentinelArray`, you can write the parent array to disk and then do `SentinelArray(Mmap.mmap(array_type, file))` (assuming usage of default sentinel/value)

Examples:
```julia
# initialize a new SentinelArray with default sentinel/value
A = SentinelVector{Float64}(undef, 10)

# wrap an existing array, manually providing sentinel/value
parent = zeros(UInt8, 10)
B = SentinelArray(parent, typemax(UInt8), nothing)
```
"""
mutable struct SentinelArray{T, N, S, V, A <: AbstractArray{T, N}} <: AbstractArray{Union{T, V}, N}
    data::A
    sentinel::S
    value::V

    SentinelArray(A::AbstractArray{Bool, N}, sentinel::S, value::V) where {N, S, V} = error("SentinelArray  not allowed for `Bool` elements (no sentinel available). Try using a `Array{Union{Bool, $V}` instead.")
    function SentinelArray(A::AbstractArray{T, N}, sentinel::S=defaultsentinel(T), value::V=defaultvalue(T)) where {T, N, S, V}
        return new{T, N, S, V, typeof(A)}(A, sentinel, value)
    end
end

Base.parent(A::SentinelArray) = A.data
# users of pointer should be careful because it's the raw storage data, which doesn't respect the sentinel semantics
Base.pointer(A::SentinelArray) = pointer(parent(A))

const SentinelVector{T} = SentinelArray{T, 1}
const SentinelMatrix{T} = SentinelArray{T, 2}

defaultvalue(T) = missing

function newsentinel(T)
    !isbitstype(T) && return undef
    return reinterpret(T, rand(RNG[Threads.threadid()], UInt8, sizeof(T)))[1]
end

defaultsentinel(T) = newsentinel(T)
defaultsentinel(::Type{Date}) = Date(Dates.UTD(typemin(Int64)))
defaultsentinel(::Type{DateTime}) = DateTime(Dates.UTM(typemin(Int64)))
defaultsentinel(::Type{Time}) = Time(Nanosecond(typemin(Int64)))
defaultsentinel(::Type{Int64}) = -8899831978349840752

# our own custom NaN bit pattern
sentinelbit(::Type{T}) where {T <: Unsigned} = Core.bitcast(T, T(1) << (8 * sizeof(T) - 1))
function defaultsentinel(::Type{T}) where {T <: Union{Float16, Float32, Float64}}
    nan = Core.bitcast(Base.uinttype(T), T(0) / T(0))
    return Core.bitcast(T, nan | sentinelbit(Base.uinttype(T)))
end

# constructors
function SentinelArray{T, N}(::UndefInitializer, dims::Tuple{Vararg{Integer}}, s=defaultsentinel(T), v=defaultvalue(T)) where {T, N}
    A = Array{T, N}(undef, dims)
    # initilize w/ missing values
    isbitstype(T) && fill!(A, s)
    return SentinelArray(A, s, v)
end

function SentinelArray{T}(::UndefInitializer, dims::Tuple{Vararg{Integer}}, s=defaultsentinel(T), v=defaultvalue(T)) where {T}
    SentinelArray{T, length(dims)}(undef, dims, s, v)
end

SentinelArray{T, N}(::UndefInitializer, dims::Vararg{Integer, N}) where {T, N} = SentinelArray{T, N}(undef, dims)
SentinelArray{T}(::UndefInitializer, dims::Integer...) where {T} = SentinelArray{T, length(dims)}(undef, dims)
SentinelVector{T}(::UndefInitializer, len::Int, s=defaultsentinel(T), v=defaultvalue(T)) where {T} = SentinelArray{T, 1}(undef, (len,), s, v)

function Base.convert(::Type{SentinelArray{T}}, arr::AbstractArray{T2, N}) where {T, T2, N}
    A = SentinelArray(Array{T, N}(undef, size(arr)))
    @inbounds for i in eachindex(arr)
        A[i] = arr[i]
    end
    return A
end

Base.convert(::Type{SentinelArray}, arr::AbstractArray{T}) where {T} = convert(SentinelArray{T}, arr)
Base.convert(::Type{SentinelVector{T}}, arr::AbstractArray) where {T} = convert(SentinelArray{T}, arr)

function Base.similar(A::SentinelArray{T, N, S, V}, ::Type{T2}, dims::Dims{N2}) where {T, N, S, V, T2, N2}
    SentinelArray(similar(parent(A), T2, dims), A.sentinel, A.value)
end

# conversion between SentinelArrays
function recode!(A::SentinelArray{T, N, S, V}, newsentinel::S) where {T, N, S, V}
    oldsentinel = A.sentinel
    P = parent(A)
    @simd for i in eachindex(P)
        @inbounds x = P[i]
        @inbounds P[i] = ifelse(eq(x, oldsentinel), newsentinel, x)
    end
    A.sentinel = newsentinel
    return
end

struct SentinelCollisionError <: Exception
    msg
end

function newsentinel!(arrays::SentinelArray{T, N, S, V}...; force::Bool=true) where {T, N, S, V}
    if S === UndefInitializer
        # undef can't be recoded
        return
    end
    A = arrays[1]
    oldsent = A.sentinel
    if !force
        if all(x->x.sentinel == oldsent, arrays)
            # all arrays have the same sentinel & we're not forcing a new one
            return
        end
    end
    attempts = 0
    newsent = newsentinel(T)
    # find a new sentinel that doesn't already exist in parent
    while true
        foundnewsent = false
        for A in arrays
            p = parent(A)
            for i in eachindex(p)
                @inbounds z = p[i]
                if eq(z, newsent)
                    foundnewsent = true
                    break
                end
            end
        end
        !foundnewsent && break
        newsent = newsentinel(T)
        attempts += 1
        attempts > 5 && throw(SentinelCollisionError("error trying to automatically find a new sentinel for SentinelArray; consider using `Vector{Union{$T, $V}}` instead"))
    end
    for A in arrays
        recode!(A, newsent)
    end
    return
end

# Basic AbstractArray interface definitions
Base.size(A::SentinelArray) = size(parent(A))
Base.size(A::SentinelArray, i) = size(parent(A), i)
Base.length(A::SentinelArray) = length(parent(A))
Base.axes(A::SentinelArray) = axes(parent(A))
Base.axes(A::SentinelArray, i) = axes(parent(A), i)
Base.stride(A::SentinelArray, k::Integer) = stride(parent(A), k)
Base.strides(A::SentinelArray) = strides(parent(A))

Base.IndexStyle(::Type{SentinelArray{T, N, S, V, A}}) where {T, N, S, V, A} = Base.IndexStyle(A)

# we define our own `eq` here because, for example, for NaN, we want ===
eq(x::T, y::T) where {T <: Real} = x === y
# but if they use `missing` as sentinel, we want isequal
eq(x, y) = isequal(x, y)

Base.@propagate_inbounds function Base.getindex(A::SentinelArray{T, N, S, V}, i::Int) where {T, N, S, V}
    checkbounds(A, i)
    if S === UndefInitializer
        @inbounds x = isassigned(parent(A), i) ? parent(A)[i] : A.value
        return x
    elseif Base.issingletontype(S) && Base.issingletontype(V) && S === V
        @inbounds x = parent(A)[i]
        return x
    else
        @inbounds x = parent(A)[i]
        return ifelse(eq(x, A.sentinel), A.value, x)
    end
end

unset!(A, i) = isassigned(A, i) && ccall(:jl_arrayunset, Cvoid, (Array, Csize_t), A, i - 1)

Base.@propagate_inbounds function Base.setindex!(A::SentinelArray{T, N, S, V}, val, i::Int) where {T, N, S, V}
    checkbounds(A, i)
    if S === UndefInitializer
        if eq(val, A.value)
            unset!(parent(A), i)
        else
            parent(A)[i] = val
        end
    elseif Base.issingletontype(S) && Base.issingletontype(V) && S === V
        parent(A)[i] = val
    else
        if eq(val, A.sentinel)
            # trying to set value of our sentinel
            # need to pick a new sentinel
            newsentinel!(A)
        end
        parent(A)[i] = ifelse(eq(val, A.value), A.sentinel, val)
    end
    return val
end

# other AbstractArray functions
function Base.empty!(A::SentinelVector)
    empty!(parent(A))
    return A
end

Base.copy(A::SentinelArray{T, N}) where {T, N} = SentinelArray(copy(parent(A)), A.sentinel, A.value)

function Base.resize!(A::SentinelVector{T}, len) where {T}
    oldlen = length(A)
    resize!(parent(A), len)
    if isbitstype(T) && len > oldlen
        sent = A.sentinel
        for i = (oldlen + 1:len)
            @inbounds A.data[i] = sent
        end
    end
    return A
end

function _growat!(A::SentinelVector, i, len)
    Base._growat!(parent(A), i, len)
    return
end

function _deleteat!(A::SentinelVector, i, len)
    Base._deleteat!(parent(A), i, len)
    return
end

function Base.push!(A::SentinelVector, val)
    _growat!(A, length(A) + 1, 1)
    @inbounds A[end] = val
    return A
end

function Base.deleteat!(A::SentinelVector, idx)
    deleteat!(parent(A), idx)
    return A
end

function Base.insert!(A::SentinelVector, idx::Integer, item)
    _growat!(A, idx, 1)
    @inbounds A[idx] = item
    return A
end

function Base.vcat(A::SentinelVector{T, S, V}, B::SentinelVector{T, S, V}) where {T, S, V}
    newsentinel!(A, B; force=false)
    return SentinelArray(vcat(parent(A), parent(B)), A.sentinel, A.value)
end

function Base.append!(A::SentinelVector{T, S, V}, B::SentinelVector{T, S, V}) where {T, S, V}
    newsentinel!(A, B; force=false)
    append!(parent(A), parent(B))
    return A
end

function Base.append!(A::SentinelVector{T}, b::AbstractVector) where T
    for x in b
        push!(A, x)
    end
    return A
end

const _default_splice = []

function Base.splice!(A::SentinelVector, i::Integer, ins=_default_splice)
    v = A[i]
    m = length(ins)
    if m == 0
        deleteat!(A, i)
    elseif m == 1
        A[i] = ins[1]
    else
        _growat!(A, i, m-1)
        k = 1
        for x in ins
            A[i+k-1] = x
            k += 1
        end
    end
    return v
end

function Base.splice!(A::SentinelVector, r::UnitRange{<:Integer}, ins=_default_splice)
    v = A[r]
    m = length(ins)
    if m == 0
        deleteat!(A, r)
        return v
    end

    n = length(A)
    f = first(r)
    l = last(r)
    d = length(r)

    if m < d
        delta = d - m
        _deleteat!(A, (f - 1 < n - l) ? f : (l - delta + 1), delta)
    elseif m > d
        _growat!(A, (f - 1 < n - l) ? f : (l + 1), m - d)
    end

    k = 1
    for x in ins
        A[f+k-1] = x
        k += 1
    end
    return v
end

function _growbeg!(A::SentinelVector, n)
    Base._growbeg!(parent(A), n)
    return
end

Base.prepend!(A::SentinelVector, items::AbstractVector) = _prepend!(A, Base.IteratorSize(items), items)

Base.prepend!(A::SentinelVector, iter) = _prepend!(A, Base.IteratorSize(iter), iter)
Base.pushfirst!(A::SentinelVector, iter...) = prepend!(A, iter)

function _prepend!(A, ::Union{Base.HasLength,Base.HasShape}, iter)
    n = length(iter)
    _growbeg!(A, n)
    i = 0
    for item in iter
        @inbounds A[i += 1] = item
    end
    return A
end

function _prepend!(A, ::Base.IteratorSize, iter)
    n = 0
    for item in iter
        n += 1
        pushfirst!(A, item)
    end
    reverse!(A, 1, n)
    return A
end

function Base.pop!(A::SentinelVector)
    if isempty(A)
        throw(ArgumentError("array must be non-empty"))
    end
    item = A[end]
    deleteat!(A, length(A))
    return item
end

function Base.pushfirst!(A::SentinelVector, item)
    _growbeg!(A, 1)
    A[1] = item
    return A
end

function Base.popfirst!(A::SentinelVector)
    if isempty(A)
        throw(ArgumentError("array must be non-empty"))
    end
    item = A[1]
    deleteat!(A, 1)
    return item
end

end # module


text:
@testset "testing simple matrix model" begin
    layerbuilder(k) = Flux.Dense(k, 2, NNlib.relu)
    x = ArrayNode(randn(Float32, 4, 5))
    m = reflectinmodel(x, layerbuilder)
    @test size(m(x).data) == (2, 5)
    @test typeof(m) <: ArrayModel
    @test eltype(m(x).data) == Float32
end

@testset "testing simple aggregation model" begin
    layerbuilder(k) = Flux.Dense(k, 2, NNlib.relu)
    x = BagNode(ArrayNode(randn(Float32, 4, 4)), [1:2, 3:4])
    m = reflectinmodel(x, layerbuilder)
    @test size(m(x).data) == (2, 2)
    @test typeof(m) <: BagModel
    @test eltype(m(x).data) == Float32
end

@testset "testing simple tuple models" begin
    layerbuilder(k) = Flux.Dense(k, 2, NNlib.relu)
    x = TreeNode((ArrayNode(randn(Float32, 3, 4)), ArrayNode(randn(Float32, 4, 4))))
    m = reflectinmodel(x, layerbuilder)

    @test eltype(m(x).data) == Float32
    @test size(m(x).data) == (2, 4)
    @test typeof(m) <: ProductModel
    @test typeof(m.ms[1]) <: ArrayModel
    @test typeof(m.ms[2]) <: ArrayModel
    x = TreeNode((BagNode(ArrayNode(randn(Float32, 3, 4)), [1:2, 3:4]),
                  BagNode(ArrayNode(randn(Float32, 4, 4)), [1:1, 2:4])))
    m = reflectinmodel(x, layerbuilder)
    @test size(m(x).data) == (2, 2)
    @test typeof(m) <: ProductModel
    @test typeof(m.ms[1]) <: BagModel
    @test typeof(m.ms[1].im) <: ArrayModel
    @test typeof(m.ms[1].bm) <: ArrayModel
    @test typeof(m.ms[2]) <: BagModel
    @test typeof(m.ms[2].im) <: ArrayModel
    @test typeof(m.ms[2].bm) <: ArrayModel
end

@testset "testing nested bag model" begin
    layerbuilder(k) = Flux.Dense(k, 2, NNlib.relu)
    bn = BagNode(ArrayNode(randn(Float32, 2, 8)), [1:1, 2:2, 3:6, 7:8])
    x = BagNode(bn, [1:2, 3:4])
    m = reflectinmodel(x, layerbuilder)
    @test size(m(x).data) == (2, 2)
    @test typeof(m) <: BagModel
    @test typeof(m.im) <: BagModel
    @test typeof(m.im.im) <: ArrayModel
    @test typeof(m.im.bm) <: ArrayModel
    @test typeof(m.bm) <: ArrayModel
    @test eltype(m(x).data) == Float32
end


text:
function fstep(state::Union{SimState, Nothing}, agent::Agent) end
struct Data
    pos::Real2D
end

d1 = Data(Real2D(1.0,1.0))
d2 = Data(Real2D(2.0,2.0))

a1 = Agent(fstep,d1)
a2 = Agent(fstep,d2)

@test a1 != a2

a3 = deepcopy(a2)

#@test pointer_from_objref(a2) != pointer_from_objref(a3)

@test a1.state.pos != a3.state.pos
@test a2.state.pos == a3.state.pos

@test a1 != a2
@test a2 == a3

@test a1.id != a2.id
@test a1.id != a3.id
@test a2.id == a3.id


text:
# Modified from Shreyas Kowshik's implementation.

"""
Data dictionary buffer
"""
@with_kw mutable struct Buffer
    exp_dict::Dict = Dict()
end

register!(b::Buffer, name::String) = b.exp_dict[name] = []

"""
Add a variable for it's history to be logged
"""
add!(b::Buffer, name::String, value::Any) = push!(b.exp_dict[name], value)

import Base.get
get(b::Buffer, name::String) = b.exp_dict[name]

function clear!(b::Buffer, name=nothing)
	if name == nothing
		for key in keys(b.exp_dict)
			register!(b, key)
		end
	else
		@assert typeof(name) <: String "Name of key must be a string"
		register!(b, name)
	end
	return nothing
end

function initialize_episode_buffer()
    eb = Buffer()
    register!(eb,"states")
    register!(eb,"actions")
    register!(eb,"rewards")
    register!(eb,"next_states")
    register!(eb,"dones")
    register!(eb,"returns")
    register!(eb,"advantages")
    register!(eb,"log_probs")
    register!(eb,"kl_params")
    return eb
end

function initialize_stats_buffer()
    sb = Buffer()
    register!(sb,"rollout_rewards")
    return sb
end


text:
# L0 pseudo-norm (times a constant)

"""
  NormL0(λ::Real=1.0)

Returns the function `g(x) = λ*countnz(x)`, for a nonnegative parameter `λ ⩾ 0`.
"""

immutable NormL0{R <: Real} <: ProximableFunction
  lambda::R
  function NormL0(lambda::R)
    if lambda < 0
      error("parameter λ must be nonnegative")
    else
      new(lambda)
    end
  end
end

NormL0{R <: Real}(lambda::R=1.0) = NormL0{R}(lambda)

function (f::NormL0){T <: RealOrComplex}(x::AbstractArray{T})
  return f.lambda*countnz(x)
end

function prox!{T <: RealOrComplex}(f::NormL0, x::AbstractArray{T}, y::AbstractArray{T}, gamma::Real=1.0)
  countnzy = 0;
  gl = gamma*f.lambda
  for i in eachindex(x)
    over = abs(x[i]) > sqrt(2*gl);
    y[i] = over*x[i];
    countnzy += over;
  end
  return f.lambda*countnzy
end

fun_name(f::NormL0) = "weighted L0 pseudo-norm"
fun_dom(f::NormL0) = "AbstractArray{Real}, AbstractArray{Complex}"
fun_expr(f::NormL0) = "x ↦ λ countnz(x)"
fun_params(f::NormL0) = "λ = $(f.lambda)"

function prox_naive{T <: RealOrComplex}(f::NormL0, x::AbstractArray{T}, gamma::Real=1.0)
  over = abs.(x) .> sqrt(2*gamma*f.lambda);
  y = x.*over;
  return y, f.lambda*countnz(y)
end


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule nghttp2_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("nghttp2")
JLLWrappers.@generate_main_file("nghttp2", UUID("8e850ede-7688-5339-a07c-302acd2aaf8d"))
end  # module nghttp2_jll


text:
# Defines `XExpr` the type representing every document and `xexpr`, a
# helper used to create `XExpr`s.

struct XExpr
    tag::Symbol
    attributes::Dict{Symbol, <:Any}
    children
end

Base.show(io::IO, x::XExpr) = print_tree(io, x, 2)

function AbstractTrees.printnode(io::IO, x::XExpr)
    print(io, x.tag)
    if !isempty(x.attributes)
        print(io, " [")
        for (i, (key, value)) in enumerate(x.attributes)
            if i != 1
                print(io, ", ")
            end
            print(io, key, " = ", value)
        end
        print(io, "]")
    end
end

AbstractTrees.children(x::XExpr) = x.children

"""
    xexpr(str)
    xexpr(tag, children...)
    xexpr(tag, attributes::Dict, children...)

Create an `XExpr`.
"""
xexpr(s) = s
xexpr(x::XExpr) = x

xexpr(t::Tuple) = xexpr(t...)

function xexpr(tag::Symbol, children...)
    return XExpr(tag, Dict{Symbol, Any}(), xexpr.(children))
end

function xexpr(tag::Symbol, attributes::Dict{Symbol, <:Any}, children...)
    return XExpr(tag, attributes, xexpr.(children))
end

function xexpr(tag::Symbol, attributes::Dict{Symbol, <:Any}, children::Vector{<:Union{XExpr, T}}) where T
    return XExpr(tag, attributes, children)
end

function Base.:(==)(x1::XExpr, x2::XExpr)
    return ((x1.tag == x2.tag) &&
            (x1.attributes == x2.attributes) &&
            (length(x1.children) == length(x2.children)) &&
            all(c1 == c2 for (c1, c2) in zip(x1.children, x2.children)))
end

Base.:(==)(::XExpr, _) = false
Base.:(==)(_, ::XExpr) = false

## API

"""
    tag(x::XExpr)

Get the tag of an x-expression.

```julia
x = xexpr(:tagname, "Hello", "World")
tag(x) == :tagname
```
"""
tag(x::XExpr) = x.tag
attributes(x::XExpr) = x.attributes

withattributes(x::XExpr, newattrs) = xexpr(
    tag(x),
    merge(attributes(x), newattrs),
    children(x),
)

withtag(x::XExpr, t) = xexpr(
    t,
    attributes(x),
    children(x)
)


text:
using PyCall
using LinearAlgebra

#using SciPy: SciPy

"""
Compute ground state
"""
function solve_gs(ham_qubit::QubitOperator, circuit::VariationalQuantumCircuit, state0::QuantumState;
    theta_init=nothing, comm=MPI_COMM_WORLD, maxiter=200, gtol=1e-5, verbose=false)
    if is_mpi_on && comm === nothing
        error("comm must be given when mpi is one!")
    end
    if comm === nothing
        rank = 0
    else
        rank = MPI.Comm_rank(comm)
    end
    # これを使わずに SciPy.optimize でよい
    scipy_opt = pyimport("scipy.optimize")

    # Define a cost function
    function cost(theta_list)
        update_circuit_param!(circuit, theta_list)
        state = copy(state0)
        update_quantum_state!(circuit, state)
        return get_expectation_value(ham_qubit, state)
    end

    if theta_init === nothing
        theta_init = rand(size(circuit.theta_offsets)[1])
    end
    if comm !== nothing
        # Make sure all processes use the same initial values
        MPI.Bcast!(theta_init, 0, comm)
    end
    cost_history = Float64[] #コスト関数の箱
    init_theta_list = theta_init
    push!(cost_history, cost(init_theta_list))

    method = "BFGS"
    options = Dict("disp" => verbose, "maxiter" => maxiter, "gtol" => gtol)
    function callback(x)
        push!(cost_history, cost(x))
        if verbose && rank == 0
            println("iter ", length(cost_history), " ", cost_history[end])
        end
    end
    # opt = SciPy.optimize.minimize(...)
    opt = scipy_opt.minimize(cost, init_theta_list, method=method,
        callback=callback, jac=generate_numerical_grad(cost),
        options=options)

    return cost_history, get_thetas(circuit)
end

max_stars_repo_name: Sakurairihito/QCMaterialNew

max_issues_repo_name: sakurairihito/QCMaterialNew

max_forks_repo_name: sakurairihito/QCMaterialNew


text:
### SO THIS WORKS

using PeaceVote
using PeaceCypher
import PeaceFounder

demespec = DemeSpec("PeaceDeme",:default,:PeaceCypher,:default,:PeaceCypher,:PeaceFounder)
deme = Deme(demespec)

max_stars_repo_name: akels/DemocracyCredit

max_issues_repo_name: akels/DemocracyCredit

max_forks_repo_name: akels/DemocracyCredit


text:
# coding: utf-8

# In[1]:

#okay. the epic one!
#quick sort, mmm, not so quick
#the idea of quick sort is be the quickest
#first thing first, we gotta pick a pivot number for the list
#normally, people use the first element as pivot number
#however, we may encounter a case that the first element is the largest or the smallest
#that would make our sorting a total failure
#here i use the median of 3 approach
#take the first, the last and the one in the middle
#and get the median of the three
#we use the median as a pivot number
#after that, we do the same trick as merge sort
#we have a left index and a right index
#we do traversal on both indices
#say the left index comes from the left part
#the right index comes from the right part
#we compare two elements with pivot number at the same time
#there are four cases assuming that we dont have any duplicate values in the list
#first case, left is larger than pivot, right is smaller than pivot
#so we swap both
#second case, left is larger than pivot, right is larger than pivot as well
#we freeze the left, and move right side index one step backwards
#right=right-1
#then if left is larger than pivot, right is smaller than pivot
#we back to the first case
#if it is still the second case, we repeat this procedure until we move to first case
#or left and right indices cross, we stop the sorting
#third case, left is smaller than pivot, right is smaller than pivot
#it is the opposite case of the second case
#fourth case, left is smaller than pivot, right is larger than pivot
#great, we do nothing but moving both indices closer to the centre
#these are four basic scenarios when we do both traversals
#when left and right indices cross, we stop the sorting
#and we insert the pivot number in where indices cross
#the next step is just like merge sort
#we divide the list in two halves (excluding the pivot number)
#we perform the same trick on both halves recursively
#until we reach the base case when there are only two elements in the list
#we sort those two elements with simple comparison

# In[2]:

function quick_sort(arr)

    #the first step is to get a pivot number
    #it only works on list with more than two elements
    #otherwise there is no point of getting a pivot number
    if length(arr)>2

        #we take three elements, first, last and middle
        #we create a new list
        three_element=[arr[1],arr[end],arr[Int32(floor((1+length(arr))/2))]]

        #get the median
        #remove the max and the min
        #what is left is the pivot
        maxvalue=max(three_element[1],three_element[2],three_element[3])
        minvalue=min(three_element[1],three_element[2],three_element[3])
        setdiff!(three_element,maxvalue)
        setdiff!(three_element,minvalue)
        pivot=three_element[1]

        #move the pivot to the beginning of the list
        setdiff!(arr,pivot)
        insert!(arr,1,pivot)

        #initialize
        #first index is at 2, cuz we wanna exclude pivot number
        left=2
        right=length(arr)

        #here comes the real deal
        #when left and right dont cross
        #excluding a case when two indices equal to each other
        while left-1<right

            #case 4
            if (arr[left]<pivot) && (arr[right]>pivot)
                left+=1
                right-=1

            #case 3
            elseif (arr[left]<pivot) && (arr[right]<pivot)
                left+=1

            #case 2
            elseif (arr[left]>pivot) && (arr[right]>pivot)
                right-=1

            #case 1
            else
                arr[left],arr[right]=arr[right],arr[left]
            end

        end

        #when left and right indices cross
        #when indices cross, we are one step after the input list is sorted
        #therefore, we insert pivot at left-1 instead of left
        if left>=right
            setdiff!(arr,pivot)
            insert!(arr,left-1,pivot)
        end

        #the recursive part
        #we do the same trick on two halves
        arr[left:end]=quick_sort(arr[left:end])
        arr[1:left-2]=quick_sort(arr[1:left-2])

    end

    #the base case
    #when we are left with two elements in a sublist
    #we just compare and return in reverse order
    #u might ask, what about one element
    #well, we dont have to do anything so no codes needed
    if length(arr)==2
        if arr[1]>arr[2]
            arr[1],arr[2]=arr[2],arr[1]
        end
    end

    return arr

end

# In[3]:

#there is a constraint for quick sort
#duplicate values would jeopardize everything we have built
#to solve that issue, we must amend the criteria for selecting pivot number
#for simplicity, i remove duplicates from our test list
#to handle duplicate values which might affect pivot
#my idea is to insert an extra if function
#if elements not at position 0 equal to pivot
#we create a temporary list to collect them
#when we plan to insert pivot
#we insert the temporary list to the full list

#since we have recursive functions
#this approach to get a quick sort
#is kinda different from my textbook
#to see the alternative version
#plz click the link below
# http://interactivepython.org/runestone/static/pythonds/SortSearch/TheQuickSort.html

# In[4]:

for _ in 1:100

    test_arr=rand(1000)

    if !(merge_sort(test_arr)==sort(test_arr))
        printstyled("Erreur",color=:red)
    end

end

max_stars_repo_path: quick sort.jl

max_stars_repo_name: je-suis-tm/search-and-sort

max_issues_repo_path: quick sort.jl

max_issues_repo_name: HafizSabia/search-and-sort

max_forks_repo_path: quick sort.jl

max_forks_repo_name: HafizSabia/search-and-sort


text:
# Code for `WageRegressions` type

max_exper(wr :: WageRegressions) = wr.maxExper
exper_groups(wr :: WageRegressions) = wr.experGroupV;
max_exper_exponent(wr :: WageRegressions) = wr.maxExperExponent;
# School groups that are recognized
CollegeStratBase.ed_symbols(wr :: WageRegressions) = vcat(exper_groups(wr)...);
CollegeStratBase.n_school(wr :: WageRegressions) = length(ed_symbols(wr));
use_parental_dummies(wr :: WageRegressions) = wr.useParentalDummies;

default_wage_regressions() =
    WageRegressions(maxExperExponent = 4, maxExper = 14);

# Two school groups
wage_regressions_two() =
    WageRegressions(experGroupV = [[:HSG, :SC], [:CG]],
        maxExperExponent = 4);

# Suffix for a regression file, such as "_inc--same"
# "_inc" determines whether parental income is a regressor.
# "_same" or "_dif" refers to same or different experience profile by schooling.
# "_OLD" means that the default category for graduates is q2 (no longer +++++)
function regr_file_suffix(wr :: WageRegressions)
    if use_parental_dummies(wr)
        incSuffix = "_inc";
    else
        incSuffix = "";
    end

    if length(exper_groups(wr)) == 1
        groupSuffix = "--same";
    elseif exper_groups(wr) == [[:HSG, :SC], [:CG]]
        groupSuffix = "--dif";
    else
        error("Invalid $wr");
    end

    return incSuffix * groupSuffix  # +++++ * "_OLD"
end

# -------------