text:
module NicePipes

if VERSION < v"1.3.0-rc4"
    @warn "Can't use binary artifacts, using your system's `grep` and `sed`."
    grep(f) = f("grep")
    sed(f) = f("sed")
else
    using grep_jll, sed_jll
end

struct ShPipe{T,C}
    val::T
    cmd::C
    args::Cmd
end

# like Base.open, but doesn't throw if exitcode is non-zero and always returns process instead
# of return value of f
function _open(f::Function, cmds::Base.AbstractCmd, args...; kwargs...)
    P = open(cmds, args...; kwargs...)
    ret = try
        f(P)
    catch
        kill(P)
        rethrow()
    finally
        close(P.in)
    end
    wait(P)
    return P
end

function Base.show(io_out::IO, x::ShPipe)
    x.cmd() do cmd
        p = _open(`$cmd $(x.args)`, "w", io_out) do io_in
            show(io_in, MIME("text/plain"), x.val)
        end
        if x.cmd === grep && p.exitcode == 1
            println(io_out, "No matches found!")
        elseif p.exitcode != 0
            print(io_out, "Command $(p.cmd) failed with exit code $(p.exitcode)")
        elseif x.cmd === grep
            # delete additional newline
            print(io_out, "\033[1A")
        end
    end
    return nothing
end

struct ShPipeEndpoint{C}
    cmd::C
    args::Cmd
end

macro p_cmd(s)
    cmd, args = match(r"^(.*)\s(.*)$", s).captures
    return :(ShPipeEndpoint(f->f($cmd)), @cmd($args))
end

(endpoint::ShPipeEndpoint)(val) = ShPipe(val, endpoint.cmd, endpoint.args)
Base.:|(val, endpoint::ShPipeEndpoint) = val |> endpoint

macro special_command(cmd)
    return quote
        export $(Symbol('@', cmd))
        macro $cmd(args...)
            args = map(args) do arg
                # interpret raw_str as raw string
                if Meta.isexpr(arg, :macrocall) && arg.args[1] === Symbol("@raw_str")
                    arg = arg.args[3]
                end
                return arg isa String ? string('"', arg, '"') : arg
            end
            args = join(args, ' ')
            return :(ShPipeEndpoint($$cmd, @cmd($args)))
        end
    end |> esc
end

@special_command grep
@special_command sed

end


text:
function __init__()
    @require CDDLib = "3391f64e-dcde-5f30-b752-e11513730f60" include("Initialization/init_CDDLib.jl")
    @require Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" include("Initialization/init_Distributions.jl")
    @require Expokit = "a1e7a1ef-7a5d-5822-a38c-be74e1bb89f4" include("Initialization/init_Expokit.jl")
    @require Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" include("Initialization/init_Makie.jl")
    @require Optim = "429524aa-4258-5aef-a3af-852621145aeb" include("Initialization/init_Optim.jl")
    @require Polyhedra = "67491407-f73d-577b-9b50-8179a7c68029" include("Initialization/init_Polyhedra.jl")
end

function initialize_mesh()
    if isdefined(@__MODULE__, :Polyhedra) && isdefined(@__MODULE__, :Makie)
       eval(load_mesh())
    end
end

"""
    require(package::Symbol; fun_name::String="", explanation::String="")

Helper method to check for optional packages and printing an error message.

### Input

- `package`     -- symbol of the package name
- `fun_name`    -- (optional; default: `""`) name of the function that requires
                   the package
- `explanation` -- (optional; default: `""`) additional explanation in the error
                   message

### Output

If the package is loaded, this function has no effect.
Otherwise it prints an error message.

### Algorithm

This function uses `@assert` and hence loses its ability to print an error
message if assertions are deactivated.
"""
function require(package::Symbol; fun_name::String="", explanation::String="")
    @assert isdefined(@__MODULE__, package) "package '$package' not loaded" *
        (fun_name == "" ? "" :
            " (it is required for executing `$fun_name`" *
            (explanation == "" ? "" : " " * explanation) * ")")
end

ext: jl


text:
using IRTools
using IRTools: var, xcall, insert!, insertafter!, func, recurse!, @dynamo

const timable_list = WhiteBlackLists()
const recursable_list = WhiteBlackLists()
"""
    timable(ex::GlobalRef)

    decides if function will be timed or not. By default, all functions are timed.
    But if any function is white-listed by adding function / module  to whitelists,
    the default becomes deny and only white-listed function function are allowed.

    If something is added to blacklist, that the default accept is kept and only
    blacklisted functions are not timed.
"""
function timable(ex::GlobalRef)
    iswhite(timable_list, ex) && return(true)
    isblack(timable_list, ex) && return(false)
    return(isempty(timable_list) ? true : false)
end

function timable(ex::Expr)
    ex.head != :call && return(false)
    isempty(ex.args) && return(false)
    timable(ex.args[1])
end

timable(ex::IRTools.Inner.Variable) = true

timable(ex) = false

"""
    recursable(ex::Expr)
    recursable(ex::GlobalRef)

    decides if we should descend into the function or not. By default, we descent into all
    functions, but with a combination of blacklists and whitelists, this can be changed.

    If something is added to blacklist, that the default accept is kept and only
"""
function recursable(ex::GlobalRef)
    ex.name ∈ (:profile_fun, :record_start, :record_end) && return(false)
    iswhite(recursable_list, ex) && return(true)
    isblack(recursable_list, ex) && return(false)
    return(isempty(recursable_list) ? true : false)
end

recursable(ex::IRTools.Inner.Variable) = true

function recursable(ex::Expr)
    ex.head != :call && return(false)
    isempty(ex.args) && return(false)
    recursable(ex.args[1])
end

recursable(ex) = false

"""
    exportname(ex)

    name of the function call that would be logged to the profiler
"""
exportname(ex::GlobalRef) = QuoteNode(ex.name)
exportname(ex::Symbol) = QuoteNode(ex)
exportname(ex::Expr) = exportname(ex.args[1])
exportname(i::Int) = QuoteNode(Symbol("Int(",i,")"))
exportname(i::IRTools.Inner.Variable) = QuoteNode(Symbol("Id(",i.id,")"))

profile_fun(f::Core.IntrinsicFunction, args...) = f(args...)
profile_fun(f::Core.Builtin, args...) = f(args...)

@dynamo function profile_fun(f, args...)
    ir = IRTools.Inner.IR(f, args...)
    for (v, ex) in ir
        if timable(ex.expr)
            fname = exportname(ex.expr)
            insert!(ir, v, xcall(LoggingProfiler, :record_start, fname))
            insertafter!(ir, v, xcall(LoggingProfiler, :record_end, fname))
        end
    end
    for (x, st) in ir
        recursable(st.expr) || continue
        ir[x] = xcall(profile_fun, st.expr.args...)
    end
    return ir
end

macro record(ex)
    esc(Expr(:call, :(LoggingProfiler.profile_fun), ex.args...))
end

macro recordfun(ex::Expr)
    @assert ex.head == :call
    fname = ex.args[1]
    x = gensym(:result)
    rex = quote
        LoggingProfiler.record_start($(QuoteNode(fname)))
        $(x) = $(ex)
        LoggingProfiler.record_end($(QuoteNode(fname)))
        $(x)
    end
    esc(rex)
end
# macro record(ex)
#     ex.head != :call && error("we can profiler only function calls")
#     s = fname(ex)
#     r = gensym(:record_result)
#     b = [
#         Expr(:call, :(LoggingProfiler.record_start), s),
#         Expr(Symbol("="), r, Expr(:call, :(LoggingProfiler.profile_fun), ex.args...)),
#         Expr(:call, :(LoggingProfiler.record_end), s),
#         r,
#     ]
#     Expr(:block, b...)
# end

max_stars_repo_name: pevnak/LoggingProfiler

max_issues_repo_name: pevnak/LoggingProfiler

max_forks_repo_name: pevnak/LoggingProfiler


text:
module GLEPlot

# Base
import Base: /, |>, take!, isempty

# stdlib
using DelimitedFiles

# external
using Colors
using OrderedCollections: LittleDict

const ∅   = nothing
const AV  = AbstractVector
const AVM = AbstractVecOrMat
const AM  = AbstractMatrix
const AVR = AV{<:Real}
const F64 = Float64

const T2F = NTuple{2, F64}
const T3F = NTuple{3, F64}
const T2R = NTuple{2, Real}
const T3R = NTuple{3, Real}

const PT_TO_CM   = 0.0352778         # 1pt in cm
const Option{T}  = Union{Nothing, T}
const CanMiss{T} = Union{Missing, T}
const Listable   = Union{Tuple, NTuple, AV}

const PALETTE_1 = [ # imitated from tableau 10 - 2
    RGB(0.33, 0.47, 0.64),
    RGB(0.90, 0.57, 0.26),
    RGB(0.82, 0.37, 0.36),
    RGB(0.51, 0.70, 0.69),
    RGB(0.42, 0.62, 0.35),
    RGB(0.91, 0.79, 0.37),
    RGB(0.66, 0.49, 0.62),
    RGB(0.95, 0.63, 0.66),
    RGB(0.59, 0.46, 0.38),
    RGB(0.72, 0.69, 0.67)
]

const GP_ENV = LittleDict{String, Any}(
    "TMP_PATH"     => mktempdir(),   # where intermediate files are stored
    "PALETTE"      => PALETTE_1,
    "CONT_PREVIEW" => true,
    "ALL_FIGS"      => nothing,
    "CUR_FIG"       => nothing,
    "CUR_AXES"      => nothing,
    "CLEAN_TMP"     => true,
)

function palette(cntr::Int)::Color
    ncols = length(GP_ENV["PALETTE"])
    idx   = mod(cntr, ncols)
    idx   = ifelse(idx == 0, ncols, idx)
    return GP_ENV["PALETTE"][idx]
end

# see subroutines/
const GLE_DRAW_SUB = LittleDict{String, String}()

include("utils.jl")
include("exceptions.jl")
include("gle_script.jl")
include("strings.jl")
include("defaults.jl")

include("types/style.jl")
include("types/drawing_2d.jl")
include("types/drawing_3d.jl")
include("types/ax_object.jl")
include("types/object_2d.jl")
include("types/object_3d.jl")
include("types/ax.jl")
include("types/figure.jl")
include("types/utils.jl")

include("set/prop_dicts.jl")
include("set/style.jl")
include("set/drawing.jl")
include("set/object.jl")
include("set/ax_object.jl")
include("set/legend.jl")
include("set/ax.jl")
include("set/figure.jl")
include("set/properties.jl")

include("apply/utils.jl")
include("apply/style.jl")
include("apply/drawing_2d.jl")
include("apply/drawing_3d.jl")
include("apply/object_2d.jl")
include("apply/ax_object.jl")
include("apply/legend.jl")
include("apply/ax.jl")
include("apply/figure.jl")

include("subroutines/boxplot.jl")
include("subroutines/heatmap.jl")
include("subroutines/marker.jl")
include("subroutines/palette.jl")
include("subroutines/plot3.jl")

include("calls/drawing_2d.jl")
include("calls/drawing_3d.jl")
include("calls/object_2d.jl")
include("calls/ax.jl")
include("calls/ax_object.jl")
include("calls/legend.jl")
include("calls/figure.jl")
include("calls/layout.jl")

include("render/show_text.jl")
include("render/show_image.jl")
include("render/savefig.jl")

end


text:
# Perform a univariate OLS regression (with non-zero intercept) to estimate the
# per-evaluation execution time of an expression.
#
# Arguments:
#
#     x::Vector{Float64}: The number of times the expression was evaluated.
#
#     y::Vector{Float64}: The total execution time of the expression's
#         (potentially repeated) evaluation.
#
# Returns:
#
#     a::Float64: The intercept of the univariate OLS model.
#
#     b::Float64: The slope of the univariate OLS model.
#
#     r²::Float64: The r-squared of the univariate OLS regresion

function ols(x::Vector{Float64}, y::Vector{Float64})
    a, b = linreg(x, y)
    r² = 1 - var(a + b * x - y) / var(y)
    return a, b, r²
end

function sem_ols(x::Vector{Float64}, y::Vector{Float64})
    a, b = linreg(x, y)
    n = length(x)
    residuals = y - (a + b * x)
    sem_b = sqrt(((1 / (n - 2)) * sum(residuals.^2)) / sum((x - mean(x)).^2))
    return sem_b
end


text:
using .SFEM
#using BenchmarkTools

A = "G://SMESH//test.cen"
B = "G://SMESH//test.ele"
C = "G://SMESH//test.facein"
D = "G://SMESH//test.faceout"
E = "G://SMESH//test.node"
F = "G://MESH//test.face"

EE, NU, FF = 2E8, 0.3, -2500
Node, Ele, Cen, Face_out, Face_in = IOsfem(A, B, C, D, E)
@time begin
KE, Q = Ksfem(Node, Ele, Cen, Face_out, Face_in, EE, NU, FF)
end
#@time begin
result = MKL(KE, Q)
#end
#Draw_mesh(Node,Disp,B)
#Draw_disp(Node,Disp,F)
#Compare(Node,result,EE,FF)
#AA = WriteDISP(Node,Ele,result)
function dd(Node,result)
    value = Vector{Float64}()
    for i = 1:size(Node, 1)
        if Node[i,2] == 0 && Node[i,3] == 0
            append!(value,result[i,3])
        end
    end
    return value
end

max_stars_repo_path: src/single-core juSFEM/main.jl

max_stars_repo_name: Elliothuo/juSFEM

max_issues_repo_path: src/single-core juSFEM/main.jl

max_issues_repo_name: Elliothuo/juSFEM

max_forks_repo_path: src/single-core juSFEM/main.jl

max_forks_repo_name: Elliothuo/juSFEM


text:
#args = [:(x::Float64), :y, :z];
#args = [:(x::Number)];
args = ["x"];
#exp = :(x+y*z+1)

#exp = :(+(x,+(*(y,z),1)))

args = [:x, :y]
exp3 = "(+(x, y, 1))"

cad = "("
for i in args
    cad = cad*String(i)*", "
end

cad = cad[1:length(cad)-2]
cad = cad*")"

eval(symbol)

valores = [1,2]#collect(-5:0.1:5)

p = cad*" -> "*exp3

h = Meta.parse(p)

println(h)
println(p)
f2 = eval(Meta.parse(p))

funcion = Meta.parse("function")
#println(hola)

#exp = Meta.parse(exp)

#funcion = eval(exp)

exp2 = "+(2,1)"
exp2 = Meta.parse(exp2)

funcion2 = eval(:(function f($(args...)) $exp2 end))

#symbol=:(($(args...)) $exp)

#codigo=:(function f($(args...)) $exp end)

#eval(:(function f($(args...)) $exp end))

valores = [1, 2]#collect(-5:0.1:5)

""""
funcion2(x)=x^2

println(f(2))

#funcion=eval(symbol)

acc = 0

for i=1:length(valores)
    global acc  += (funcion2(valores[i])-f(valores[i]))^2
end
acc /=length(valores)
"""

max_stars_repo_name: SergioGmezM/GeneticJulia

max_issues_repo_name: SergioGmezM/GeneticJulia

max_forks_repo_name: SergioGmezM/GeneticJulia


text:
module DeepConvert

include("deepconv.jl")
include("intsandfloats.jl")

end # module


text:
using StaticValues, Test
import StaticValues: BaseNumber, S2B, B2S

#=
@generated function sval(x::Int)
    y = Val{x}()
    :($y)
end

base_unsigned = (UInt128,UInt16,UInt32,UInt64,UInt8)
base_signed = (Int128,Int16,Int32,Int64,Int8)
base_integer = (base_unsigned..., base_signed...)
base_float = (Float64,Float32,Float16)
base_real = (base_integer..., base_float..., Rational, Irrational)
base_number = (base_real..., Complex)
static_unsigned = (SUInt128,SUInt16,SUInt32,SUInt64,SUInt8)
static_signed = (SInt128,SInt16,SInt32,SInt64,SInt8)
static_integer = (static_unsigned..., static_signed...)
static_float = (SFloat64,SFloat32,SFloat16)

=#
# test for preservation of 'staticness'
getvalues(x::SReal) = values(x)
getvalues(x::BaseNumber) = error("Got $(typeof(x)) instead of static value.")

@testset "inference" begin
    include("inferencetests.jl")
end

@testset "ranges" begin
    include("rangetests.jl")
end


text:
module JSON_PBP

include("shared.jl")
import JSON

function get_pbp(game_id::String)::Dict{String, Any}
    request = HTTP.request("GET", "http://statsapi.web.nhl.com/api/v1/game/$game_id/feed/live")
    response = String(request.body)
    return JSON.parse(response)
end

function get_teams(response::Dict{String, Any})::Dict{Symbol, String}
    return Dict{Symbol, String}(
        :home => uppercase(response["gameData"]["teams"]["home"]["name"]),
        :away => uppercase(response["gameData"]["teams"]["away"]["name"])
    )
end

function event_type(event_desc::String)::Union{Symbol, Nothing}
    EVENTS = Dict{String, Symbol}(
        "PERIOD_START" => :PSTR,
        "FACEOFF" => :FAC,
        "BLOCKED_SHOT" => :BLOCK,
        "GAME_END" => :GEND,
        "GIVEAWAY" => :GIVE,
        "GOAL" => :GOAL,
        "HIT" => :HIT,
        "MISSED_SHOT" => :MISS,
        "PERIOD_END" => :PEND,
        "SHOT" => :SHOT,
        "STOP" => :STOP,
        "TAKEAWAY" => :TAKE,
        "PENALTY" => :PENL,
        "EARLY_INT_START" => :EISTR,
        "EARLY_INT_END" => :EIEND,
        "SHOOTOUT_COMPLETE" => :SOC,
        "CHALLENGE" => :CHL,
        "EMERGENCY_GOALTENDER" => :EGPID
    )
    event = [EVENTS[e] for e in keys(EVENTS) if occursin(e, event_desc)]
    return length(event) > 0 ? event[1] : nothing
end

function create_missing_columns(play_dict::Dict{Symbol, Any}, event::Dict{String,Any})
    N = "players" in keys(event) ? length(event["players"]) : 1
    # There are at most three events...
    MAX_N::Int8 = 4

    for i in range(N, MAX_N)
        play_dict[Symbol("p$(i)_name")] = nothing
        play_dict[Symbol("p$(i)_id")] = nothing
    end
end

function parse_event(event::Dict{String, Any})::Dict{Symbol, Any}
    play = Dict{Symbol, Any}()
    play[:event_id] = event["about"]["eventIdx"]
    play[:period] = event["about"]["period"]
    play[:event] = event_type(uppercase(event["result"]["eventTypeId"]))
    play[:time_elapsed] = convert_to_seconds(event["about"]["periodTime"])

    # If an event occured on the play, then that means an event occured on the play
    if "players" in keys(event)
        for i in eachindex(event["players"])
            if event["players"][i]["playerType"] !== "Goalie"
                play[Symbol("p$(i)_name")] = uppercase(event["players"][i]["player"]["fullName"])
                play[Symbol("p$(i)_id")] = string(event["players"][i]["player"]["id"])
            end
        end
    end

    # All rows need to have the same exact number of columns. So this just zeros
    # them out if they don't exist (or if the event only has one player for instance)
    create_missing_columns(play, event)

    try
        play[:xC] = event["coordinates"]["x"]
        play[:yC] = event["coordinates"]["y"]
    catch
        play[:xC] = nothing
        play[:yC] = nothing
    end

    return play
end

function parse_json(response::Dict{String, Any})::DataFrame
    plays = response["liveData"]["plays"]["allPlays"]
    events = [parse_event(play) for play in plays]
    # Filter out events that are not common with the HTML PBP
    filter!(e -> e[:event] !== nothing, events)
    df = convert_dict_to_dataframe(events)
    sort!(df, (:event_id), rev=(false))
    return df
end

function scrape_game(game_id::String)
    response = get_pbp(game_id)
    return parse_json(response)
end

end
# ###############################
#
# response = get_pbp("foobar")
# println(get_teams(response))
#
# x = parse_json(response)


text:
for P in (:Year, :Month, :Week, :Day, :Hour, :Minute, :Second,
          :Millisecond, :Microsecond, :Nanosecond)
  @eval begin
     function $P(x::CompoundPeriod)
         typs = typesof(x)
         idx = findall(typs .=== $P)
         isempty(idx) && return $P(0)
         return x.periods[idx[1]]
     end
  end
end

for (P,Q) in ((:Year, :year), (:Month, :month), (:Week, :week), (:Day, :day),
              (:Hour, :hour), (:Minute, :minute), (:Second, :second),
              (:Millisecond, :millisecond), (:Microsecond, :microsecond),
              (:Nanosecond, :nanosecond))
  @eval begin
     function $Q(x::CompoundPeriod)
         typs = typesof(x)
         idx = findall(typs .=== $P)
         isempty(idx) && return 0
         return x.periods[idx[1]].value
     end
  end
end

convert(::Type{Year}, x::T) where {T<:Diurnal} = Year(0)
convert(::Type{Month}, x::T) where {T<:Diurnal} = Month(0)
convert(::Type{T}, x::Year) where {T<:Diurnal} = T(0)
convert(::Type{T}, x::Month) where {T<:Diurnal} = T(0)


text:
"""
    abstract type AbstractQuadratureRule{D<:AbstractReferenceShape}

A quadrature rule for integrating a function over the domain `D`.

An instance `q` of `AbstractQuadratureRule{D}` is expected to implement the
following methods:

- `q()` : return the nodes `x` and weights `w` of the quadrature rule on the
  reference domain `D`. For performance reasons, the result should depend only
  on the type of `q`.
"""
abstract type AbstractQuadratureRule{D} end

domain(q::AbstractQuadratureRule{D}) where {D} = D()

"""
    qnodes(Y)

Return the quadrature nodes associated with `Y`.
"""
qnodes(q::AbstractQuadratureRule) = q()[1]

"""
    qweights(Y)

Return the quadrature weights associated with `Y`.
"""
qweights(q::AbstractQuadratureRule) = q()[2]

"""
    (q::AbstractQuadratureRule)()

Return the quadrature nodes `x` and weights `w` on the `domain(q)`.
"""
function (q::AbstractQuadratureRule)() end

"""
    integrate(f,q::AbstractQuadrature)
    integrate(f,x,w)

Integrate the function `f` using the quadrature rule `q`. This is simply
`sum(f.(x) .* w)`, where `x` and `w` are the quadrature nodes and weights, respectively.
"""
function integrate(f,q::AbstractQuadratureRule)
    x,w = q()
    if domain(q) == ReferenceLine()
        return integrate(x->f(x[1]),x,w)
    else
        return integrate(f,x,w)
    end
end

function integrate(f,x,w)
    sum(zip(x,w)) do (x,w)
        f(x)*prod(w)
    end
end

## Define some one-dimensional quadrature rules

"""
    struct Trapezoidal{N} <: AbstractQuadratureRule{ReferenceLine}

Closed `N`-point trapezoidal rule for integrating a function over the interval `[0,1]`.

# Examples:
```julia
q    = Trapezoidal(10)
f(x) = exp(x)*cos(x)
integrate(f,q)
```
"""
struct Trapezoidal{N} <: AbstractQuadratureRule{ReferenceLine} end

Trapezoidal(n::Int) = Trapezoidal{n}()

function (q::Trapezoidal{N})() where {N}
    h = 2 / (N - 1)
    x = [-1.0 + k * h for k in 0:N - 1]
    w = [h for _ in 1:N]
    w[1]   = h / 2
    w[end] = h / 2
    # convert to static arrays
    xs = svector(i -> SVector(0.5 * (x[i] + 1)), N)
    ws = svector(i -> w[i] / 2, N)
    return xs, ws
end

"""
    struct TrapezoidalOpen{N} <: AbstractQuadratureRule{ReferenceLine}

Open trapezoidal rule.
"""
struct TrapezoidalOpen{N} <: AbstractQuadratureRule{ReferenceLine} end

TrapezoidalOpen(n::Int) = TrapezoidalOpen{n}()

# open trapezoidal rule
function _trapezoidal_open(n)
    h = 1 / n
    x = [(k-0.5) * h for k in 1:n]
    w = [h for _ in 1:n]
    return x, w
end

function (q::TrapezoidalOpen{N})() where {N}
    x, w = _trapezoidal_open(N)
    # convert to static arrays
    xs = SVector{N}(SVector{1}.(x))
    ws = SVector{N}(w)
    return xs, ws
end

"""
    struct Fejer{N}

`N`-point Fejer's first quadrature rule for integrating a function over `[0,1]`.
Exactly integrates all polynomials of degree `≤ N-1`.
"""
struct Fejer{N} <: AbstractQuadratureRule{ReferenceLine} end

Fejer(n::Int) = Fejer{n}()

Fejer(;order::Int) = Fejer(order+1)

# N point fejer quadrature integrates all polynomials up to degree N-1
order(::Fejer{N}) where {N} = N-1
function (q::Fejer{N})() where {N}
    theta = [(2j - 1) * π / (2 * N) for j = 1:N]
    x = -cos.(theta)
    w = zero(x)
    for j in 1:N
        tmp = 0.0
        for l in 1:floor(N / 2)
            tmp += 1 / (4 * l^2 - 1) * cos(2 * l * theta[j])
        end
        w[j] = 2 / N * (1 - 2 * tmp)
    end
    xs = svector(i -> SVector(0.5 * (x[i] + 1)), N)
    ws = svector(i -> w[i] / 2, N)
    return xs, ws
end

"""
    struct Gauss{D,N} <: AbstractQuadratureRule{D}

Tabulated `N`-point symmetric Gauss quadrature rule for integration over `D`.
"""
struct Gauss{D,N} <: AbstractQuadratureRule{D}
    # gauss quadrature should be constructed using the order, and not the number
    # of nodes. This ensures you don't instantiate quadratures which are not
    # tabulated.
    function Gauss(;domain,order)
        if domain isa ReferenceTriangle
            msg = "quadrature of order $order not available for ReferenceTriangle"
            haskey(TRIANGLE_GAUSS_ORDER_TO_NPTS,order) || error(msg)
            n = TRIANGLE_GAUSS_ORDER_TO_NPTS[order]
        elseif domain isa ReferenceTetrahedron
            msg = "quadrature of order $order not available for ReferenceTetrahedron"
            haskey(TETRAHEDRON_GAUSS_ORDER_TO_NPTS,order) || error(msg)
            n = TETRAHEDRON_GAUSS_ORDER_TO_NPTS[order]
        else
            error("Tabulated Gauss quadratures only available for `ReferenceTriangle` or `ReferenceTetrahedron`")
        end
        return new{typeof(domain),n}()
    end
end

function order(q::Gauss{ReferenceTriangle,N}) where {N}
    TRIANGLE_GAUSS_NPTS_TO_ORDER[N]
end

function order(q::Gauss{ReferenceTetrahedron,N}) where {N}
    TETRAHEDRON_GAUSS_NPTS_TO_ORDER[N]
end

@generated function (q::Gauss{D,N})() where {D,N}
    x, w = _get_gauss_qnodes_and_qweights(D, N)
    return :($x, $w)
end

"""
    TensorProductQuadrature{Q}

A tensor-product of one-dimension quadrature rules. Integrates over `[0,1]^d`,
where `d=length(quad)`.

# Examples
```julia
qx = Fejer(10)
qy = TrapezoidalOpen(15)
q  = TensorProductQuadrature(qx,qy)
```
"""
struct TensorProductQuadrature{Q} <: AbstractQuadratureRule{ReferenceSquare}
    quad::Q
end

# FIXME: this is a workaround the need to easily construct a tensor quadrature
# based only on the the types of the quadratures. Useful in generated functions,
# but there is probably a better way
function TensorProductQuadrature{Tuple{Q1,Q2}}() where {Q1,Q2}
    TensorProductQuadrature(Q1(),Q2())
end

function TensorProductQuadrature(q...)
    TensorProductQuadrature(q)
end

# FIXME: the current implementation is rather obscure. How should we handle the
# product quadrature rules in general? Also make this into a generated function.
function (q::TensorProductQuadrature)()
    N       = length(q.quad)
    nodes   = map(q->q()[1],q.quad)
    weights = map(q->q()[2],q.quad)
    nodes_iter   = Iterators.product(nodes...)
    weights_iter = Iterators.product(weights...)
    x = map(x->vcat(x...),nodes_iter)
    w = map(w->prod(w),weights_iter)
    return SArray(x), w
end

# some helper functions
function integration_measure(jac::SMatrix)
    M,N = size(jac)
    if M == N
        # cheaper version when `M=N`
        det(jac) |> abs
    else
        transpose(jac)*jac |> det |> sqrt
    end
end

"""
    qrule_for_reference_shape(ref,order)

Given a `ref`erence shape and a desired quadrature `order`, return
an appropiate quadrature rule.
"""
function qrule_for_reference_shape(ref,order)
    if ref isa ReferenceLine
        return Fejer(;order)
    elseif ref isa ReferenceSquare
        qx = qrule_for_reference_shape(ReferenceLine(),order)
        qy = qx
        return TensorProductQuadrature(qx,qy)
    elseif ref isa ReferenceTriangle
        return Gauss(;domain=ref,order=order)
    elseif ref isa ReferenceTetrahedron
        return Gauss(;domain=ref,order=order)
    else
        error("no appropriate quadrature rule found.")
    end
end

abstract type CompositeQuadratureRule{D} <: AbstractQuadratureRule{D} end

struct GaussKronrod{N} <: CompositeQuadratureRule{ReferenceLine}
end

@generated function (::GaussKronrod{N})() where {N}
    x,w,wg = kronrod(N)
    xk   = [x;reverse(-x[1:end-1])]
    wk   = [w;reverse(w[1:end-1])]
    wg   = [wg;reverse(wg[1:end-1])]
    idxs = 2:2:length(xk)
    xg   = xk[idxs]
    return SVector{2*N+1}(xk),SVector{2*N+1}(wk),SVector{N}(xg),SVector{N}(wg)
end

function integrate(f,q::GaussKronrod{N}) where {N}
    xk,wk,_,wg = q()
    acc1 = 0.0
    acc2 = 0.0
    for n in 1:2N+1
        v    = f(xk[n])
        acc1 += v*wk[n]
        if iseven(n)
            acc2 += v*wg[n>>1]
        end
    end
    E = abs(acc1 - acc2)
    return acc1,E
end

max_stars_repo_name: WaveProp/WavePropBase

max_issues_repo_name: WaveProp/WavePropBase

max_forks_repo_name: WaveProp/WavePropBase


text:
using ModelingToolkit
using ModelingToolkit: sys2bigraph
using DiffEqBase
using Test

# Define some variables
@parameters t L g
@variables x(t) y(t) w(t) z(t) T(t) xˍt(t) yˍt(t)
@derivatives D'~t

eqs2 = [D(D(x)) ~ T*x,
        D(D(y)) ~ T*y - g,
        0 ~ x^2 + y^2 - L^2]
pendulum2 = ODESystem(eqs2, t, [x, y, T], [L, g], name=:pendulum)
lowered_sys = ModelingToolkit.ode_order_lowering(pendulum2)

lowered_eqs = [D(xˍt) ~ T*x,
               D(yˍt) ~ T*y - g,
               0 ~ x^2 + y^2 - L^2,
               D(x) ~ xˍt,
               D(y) ~ yˍt]
@test ODESystem(lowered_eqs, t, [xˍt, yˍt, x, y, T], [L, g]) == lowered_sys
@test isequal(lowered_sys.eqs, lowered_eqs)

# Simple pendulum in cartesian coordinates
eqs = [D(x) ~ w,
       D(y) ~ z,
       D(w) ~ T*x,
       D(z) ~ T*y - g,
       0 ~ x^2 + y^2 - L^2]
pendulum = ODESystem(eqs, t, [x, y, w, z, T], [L, g], name=:pendulum)

edges, vars, vars_asso = sys2bigraph(pendulum)
@test ModelingToolkit.matching(edges, length(vars), vars_asso .== 0) == [0, 0, 0, 0, 1, 2, 3, 4, 0]

edges, assign, vars_asso, eqs_asso = ModelingToolkit.pantelides(pendulum)

@test sort.(edges) == sort.([
 [5, 3],               # 1
 [6, 4],               # 2
 [7, 9, 1],            # 3
 [8, 9, 2],            # 4
 [2, 1],               # 5
 [2, 1, 6, 5],         # 6
 [5, 3, 10, 7],        # 7
 [6, 4, 11, 8],        # 8
 [2, 1, 6, 5, 11, 10], # 9
])
#                  [1, 2, 3, 4, 5,  6,  7,  8,  9, 10,   11]
#                  [x, y, w, z, x', y', w', z', T, x'', y'']
@test vars_asso == [5, 6, 7, 8, 10, 11, 0,  0,  0,  0,    0]
#1: D(x) ~ w
#2: D(y) ~ z
#3: D(w) ~ T*x
#4: D(z) ~ T*y - g
#5: 0 ~ x^2 + y^2 - L^2
# ----
#6: D(5) -> 0 ~ 2xx'+ 2yy'
#7: D(1) -> D(D(x)) ~ D(w)
#8: D(2) -> D(D(y)) ~ D(z)
#9: D(6) -> 0 ~ 2xx'' + 2x'x' + 2yy'' + 2y'y'
#                 [1, 2, 3, 4, 5, 6, 7, 8, 9]
@test eqs_asso == [7, 8, 0, 0, 6, 9, 0, 0, 0]


text:
module FoldsChainRulesTests

import FoldsChainRules

include("utils.jl")
include("test_foldl.jl")
include("test_sum.jl")

end  # module FoldsChainRulesTests


text:
#=
    Badges problem

    See my analysis of the Badges problem written:
    http://www.hakank.org/data_mining/badge_problem.html
    """
    One of the best collections of machine learning data sets includes this "recreational" data set called "Badge Problem". I haven't seen any solutions to the problem anywhere,
    so here is my take.

    [This info is from the datafile:]

    1. Title: ML94/COLT94 Badge Problem

    2. Source Information
    -- Creator: Haym Hirsh, after an idea by Rob Schapire
    -- Donor: Haym Hirsh (hirsh@cs.rutgers.edu)
    -- Date: September, 1994

    3. Past Usage:
        Every pre-registered attendee at the 1994 Machine Learning
        Conference and 1994 Computational Learning Theory Conference received
        a badge labeled with a "+" or "-".  The labeling was due to some
        function known only to the badge generator (Haym Hirsh), and it
        depended only on the attendee's name.  The goal for conference
        attendees was to identify the unknown function used to generate the
        +/- labeling.

    4. Relevant Information:
        Part of the problem in using an automated program to discover the
        unknown target function is to decide how to encode names such that
        the program can be used.  The data below are presented in the form
        of a +/- label followed by the person's name.  It is up to the
        learning-system user to decide how to convert this data into something
        usable by the system (e.g., what attributes to use if your favorite
        learner requires feature-vector data).

    """

    Note: The ARFF file http://www.hakank.org/data_mining/badges_plain.arff
    includes all the preprocessed dataset, but it's more fun to do everything
    in Julia.
    I have saved the converted CSV file to badges.csv for easier handling.

    Here are the (derived) attributes:
        even_odd         {0,1} 	length of name even?
        first_char_vowel {0,1} 	is first character a vowel?
        second_char_vowel {0,1} 	is second character a vowel?
        vowels numeric 	number of vowels in the name
        consonants numeric 	number of consonants
        vowel_consonant_ratio numeric 	the ratio of vowels / consonant (this is not farfetched???)
        spaces numeric 	number of spaces
        dots: number of "." in the name, i.e. name initials
        words: number of words, i.e number of names, including name initials..

        class {1,0} 	the badge labels

    The best attribute is the params[3] (second_char_vowel) which happens to be correct.

    Compared to badges.jl this version use a different training and test set.

    Note: by guessing 1 all the time, we get an accuracy of 71%.

    Some results: A training percentage below about 20% we start to get quite bad
    results, i.e. <= 71%.

    * train pct 0.66

    Num correct: 100.0
    Num incorrect: 0.0
    Accuracy: 1.0
    Confusion matrix:
    2×2 Matrix{Float64}:
    31.0   0.0
    0.0  69.0
    num bad: 0

    Distribution of the categories (training set):
    0.0 : 53 (27.46113989637306%)
    1.0 : 140 (72.53886010362694%)

    * train_pct 0.33
    Num correct: 186.0
    Num incorrect: 10.0
    Accuracy: 0.9489795918367347
    Confusion matrix:
    2×2 Matrix{Float64}:
    49.0   10.0
    0.0  137.0
    num bad: 10

    Distribution of the categories (training set):
    0.0 : 25 (25.77319587628866%)
    1.0 : 72 (74.22680412371135%)

    * traininc_pct 0.10
    Num correct: 195.0
    Num incorrect: 69.0
    Accuracy: 0.7386363636363636
    Confusion matrix:
    2×2 Matrix{Float64}:
     9.0   69.0
     0.0  186.0
    num bad: 69

    Distribution of the categories (training set):
    0.0 : 6 (20.689655172413794%)
    1.0 : 23 (79.3103448275862%)

=#
using Turing, StatsPlots, Distributions
using LinearAlgebra
using DataFrames
using HTTP, CSV
# pyplot()
include("jl_utils.jl")

#=
# Read the dataset (from my site)
# Saved as a CSV file for easier handling in this program.
badges_url = "http://www.hakank.org/data_mining/badges.data"
dataset = CSV.read(Download.download(badges_url))
CSV.write("badges.csv",dataset)
=#

function convert_name(str)
  xcat,name = String.(split(str," ",limit=2))
  len = length(name)
  even_len = len % 2 == 0 ? 1 : 0 # Is length even?
  first_char_vowel = name[1] in vowels ? 1 : 0 # is first char in name a vowel?
  second_char_vowel = name[2] in vowels ? 1 : 0 # is second char in name a vowel?
  num_vowels = filter(c -> c in vowels, name) |> length # number of vowels
  num_consonants = filter(c -> !(c in vowels), name) |> length # number of consonants
  vowel_consonant_ratio = num_vowels / num_consonants > 0.5 ? 1 : 0  # vowels / consonants
  num_spaces = filter(c -> c == " ", name) |> length # number of spaces
  num_dots = filter(c -> c == ".", name) |> length # number of dots
  num_words = split(name, " ") |> length
          #    1        2                3                 4          5              6                     7          8        9
  return xcat, [even_len first_char_vowel second_char_vowel num_vowels num_consonants vowel_consonant_ratio num_spaces num_dots num_words]

end

@model function badges_model(x,y=missing, ::Type{T} = Float64) where {T}
    n,p = size(x)
    if y === missing
      y = Vector{T}(undef, n)
    end

    a ~ Uniform(0,1)
    params ~ filldist(Uniform(0,1),p)
    sigma ~ Uniform(0,1)

    ## Very slow, and give bad rhats
    # for i in 1:n
    #     mu = a + dot(x[i,:], params)
    #     # mu = a .+ x[i,:]' * params
    #     # y[i] ~ Normal(mu,sigma) # very slowe, bad rhat
    #     y[i] ~ Logistic(mu,sigma) # very slow, nice rhat
    # end

    # Better, much faster and neater
    mu = a .+ x*params
    y ~ MvNormal(mu, 1)

    # Return the classes for prediction.
    return y

end

df = CSV.read("badges.csv",DataFrame)
vowels = "aeiou" # note: y is not a vowel
# println(data)
x = []
y = []
for d in eachrow(df)
    cat, v = convert_name(d[1])
    push!(x,v)
    if cat == "+"
        push!(y,1)
    else
        push!(y,0)
    end
end
y = 1.0*y
x = 1.0 .* x
x = vcat(x...)
num_rows, num_cols = size(x)
println("num_rows: $num_rows num_cols: $num_cols")

println("\nDistribution of the categories (full dataset):")
for (k,v) in make_hash(y)
  println("$k : $v ($(100*v/num_rows)%)")
end

println("\n3 random instances of x:")
display(x[rand(1:size(x)[1],3),:])

# Training percentage
# train_pct = 0.99
# train_pct = 0.75
train_pct = 0.66
# train_pct = 0.33
# train_pct = 0.20
# train_pct = 0.15
# train_pct = 0.10
# train_pct = 0.05 # 15 instances
# train_pct = 10/num_rows
# train_pct = 2/num_rows

x_train, x_test, y_train, y_test = split_train_test(x,y,train_pct)

num_train_rows = length(y_train)
num_test_rows = length(y_test)

println("Num training instances: $num_train_rows")
println("Num test instances: $num_test_rows")

conv = Dict(0=>1,1=>2)
println("\nMean value for each categories (training set): ", keys(conv))
display(keys(conv) .|> s -> (s,mean(y_train .==s, dims=1)))
println("\n")

println("\nDistribution of the categories (training set):")
for (k,v) in make_hash(y_train)
  println("$k : $v ($(100*v/num_train_rows)%)")
end

#
# Train the model
#

model = badges_model(x_train,y_train)

println("\nModel:")
# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 1_000)
# chns = sample(model, PG(5), 1_000)
# chns = sample(model, SMC(), 1_000)
# chns = sample(model, IS(), 1_000)

# chns = sample(model, HMC(0.1,6), 1_000)
chns = sample(model, NUTS(), 1_000)

display(chns)
# display(plot(chns))

println("\nPredictions:")

mpred = badges_model(x_test,missing)
pred = predict(mpred,chns)
display(pred)
println()

cats = Dict(0=>1,1=>2)
confusion, bads = confusion_matrix(y_test,pred,cats,false)
println("Confusion matrix:")
display(confusion)
# println("\nBad classfied:")
# display(bads)
println("num bad: $(length(bads))")

println("\nDistribution of the categories (training set):")
for (k,v) in make_hash(y_train)
  println("$k : $v ($(100*v/num_train_rows)%)")
end

max_stars_repo_name: tias/hakank

max_issues_repo_name: tias/hakank

max_forks_repo_name: tias/hakank


text:
module Db

using LibPQ, Tables, DotEnv
using ..MangroveBase: ReviewInfo, ReviewContent, Reviews

DotEnv.config(path = "../.env")
const ENV_CONN = get(ENV, "MANGROVE_DATABASE", nothing)

function mangrove_reviews(conn_string::String = ENV_CONN)::Reviews
    conn = LibPQ.Connection(conn_string)
    result = execute(conn, "SELECT * FROM reviews")
    data = columntable(result)
    close(conn)
    Dict(
        ReviewInfo(row.sub, row.kid) => ReviewContent(row.rating, row.opinion)
        for row in Tables.rows(data) if !ismissing(row.rating)
    )
end

struct DbInput
    subs::Vector{String}
    qualities::Vector{Int16}
    kids::Vector{String}
    neutralities::Vector{Float32}
end

function insert(conn_string::String, in::DbInput)
    conn = LibPQ.Connection(conn_string)
    execute(conn, "BEGIN;")
    LibPQ.load!(
        (sub = in.subs, quality = in.qualities),
        conn,
        """
        INSERT INTO subjects (sub, quality) VALUES (\$1, \$2)
        ON CONFLICT (sub) DO UPDATE
            SET sub = excluded.sub,
                quality = excluded.quality;
        """,
    )
    LibPQ.load!(
        (pem = in.kids, neutrality = in.neutralities),
        conn,
        """
        INSERT INTO reviewers (pem, neutrality) VALUES (\$1, \$2)
        ON CONFLICT (pem) DO UPDATE
            SET pem = excluded.pem,
                neutrality = excluded.neutrality;
        """,
    )
    execute(conn, "COMMIT;")
    close(conn)
end

end

max_stars_repo_name: Open-Reviews/Mangrove

max_issues_repo_name: Open-Reviews/Mangrove

max_forks_repo_name: Open-Reviews/Mangrove


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: cloud9
using AWS.Compat
using AWS.UUIDs

"""
    create_environment_ec2(instance_type, name)
    create_environment_ec2(instance_type, name, params::Dict{String,<:Any})

Creates an Cloud9 development environment, launches an Amazon Elastic Compute Cloud (Amazon
EC2) instance, and then connects from the instance to the environment.

# Arguments
- `instance_type`: The type of instance to connect to the environment (for example,
  t2.micro).
- `name`: The name of the environment to create. This name is visible to other IAM users in
  the same Amazon Web Services account.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"automaticStopTimeMinutes"`: The number of minutes until the running instance is shut
  down after the environment has last been used.
- `"clientRequestToken"`: A unique, case-sensitive string that helps Cloud9 to ensure this
  operation completes no more than one time. For more information, see Client Tokens in the
  Amazon EC2 API Reference.
- `"connectionType"`: The connection type used for connecting to an Amazon EC2 environment.
  Valid values are CONNECT_SSH (default) and CONNECT_SSM (connected through Amazon EC2
  Systems Manager). For more information, see Accessing no-ingress EC2 instances with Amazon
  EC2 Systems Manager in the Cloud9 User Guide.
- `"description"`: The description of the environment to create.
- `"dryRun"`: Checks whether you have the required permissions for the action, without
  actually making the request, and provides an error response. If you have the required
  permissions, the error response is DryRunOperation. Otherwise, it is UnauthorizedOperation.
- `"imageId"`: The identifier for the Amazon Machine Image (AMI) that's used to create the
  EC2 instance. To choose an AMI for the instance, you must specify a valid AMI alias or a
  valid Amazon EC2 Systems Manager (SSM) path. The default AMI is used if the parameter isn't
  explicitly assigned a value in the request. Because Amazon Linux AMI has ended standard
  support as of December 31, 2020, we recommend you choose Amazon Linux 2, which includes
  long term support through 2023.  AMI aliases      Amazon Linux (default):
  amazonlinux-1-x86_64     Amazon Linux 2: amazonlinux-2-x86_64    Ubuntu 18.04:
  ubuntu-18.04-x86_64     SSM paths     Amazon Linux (default):
  resolve:ssm:/aws/service/cloud9/amis/amazonlinux-1-x86_64     Amazon Linux 2:
  resolve:ssm:/aws/service/cloud9/amis/amazonlinux-2-x86_64    Ubuntu 18.04:
  resolve:ssm:/aws/service/cloud9/amis/ubuntu-18.04-x86_64
- `"ownerArn"`: The Amazon Resource Name (ARN) of the environment owner. This ARN can be
  the ARN of any IAM principal. If this value is not specified, the ARN defaults to this
  environment's creator.
- `"subnetId"`: The ID of the subnet in Amazon VPC that Cloud9 will use to communicate with
  the Amazon EC2 instance.
- `"tags"`: An array of key-value pairs that will be associated with the new Cloud9
  development environment.
"""
function create_environment_ec2(
    instanceType, name; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "CreateEnvironmentEC2",
        Dict{String,Any}("instanceType" => instanceType, "name" => name);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_environment_ec2(
    instanceType,
    name,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "CreateEnvironmentEC2",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("instanceType" => instanceType, "name" => name),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_environment_membership(environment_id, permissions, user_arn)
    create_environment_membership(environment_id, permissions, user_arn, params::Dict{String,<:Any})

Adds an environment member to an Cloud9 development environment.

# Arguments
- `environment_id`: The ID of the environment that contains the environment member you want
  to add.
- `permissions`: The type of environment member permissions you want to associate with this
  environment member. Available values include:    read-only: Has read-only access to the
  environment.    read-write: Has read-write access to the environment.
- `user_arn`: The Amazon Resource Name (ARN) of the environment member you want to add.

"""
function create_environment_membership(
    environmentId, permissions, userArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "CreateEnvironmentMembership",
        Dict{String,Any}(
            "environmentId" => environmentId,
            "permissions" => permissions,
            "userArn" => userArn,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_environment_membership(
    environmentId,
    permissions,
    userArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "CreateEnvironmentMembership",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "environmentId" => environmentId,
                    "permissions" => permissions,
                    "userArn" => userArn,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_environment(environment_id)
    delete_environment(environment_id, params::Dict{String,<:Any})

Deletes an Cloud9 development environment. If an Amazon EC2 instance is connected to the
environment, also terminates the instance.

# Arguments
- `environment_id`: The ID of the environment to delete.

"""
function delete_environment(
    environmentId; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DeleteEnvironment",
        Dict{String,Any}("environmentId" => environmentId);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_environment(
    environmentId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "DeleteEnvironment",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("environmentId" => environmentId), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_environment_membership(environment_id, user_arn)
    delete_environment_membership(environment_id, user_arn, params::Dict{String,<:Any})

Deletes an environment member from an Cloud9 development environment.

# Arguments
- `environment_id`: The ID of the environment to delete the environment member from.
- `user_arn`: The Amazon Resource Name (ARN) of the environment member to delete from the
  environment.

"""
function delete_environment_membership(
    environmentId, userArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DeleteEnvironmentMembership",
        Dict{String,Any}("environmentId" => environmentId, "userArn" => userArn);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_environment_membership(
    environmentId,
    userArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "DeleteEnvironmentMembership",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("environmentId" => environmentId, "userArn" => userArn),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_environment_memberships()
    describe_environment_memberships(params::Dict{String,<:Any})

Gets information about environment members for an Cloud9 development environment.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"environmentId"`: The ID of the environment to get environment member information about.
- `"maxResults"`: The maximum number of environment members to get information about.
- `"nextToken"`: During a previous call, if there are more than 25 items in the list, only
  the first 25 items are returned, along with a unique string called a next token. To get the
  next batch of items in the list, call this operation again, adding the next token to the
  call. To get all of the items in the list, keep calling this operation with each subsequent
  next token that is returned, until no more next tokens are returned.
- `"permissions"`: The type of environment member permissions to get information about.
  Available values include:    owner: Owns the environment.    read-only: Has read-only
  access to the environment.    read-write: Has read-write access to the environment.   If no
  value is specified, information about all environment members are returned.
- `"userArn"`: The Amazon Resource Name (ARN) of an individual environment member to get
  information about. If no value is specified, information about all environment members are
  returned.
"""
function describe_environment_memberships(;
    aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DescribeEnvironmentMemberships";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_environment_memberships(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DescribeEnvironmentMemberships",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_environment_status(environment_id)
    describe_environment_status(environment_id, params::Dict{String,<:Any})

Gets status information for an Cloud9 development environment.

# Arguments
- `environment_id`: The ID of the environment to get status information about.

"""
function describe_environment_status(
    environmentId; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DescribeEnvironmentStatus",
        Dict{String,Any}("environmentId" => environmentId);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_environment_status(
    environmentId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "DescribeEnvironmentStatus",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("environmentId" => environmentId), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_environments(environment_ids)
    describe_environments(environment_ids, params::Dict{String,<:Any})

Gets information about Cloud9 development environments.

# Arguments
- `environment_ids`: The IDs of individual environments to get information about.

"""
function describe_environments(
    environmentIds; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "DescribeEnvironments",
        Dict{String,Any}("environmentIds" => environmentIds);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_environments(
    environmentIds,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "DescribeEnvironments",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("environmentIds" => environmentIds), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_environments()
    list_environments(params::Dict{String,<:Any})

Gets a list of Cloud9 development environment identifiers.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of environments to get identifiers for.
- `"nextToken"`: During a previous call, if there are more than 25 items in the list, only
  the first 25 items are returned, along with a unique string called a next token. To get the
  next batch of items in the list, call this operation again, adding the next token to the
  call. To get all of the items in the list, keep calling this operation with each subsequent
  next token that is returned, until no more next tokens are returned.
"""
function list_environments(; aws_config::AbstractAWSConfig=global_aws_config())
    return cloud9(
        "ListEnvironments"; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end
function list_environments(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "ListEnvironments", params; aws_config=aws_config, feature_set=SERVICE_FEATURE_SET
    )
end

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Gets a list of the tags associated with an Cloud9 development environment.

# Arguments
- `resource_arn`: The Amazon Resource Name (ARN) of the Cloud9 development environment to
  get the tags for.

"""
function list_tags_for_resource(
    ResourceARN; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "ListTagsForResource",
        Dict{String,Any}("ResourceARN" => ResourceARN);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_tags_for_resource(
    ResourceARN,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "ListTagsForResource",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("ResourceARN" => ResourceARN), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Adds tags to an Cloud9 development environment.  Tags that you add to an Cloud9 environment
by using this method will NOT be automatically propagated to underlying resources.

# Arguments
- `resource_arn`: The Amazon Resource Name (ARN) of the Cloud9 development environment to
  add tags to.
- `tags`: The list of tags to add to the given Cloud9 development environment.

"""
function tag_resource(ResourceARN, Tags; aws_config::AbstractAWSConfig=global_aws_config())
    return cloud9(
        "TagResource",
        Dict{String,Any}("ResourceARN" => ResourceARN, "Tags" => Tags);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function tag_resource(
    ResourceARN,
    Tags,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "TagResource",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("ResourceARN" => ResourceARN, "Tags" => Tags),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    untag_resource(resource_arn, tag_keys)
    untag_resource(resource_arn, tag_keys, params::Dict{String,<:Any})

Removes tags from an Cloud9 development environment.

# Arguments
- `resource_arn`: The Amazon Resource Name (ARN) of the Cloud9 development environment to
  remove tags from.
- `tag_keys`: The tag names of the tags to remove from the given Cloud9 development
  environment.

"""
function untag_resource(
    ResourceARN, TagKeys; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "UntagResource",
        Dict{String,Any}("ResourceARN" => ResourceARN, "TagKeys" => TagKeys);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function untag_resource(
    ResourceARN,
    TagKeys,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "UntagResource",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}("ResourceARN" => ResourceARN, "TagKeys" => TagKeys),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_environment(environment_id)
    update_environment(environment_id, params::Dict{String,<:Any})

Changes the settings of an existing Cloud9 development environment.

# Arguments
- `environment_id`: The ID of the environment to change settings.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"description"`: Any new or replacement description for the environment.
- `"managedCredentialsAction"`: Allows the environment owner to turn on or turn off the
  Amazon Web Services managed temporary credentials for an Cloud9 environment by using one of
  the following values:    ENABLE     DISABLE     Only the environment owner can change the
  status of managed temporary credentials. An AccessDeniedException is thrown if an attempt
  to turn on or turn off managed temporary credentials is made by an account that's not the
  environment owner.
- `"name"`: A replacement name for the environment.
"""
function update_environment(
    environmentId; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "UpdateEnvironment",
        Dict{String,Any}("environmentId" => environmentId);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_environment(
    environmentId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "UpdateEnvironment",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("environmentId" => environmentId), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    update_environment_membership(environment_id, permissions, user_arn)
    update_environment_membership(environment_id, permissions, user_arn, params::Dict{String,<:Any})

Changes the settings of an existing environment member for an Cloud9 development
environment.

# Arguments
- `environment_id`: The ID of the environment for the environment member whose settings you
  want to change.
- `permissions`: The replacement type of environment member permissions you want to
  associate with this environment member. Available values include:    read-only: Has
  read-only access to the environment.    read-write: Has read-write access to the
  environment.
- `user_arn`: The Amazon Resource Name (ARN) of the environment member whose settings you
  want to change.

"""
function update_environment_membership(
    environmentId, permissions, userArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return cloud9(
        "UpdateEnvironmentMembership",
        Dict{String,Any}(
            "environmentId" => environmentId,
            "permissions" => permissions,
            "userArn" => userArn,
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function update_environment_membership(
    environmentId,
    permissions,
    userArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return cloud9(
        "UpdateEnvironmentMembership",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "environmentId" => environmentId,
                    "permissions" => permissions,
                    "userArn" => userArn,
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end


text:
using Cascadia
using Gumbo
using HTTP
using Serialization
using SQLite
using ConfParser
import Mongoc
#using AbstractTrees
using DataFrames

println("---")

mutable struct Person
    id::String
    name::String
    birthday::String
    movie_ids::Set{String}
    job_categories::Set{String}
    Person() = new()
end

function Base.:(==)(x::Person, y::Person)
    (x.name == y.name)
end

function Base.hash(x::Person, h::UInt )
    hash(x.name)
end

mutable struct Cast
    movie_id::String
    director::Person
    writers::Set{Person}
    actors::Dict{Int,Person}   # casting_rank::Int  => actor::Person     #Set{Person}
    producers::Set{Person}
    Cast() = new()
end
function Base.:(==)(x::Cast, y::Cast)
    (x.movie_id == y.movie_id)
end

function Base.hash(x::Cast, h::UInt )
    hash(x.movie_id * "Cast")
end

function Base.show(io::IO, cast::Cast)
    for person in values(cast.actors)
        print(io, person.name, "|")
    end
end

mutable struct Movie
    id::String
    title::String
    year::String
    rating::String
    genre::String
    time::String
    cast::Cast
    Movie() = new()
end
function Base.:(==)(x::Movie, y::Movie)
    (x.id == y.id)
end

function Base.hash(x::Movie, h::UInt )
    hash(x.id)
end

function Base.show(io::IO, m::Movie)
    print(io, m.id)
    try
        print(io, " => ", m.title, " (", m.year, ")", " [", m.genre, "] ", m.rating )
        println(io,"")
        print(io, "    \\_: ", m.cast)
    catch
    end
end

function get_person_ids(movie::Movie)
    persons = []
    cast = movie.cast

    persons = union(persons, [cast.director])
    persons = union(persons, cast.writers)
    persons = union(persons, values(cast.actors))
    persons = union(persons, cast.producers)
    map( p -> p.id , persons)
end

function getMovieIdFromImdbUrl(url::String)
    string(match(r"tt\d*",url).match)
end

function getPersonIdFromImdbUrl(url::String)
    match(r"nm\d*",url).match
end

function getUrlMainFromMovieId(id::String)
    "https://www.imdb.com/title/$id"
end

function getUrlCastFromMovieId(id::String)
    "https://www.imdb.com/title/$id/fullcredits/"
end

function scrape_movie(id::String)
    print("scrape_movie($id)-")
    movie = Movie()
    cast = Cast()
    movie.id = id
    cast.movie_id = id

    url = getUrlMainFromMovieId(id)
    doc = parsehtml(String(HTTP.get(url).body))
    head = doc.root[1]
    body = doc.root[2]

    print("Main-")

    # movie
    movie.title = strip(eachmatch(sel".title_wrapper > h1", body)[1][1].text)
    try
        movie.year =  eachmatch(sel".title_wrapper > h1", body)[1][2][2][1].text
    catch
        movie.year = "n/a"
    end
    try
        movie.rating = eachmatch(sel"[itemprop=ratingValue]", body)[1][1].text
    catch
        movie.rating ="n/a"
    end
    movie.genre =  strip(eachmatch(sel".subtext", body)[1][5][1].text)
    try
        movie.time = strip(eachmatch(sel"[datetime]", body)[1][1].text)
    catch
        movie.time = "n/a"
    end

    url = getUrlCastFromMovieId(id)
    doc = parsehtml(String(HTTP.get(url).body))
    body = doc.root[2]

    print("Director-")
    # director
    fullcredits_content = eachmatch(sel"#fullcredits_content", body)
    director_name = strip(fullcredits_content[1][2][2][1][1][1][1].text)
    director_id = getPersonIdFromImdbUrl(fullcredits_content[1][2][2][1][1][1].attributes["href"])
    cast_director = Person()
    cast_director.name = director_name
    cast_director.id = director_id

    print("Writers-")
    # writers
    cast_writers = Set{Person}()

    try
        for i in 1:size(fullcredits_content[1][4][2].children)[1]
            try
                writer_name = strip(fullcredits_content[1][4][2][i][1][1][1].text)
                writer_id = getPersonIdFromImdbUrl(fullcredits_content[1][4][2][1][1][1].attributes["href"])
                writer = Person()
                writer.name = writer_name
                writer.id = writer_id
                push!(cast_writers, writer)
            catch e
                println("*skipping an entry for writers")
            end
        end
    catch
        println("*writers not found")
    end

    print("Actors-")
    # actors
    table_rows  = eachmatch(sel".cast_list", body)[1].children[1]
    cast_actors = Dict{Int,Person}()    # cast_rank::Int => actor::Person
    s = size(table_rows.children)[1]
    cast_rank = 1
    for i in 2:s
        try
            actor = Person()
            actor.name = strip(table_rows[i][2][1][1].text)
            actor.id = getPersonIdFromImdbUrl(table_rows[i][2][1].attributes["href"])
            push!(cast_actors, cast_rank => actor)
            cast_rank +=1
        catch e
            println("*skipping an entry in actor casting")
        end
    end

    println("Producers|")
    #producers
    cast_producers = Set{Person}()
    try
        for i in 1:size(fullcredits_content[1][8][2].children)[1]
            try
                producer_name = strip(fullcredits_content[1][8][2][i][1][1][1].text)
                producer_id = getPersonIdFromImdbUrl(fullcredits_content[1][8][2][i][1][1].attributes["href"])
                producer = Person()
                producer.name = producer_name
                producer.id = producer_id
                push!(cast_producers, producer)
            catch e
                println("*skipping an entry for producers")
            end
        end
    catch
        println("*no producers found")
    end

    cast.director = cast_director
    cast.producers = cast_producers
    cast.writers = cast_writers
    cast.actors = cast_actors
    movie.cast = cast
    movie
end

function scrape_movies(movie_ids) ::Dict{String,Movie}
    movie_ids_to_scrape = copy(movie_ids)
    movie_ids_stored = []
    count = 1
    result = Dict{String,Movie}()

    while length(movie_ids_to_scrape)!=0
        for id in movie_ids_to_scrape
            try
                println("count : $count")
                movie = scrape_movie(id)
                push!(result, movie.id => movie)
                push!(movie_ids_stored, movie.id)
                count +=1
                #sleep(1.2)  # let's be gentle with the server
            catch e
                println("**** scraping error for $id")
                println(string(e) * " => @line ~225")
            end
        end

        movie_ids_to_scrape=[]
        for id in movie_ids
            if !(id in movie_ids_stored)
                push!(movie_ids_to_scrape, id)
            end
        end
    end
    result
end

function scrape_person(id)
    print("scrape_person($id)-")
    person = Person()
    person.id = id
    url="https://www.imdb.com/name/$id/"
    body = parsehtml(String(HTTP.get(url).body)).root[2]

    # name
    print("name-")
    name_widget = eachmatch(sel"#name-overview-widget", body)
    name = "n/a"
    try
        name = name_widget[1][1][1][1][1][1][1][1].text
    catch
        try
           name = name_widget[1][1][1][1][2][2][1][1].text
        catch
            print("*name not found*")
        end
    end
    person.name = name

    # birthday
    print("birthday-")
    #overview_top = eachmatch(sel"#overview-top", body)
    born_info = eachmatch(sel"#name-born-info > time", body)
    birthday = "n/a"
    try
        birthday = born_info[1].attributes["datetime"]
    catch
        print("*birthday not found*")
    end

    person.birthday = birthday

    # movies
    print("movies-")
    filmo = eachmatch(sel".filmo-category-section", body)
    movie_ids = Set{String}()
    for i in 1:size(filmo[1].children)[1]
        try
            movie_id = getMovieIdFromImdbUrl(filmo[1][i][2][1].attributes["href"])
            push!(movie_ids, movie_id)
        catch
            println("skipped an entry for movies")
        end
    end

    # job_categories ( "actor" | "producer" | "writer" ...)
    print("job_cat-")
    job_categories_html = eachmatch(sel"#name-job-categories", body)
    job_categories = Set{String}()
    for i in 1:size(job_categories_html[1].children)[1]
        try
            job_category = lowercase(strip(job_categories_html[1][i][1][1].text))
            push!(job_categories, job_category)
        catch
            print("*skipped an entry for job_categories*")
        end
    end
    println("\n|> $(person.name)\n")
    person.movie_ids = movie_ids
    person.job_categories = job_categories
    person
end

# return an array of movie ids from the imdb top 250
function scrape_top250()
    url = "https://www.imdb.com/chart/top"
    doc = parsehtml(String(HTTP.get(url).body))
    body = doc.root[2]
    movie_ids = []
    elem_array = eachmatch(sel"tbody.lister-list > tr > td.titleColumn > a", body)
    for e in elem_array
        push!(movie_ids, getMovieIdFromImdbUrl(e.attributes["href"]) )
    end
    movie_ids
end

function test_scrape(id::String)
    url = "https://www.imdb.com/title/$id/fullcredits/"
    doc = parsehtml(String(HTTP.get(url).body))
    body = doc.root[2]

    #fullcredits_content = eachmatch(sel"#fullcredits_content", body)
    fullcredits_content = eachmatch(sel"#fullcredits_content", body)
    writers_table = eachmatch(sel"div.header > h4#writer", body)
end

function db_movies_all_ids(collection, criteria::Dict=Dict()) :: Vector{String}
    result = Vector{String}()
    let
        bson_filter = Mongoc.BSON(criteria)
        bson_options = Mongoc.BSON("""
                {
                    "projection" :
                        {
                            "id" : 1,
                            "title" : 1,
                            "year" : 1
                        },
                    "sort" :
                        {
                            "year" : 1
                        }
                }
            """)
        for bson_document in Mongoc.find(collection, bson_filter, options=bson_options)
            str = bson_document["id"]
            push!(result, str)
        end
    end
    return result
end

function db_persons_all_ids(collection, criteria::Dict=Dict()) ::Vector{String}
    result = Vector{String}()
    let
        bson_filter = Mongoc.BSON(criteria)
        bson_options = Mongoc.BSON("""
                {
                    "projection" :
                        {
                            "id" : 1
                        }
                }
            """)
        for bson_document in Mongoc.find(collection, bson_filter, options=bson_options)
            str = bson_document["id"]
            push!(result, str)
        end
    end
    return result
end

function db_movie_save(collection, movie::Movie)
    try
        writers_ids = []
        for p in movie.cast.writers
            push!(writers_ids, p.id)
        end

        actors_ids = Dict{String,String}()
        for p in movie.cast.actors
            push!(actors_ids, string(p[1]) => p[2].id )
        end

        producers_ids = []
        for p in movie.cast.producers
            push!(producers_ids, p.id)
        end

        document2 = Mongoc.BSON(Dict(
            "id" => movie.id,
            "title" => movie.title,
            "year" => movie.year,
            "rating" => movie.rating,
            "genre" => movie.genre,
            "time" => movie.time,
            "cast" => Dict(
                "director" => movie.cast.director.id,
                "writers"  => writers_ids,
                "actors"   => actors_ids,
                "producers"=> producers_ids
            )
        ))
        push!(collection, document2)
    catch e
        println(string(e) * " => function movie_save() @line ~360" )
    end
end

function db_movies_save(collection, movies::Dict{String,Movie} )
        movie_ids = db_movies_all_ids(collection)
        for id in keys(movies)
            if !(id in movie_ids)
                movie = movies[id]
                db_movie_save(collection, movie)
            end
        end
end

function db_person_save(collection, person::Person)
    try
        document = Mongoc.BSON()
        document["id"] = person.id
        document["name"] = person.name
        document["birthday"] = person.birthday
        document["movie_ids"] = collect(person.movie_ids)
        document["job_categories"] = collect(person.job_categories)
        push!(collection, document)
    catch e
        println(string(e) * " => person_save() @line ~425")
    end
end

#==================================================================================================#
#=

                                ███▄ ▄███▓ ▄▄▄       ██▓ ███▄    █
                                ▓██▒▀█▀ ██▒▒████▄    ▓██▒ ██ ▀█   █
                                ▓██    ▓██░▒██  ▀█▄  ▒██▒▓██  ▀█ ██▒
                                ▒██    ▒██ ░██▄▄▄▄██ ░██░▓██▒  ▐▌██▒
                                ▒██▒   ░██▒ ▓█   ▓██▒░██░▒██░   ▓██░
                                ░ ▒░   ░  ░ ▒▒   ▓▒█░░▓  ░ ▒░   ▒ ▒
                                ░  ░      ░  ▒   ▒▒ ░ ▒ ░░ ░░   ░ ▒░
                                ░      ░     ░   ▒    ▒ ░   ░   ░ ░
                                    ░         ░  ░ ░           ░
=#
#==================================================================================================#

const MOVIES_DATA_FILE_PATH = "/julia/resources/movies.serialized"
const PERSONS_DATA_FILE_PATH = "/julia/resources/persons.serialized"
const PERSONS_FAILED_SCRAPE = "/julia/resources/failed-persons.serialized"

# movid1 = "tt0120586" #american history x
# movid2 = "tt0114814" #the usual suspects
# movid3 = "tt0083944" # rambo

#movies = Set{Movie}()
movies = Dict{String,Movie}()       # {movie.id, movie}
persons = Dict{String,Person}()     # {person.id, person}

#===Serialization==================================================================================#

if isfile(MOVIES_DATA_FILE_PATH)
    println("Loading movies data from -> $MOVIES_DATA_FILE_PATH")
    movies = deserialize(MOVIES_DATA_FILE_PATH)
else
    println("No Movies data file found. fetching some movies to create one.")
    movie_ids_to_scrape = scrape_top250()
    movies = scrape_movies(movie_ids_to_scrape)
    serialize(MOVIES_DATA_FILE_PATH, movies)
    println("movies data serialized to $MOVIES_DATA_FILE_PATH")
end

# elem = 0
# for elem in movies
#     movie = elem[2]
#     println("$(movie.id) $(movie.title) ($(movie.year))   $(movie.rating) ")
# end

if isfile(PERSONS_DATA_FILE_PATH)
    println("Loading persons data from -> $PERSONS_DATA_FILE_PATH")
    persons = deserialize(PERSONS_DATA_FILE_PATH)
end

#===MongoDB=========================================================================================#

conf = ConfParse("$(homedir())/.config/imdb-scraper.jl.conf")
parse_conf!(conf)
user     = retrieve(conf, "database", "user")
password = retrieve(conf, "database", "password")
url      = retrieve(conf, "database", "url")

connect_string = "mongodb+srv://$user:$password@$url"
client = Mongoc.Client(connect_string)
db = client["movie_industry"]

# save movies in mongodb
println("inserting movies to db")
collection = db["movies"]
db_movies_save(collection, movies)
println("done.")

# save persons in mongodb
db_collection = db["persons"]
println("fetching some persons.")

person_ids_scrape_failed = []
if isfile(PERSONS_FAILED_SCRAPE)
    println("Loading failed person scrape ids -> $PERSONS_FAILED_SCRAPE")
    person_ids_scrape_failed = deserialize(PERSONS_FAILED_SCRAPE)
end

total_persons = 0
movie_count = 1
person_count = 1
stored_ids = db_persons_all_ids(db_collection)
persons=Dict{String,Person}()
for movie in values(movies)
    person_ids_to_scrape = get_person_ids(movie)
    for id in person_ids_to_scrape
        if !( id in stored_ids) && !(id in person_ids_scrape_failed)
            try
                person = scrape_person(id)
                persons[person.id] = person
                if person.name != "n/a"
                    db_person_save(db_collection, person)
                else
                    push!(person_ids_scrape_failed, id)
                end
                push!(stored_ids, person.id)
            catch
                push!(person_ids_scrape_failed, id)
            end
        end
        #println("person_count = $person_count")
        #println("#failed ids : $(length(person_ids_scrape_failed))")
        person_count += 1
    end
    println("movie_count = $movie_count")
    movie_count += 1
    total_persons = total_persons + person_count - 1
    serialize(PERSONS_FAILED_SCRAPE, person_ids_scrape_failed)
end
println("total_persons : $total_persons")
println("Failed scrape : $(length(person_ids_scrape_failed))")

max_stars_repo_name: leroy-walton/playgrounds

max_issues_repo_name: leroy-walton/playgrounds

max_forks_repo_name: leroy-walton/playgrounds


text:
abstract type F84 <: NASM end

const _πA(mod::F84) = mod.πA
const _πC(mod::F84) = mod.πC
const _πG(mod::F84) = mod.πG
const _πT(mod::F84) = mod.πT

function P(mod::F84, t::Array{Float64})
  return [P(mod, i) for i in t]
end


text:
"""
    Oracle

Each call to the oracle will generate a finite number of columns.

An oracle must return:
- The set of newly generated columns, if any
- The best known lower bound, if applicable
"""
abstract type Oracle end

"""
    update!(o::Oracle, π, σ)

Update objective function with new dual variables
"""
function update! end

"""
    optimize!(o::Oracle)

Solve the sub-problem.
"""
function optimize! end

"""
    get_columns

Return newly generated columns and corresponding reduced costs.
"""
function get_columns end

"""
    get_dual_bound

Get best known dual bound from last call to the oracle. If no dual bound is
    available (e.g. sub-problem solved heuristically), infinity is returned.
"""
function get_dual_bound end

"""
    get_objective_value

Get best objective value of best solution, i.e., the most negative reduced cost.
"""
function get_objective_value end


text:
"""Returns the tuple (F(n), F(n+1))."""
fib_bk(x::Int) = fib_bk(BigInt(x))

function fib_bk(n::BigInt)
	if n == 0
		return (BigInt(0), BigInt(1))
	else
		a, b = fib_bk(div(n,2))
		c = a * (b * BigInt(2) - a)
		d = a * a + b * b
		if iseven(n)
			return (c, d)
		else
			return (d, c + d)
        end
    end
end
print(fib_bk(50))


text:
## -----------------------------------------------------------------------------------------------
function plot_pol_vol_study(simid, Ds, ϵs, cg , ; bins = 150)
    ## -----------------------------------------------------
    # heatmap
    net = lglob(Dyn, :net3D)

    Lz, Uz = Dyn.bounds(net, :z)
    Ds_ = range(Lz, Uz; length = bins)
    Lug, Uug = Dyn.bounds(net, :ug)
    cgD_Xs = range(Lug, Uug; length = bins)

    mat = fill(NaN, bins, bins)

    for (Di, D) in enumerate(Ds_)
        Dyn.fix!(net, :z, D)
        ugl, ugu = Dyn.fva(net, :ug)

        for (cgD_Xi, cgD_X) in enumerate(cgD_Xs)
            !(ugl <= cgD_X <= ugu) && continue
            mat[Di, cgD_Xi] = (cgD_X - ugl)
        end
    end

    p = heatmap(Ds_, cgD_Xs, mat';
        label = "",
        color = :greys,
        title = "",
        xlabel = _textbf("D"),
        ylabel = _textbf("c_g", "D/X"),
        # colorbar_title = _textbf("polytope volume")
    )

    fontsize = 19
    text_ = _textbf("polytope volume")
    plot!(p; rightmargin = 15mm)
    annotate!(p, 0.65, 13.0 / 2, text(text_, fontsize, :black, rotation = 90))

    ## -----------------------------------------------------
    # simulation
    max_ϵs_ = maximum(ϵs)
    ms0 = 5

    @showprogress for (ϵi, ϵ) in enumerate(ϵs)
        zs = []
        ugs = []

        for D in Ds
            simparams = (;D, ϵ, cg)
            status = get_status(simid, simparams)
            (status != STST_SIM_STATUS) && continue
            fn = Dyn.simdat_file(simid, simparams)
            !isfile(fn) && continue
            S = ldat(fn)
            push!(zs, S.z_av)
            push!(ugs, S.ug_av)
        end

        isempty(zs) && continue
        scatter!(p, zs, ugs;
            # label = _textbf("\\epsilon", "=", round(ϵ; sigdigits=3)),
            label = "",
            # m = (9, shape[ϵi]),
            markerstrokecolor = :white,
            color = :black,
            ms = ms0 + (2 * ms0) * (ϵ / max_ϵs_),
            # color = :red
        )
    end
    plot!(p; tite = _textbf("A"), legend = :topleft)

    return p

end


text:
using StatsBase
import StatsBase: transform, reconstruct
using Statistics
using Test

@testset "Transformations" begin
    X = rand(5, 8)

    t = fit(ZScoreTransform, X; center=false, scale=false)
    Y = transform(t, X)
    @test isa(t, AbstractDataTransform)
    @test isempty(t.mean)
    @test isempty(t.scale)
    @test isequal(X, Y)
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    t = fit(ZScoreTransform, X; center=false)
    Y = transform(t, X)
    @test isempty(t.mean)
    @test length(t.scale) == 5
    @test Y ≈ X ./ std(X, dims=2)
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    t = fit(ZScoreTransform, X; scale=false)
    Y = transform(t, X)
    @test length(t.mean) == 5
    @test isempty(t.scale)
    @test Y ≈ X .- mean(X, dims=2)
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    t = fit(ZScoreTransform, X)
    Y = transform(t, X)
    @test length(t.mean) == 5
    @test length(t.scale) == 5
    @test Y ≈ (X .- mean(X, dims=2)) ./ std(X, dims=2)
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    t = fit(UnitRangeTransform, X)
    Y = transform(t, X)
    @test isa(t, AbstractDataTransform)
    @test length(t.min) == 5
    @test length(t.scale) == 5
    @test Y ≈ (X .- minimum(X, dims=2)) ./ (maximum(X, dims=2) .- minimum(X, dims=2))
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    t = fit(UnitRangeTransform, X; unit=false)
    Y = transform(t, X)
    @test length(t.min) == 5
    @test length(t.scale) == 5
    @test Y ≈ X ./ (maximum(X, dims=2) .- minimum(X, dims=2))
    @test transform(t, X[:,1]) ≈ Y[:,1]
    @test reconstruct(t, Y[:,1]) ≈ X[:,1]
    @test reconstruct(t, Y) ≈ X

    @test Y == standardize(UnitRangeTransform, X; unit=false)
end


text:
using Pkg, MPI, DistStat, Random, Test, LinearAlgebra

if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
    using CUDA
    A = CuArray
else
    A = Array
end

type=[Float64,Float32]

for T in type
    N=6; M=5
    X = Array{T}(reshape(collect(1:N*M),N,M)); X_dist=distribute(X)
    XX = Array{T}(reshape(collect(1:M*M),M,M)); XX_dist=distribute(XX)
    y = Array{T}(undef,N,N); y_dist=distribute(y)
    cols1=X_dist.partitioning[DistStat.Rank()+1][2]
    cols2=y_dist.partitioning[DistStat.Rank()+1][2]

    w1 = similar(y); w1=distribute(w1)
    result1=X*transpose(X)

    LinearAlgebra.mul!(w1,X_dist,transpose(X_dist))
    @test isapprox(result1[:,cols2],w1.localarray)

    LinearAlgebra.mul!(transpose(w1), X_dist,transpose(X_dist))
    @test isapprox(result1[:,cols2],w1.localarray)

    w2=similar(XX); w2=distribute(w2)
    result2=transpose(X)*X

    LinearAlgebra.mul!(w2,transpose(X_dist),X_dist)
    @test isapprox(result2[:,cols1],w2.localarray)

    LinearAlgebra.mul!(transpose(w2),transpose(X_dist),X_dist)
    @test isapprox(result2[:,cols1],w2.localarray)

    w3=similar(X); w3=distribute(w3)
    result3=X*XX

    LinearAlgebra.mul!(w3,X_dist,XX_dist)
    @test isapprox(result3[:,cols1],w3.localarray)

    LinearAlgebra.mul!(transpose(w3),transpose(XX_dist),transpose(X_dist))
    @test isapprox(result3[:,cols1],w3.localarray)

    LinearAlgebra.mul!(y,X_dist,transpose(X_dist))

    w4=similar(X); w4=distribute(w4)
    result4=y*X

    LinearAlgebra.mul!(w4,y,X_dist)
    @test isapprox(result4[:,cols1],w4.localarray)

    LinearAlgebra.mul!(transpose(w4),transpose(X_dist),transpose(y))
    @test isapprox(result4[:,cols1],w4.localarray)

    r_local = Array{T}(undef, N, M)
    LinearAlgebra.mul!(r_local, X_dist, XX)
    @test isapprox(r_local, result3)

    LinearAlgebra.mul!(transpose(r_local), transpose(XX), transpose(X_dist))
    @test isapprox(r_local, result3)

    v1 = Array{T}(reshape(collect(1:6),6))
    v1_dist=distribute(v1)
    v2 = Array{T}(reshape(collect(1:5),5))
    v2_dist=distribute(v2)
    lv1 = Array{T}(undef, 6)
    lv2 = Array{T}(undef, 5)
    lv2=distribute(lv2)

    ans1=LinearAlgebra.mul!(lv1,X_dist,v2)
    ans2=LinearAlgebra.mul!(lv1,X_dist,v2_dist)

    @test isapprox(ans1,ans2)

    ans3=LinearAlgebra.mul!(lv2,transpose(X_dist),v1)
    ans4=LinearAlgebra.mul!(lv2,transpose(X_dist),v1_dist)

    @test isapprox(ans3.localarray,ans4.localarray)

end


text:
function ABCDQR_kfilter(y,u,A,B,C,D,Q,R,x10,P10)
	#
	# Kalman filter for model
	#
	# x_{t+1} = A*x_{t} + B*u_{t} + w_{t}
	# y_{t}   = C*x_{t} + D*u_{t} + v_{t}
	#
	# cov(w_{t},v_{t}) = [Q 0;0 R]
	#
	# javier.cara@upm.es, 2015-10
	#

	(ny,nt) = size(y)
	nx = size(A,1)

	# allocation
	xtt = zeros(nx,nt)
	Ptt = zeros(nx,nx,nt)
	xtt1 = zeros(nx,nt+1)
	Ptt1 = zeros(nx,nx,nt+1)
	et = zeros(ny,nt)
	St = zeros(ny,ny,nt)
	Kt = zeros(nx,ny,nt)
	loglik = 0.0

	# Filter
	xtt1[:,1] = x10
	Ptt1[:,:,1] = P10
	for t in 1:nt
		#  innovations
		et[:,t] = y[:,t] - C*xtt1[:,t] - D*u[:,t]
		St[:,:,t] = C*Ptt1[:,:,t]*C' + R # et variance

		# Kalman gain
		Stinv = eye(ny,ny)/St[:,:,t] # numerically preferible to Stinv=inv(St)
		Kt[:,:,t] = Ptt1[:,:,t]*C'*Stinv # kalman gain

		# filtered values
		xtt[:,t] = xtt1[:,t] + Kt[:,:,t]*et[:,t]
		Ptt[:,:,t] = (eye(nx) - Kt[:,:,t]*C)*Ptt1[:,:,t]

		# one-step ahead prediction
		xtt1[:,t+1] = A*xtt[:,t] + B*u[:,t]
		Ptt1[:,:,t+1] = A*Ptt[:,:,t]*A' + Q

		# likelihood
		l0 = et[:,t]'*Stinv*et[:,t] # typeof(l0) = Array{Float64,1}
		loglik = loglik + log(det(St[:,:,t])) + l0[1]
	end

	loglik =  - ny*nt/2*log(2*pi) - 0.5*loglik

	return xtt,Ptt,xtt1,Ptt1,et,St,Kt,loglik

end


text:
"""
Abstract type for equations.
"""
abstract type AbstractMeanfieldEquations end

"""
    MeanfieldEquations <: AbstractMeanfieldEquations

Type defining a system of differential equations, where `lhs` is a vector of
derivatives and `rhs` is a vector of expressions. In addition, it keeps track
of the Hamiltonian, the collapse operators and the corresponding decay rates of
the system.

# Fields
*`equations`: Vector of the differential equations of averages.
*`operator_equations`: Vector of the operator differential equations.
*`states`: Vector containing the averages on the left-hand-side of the equations.
*`operators`: Vector containing the operators on the left-hand-side of the equations.
*`hamiltonian`: Operator defining the system Hamiltonian.
*`jumps`: Vector of operators specifying the decay processes.
*`jumps`: Vector of operators specifying the adjoint of the decay processes.
*`rates`: Decay rates corresponding to the `jumps`.
*`iv`: The independent variable (time parameter) of the system.
*`varmap`: Vector of pairs that map the averages to time-dependent variables.
    That format is necessary for ModelingToolkit functionality.
*`order`: The order at which the [`cumulant_expansion`](@ref) has been performed.

"""
struct MeanfieldEquations <: AbstractMeanfieldEquations
    equations::Vector{Symbolics.Equation}
    operator_equations::Vector{Symbolics.Equation}
    states::Vector
    operators::Vector{QNumber}
    hamiltonian::QNumber
    jumps::Vector
    jumps_dagger
    rates::Vector
    iv::SymbolicUtils.Sym
    varmap::Vector{Pair}
    order::Union{Int,Vector{<:Int},Nothing}
end

Base.getindex(de::AbstractMeanfieldEquations, i::Int) = de.equations[i]
Base.getindex(de::AbstractMeanfieldEquations, i) = de.equations[i]
Base.lastindex(de::AbstractMeanfieldEquations) = lastindex(de.equations)
Base.length(de::AbstractMeanfieldEquations) = length(de.equations)

function _append!(de::T, me::T) where T<:AbstractMeanfieldEquations
    append!(de.equations, me.equations)
    append!(de.operator_equations, me.operator_equations)
    append!(de.states, me.states)
    append!(de.operators, me.operators)
    append!(de.varmap, me.varmap)
    return de
end

# Substitution
function substitute(de::T,dict) where T<:AbstractMeanfieldEquations
    eqs = [substitute(eq, dict) for eq∈de.equations]
    states = getfield.(eqs, :lhs)
    fields = [getfield(de, s) for s∈fieldnames(T)[4:end]]
    return T(eqs, de.operator_equations, states, fields...)
end

# Simplification
function SymbolicUtils.simplify(de::T;kwargs...) where T<:AbstractMeanfieldEquations
    eqs = [SymbolicUtils.simplify(eq;kwargs...) for eq∈de.equations]
    eqs_op = [SymbolicUtils.simplify(eq;kwargs...) for eq∈de.operator_equations]
    fields = [getfield(de, s) for s∈fieldnames(T)[3:end]]
    return T(eqs,eqs_op,fields...)
end

# Adding MTK variables
function add_vars!(varmap, vs, t)
    keys = getindex.(varmap, 1)
    vals = getindex.(varmap, 2)
    hashkeys = map(hash, keys)
    hashvals = map(hash, vals)
    hashvs = map(hash, vs)
    for i=1:length(vs)
        if !(hashvs[i] ∈ hashkeys)
            var = make_var(vs[i], t)
            !(hash(var) ∈ hashvals) || @warn string("Two different averages have the exact same name. ",
                    "This may lead to unexpected behavior when trying to access the solution for $(vals[i])")
            push!(keys, vs[i])
            push!(vals, var)
            push!(hashkeys, hashvs[i])
        end
    end
    for i=length(varmap)+1:length(keys)
        push!(varmap, keys[i]=>vals[i])
    end
    return varmap
end

function make_var(v, t)
    sym = Symbol(string(v))
    d = source_metadata(:make_var, sym)
    var_f = SymbolicUtils.Sym{SymbolicUtils.FnType{Tuple{Any}, Complex}}(sym; metadata=d)
    return SymbolicUtils.Term{Complex}(var_f, [t]; metadata=d)
end

function make_varmap(vs, t)
    varmap = Pair{Any,Any}[]
    add_vars!(varmap, vs, t)
    return varmap
end

struct ScaledMeanfieldEquations <: AbstractMeanfieldEquations
    equations::Vector{Symbolics.Equation}
    operator_equations::Vector{Symbolics.Equation}
    states::Vector
    operators::Vector{QNumber}
    hamiltonian::QNumber
    jumps::Vector
    jumps_dagger
    rates::Vector
    iv::SymbolicUtils.Sym
    varmap::Vector{Pair}
    order::Union{Int,Vector{<:Int},Nothing}
    scale_aons
    names::Vector
    was_scaled::Vector{Bool}
end


text:
# Load the module and generate the functions
module CppHello
  using CxxWrap
  @wrapmodule(joinpath("/home/xiaodai/git/ParquetWriter.jl/test-include-julia/lib","libtestlib"))

  function __init__()
    @initcxx
  end
end

# Call greet and show the result
@show CppHello.greet("ok")

@show CppHello.greet("hello")


text:
using Documenter, Sandbox

makedocs(
    modules = [Sandbox],
    sitename = "Sandbox.jl",
)

deploydocs(
    repo = "github.com/staticfloat/Sandbox.jl.git",
    push_preview = true,
    devbranch = "main",
)


text:
using Images, ImageView
using Video4Linux
using AprilTags
using FixedPointNumbers
using LCMCore
using CaesarLCMTypes
using ArgParse

function commands()
  s = ArgParseSettings()
  @add_arg_table s begin
      "--device", "-d"
          help = "Device location for camera"
          arg_type = String
          default = "/dev/video1"
      "--frames", "-f"
          help = "number of frames to transmit"
          arg_type = Int
          default = 1000
  end
  parse_args(ARGS, s)
end

function showImage(image, tags)
    # Convert image to RGB
    imageCol = RGB.(image)
    #traw color box on tag corners
    foreach(tag->drawTagBox!(imageCol, tag), tags)

    imageCol
end

function apriltag_converter(tag::AprilTag, utime::Int64)
    msg=apriltag_t();
    msg.utime = utime
    msg.id=tag.id;
    msg.familyName=tag.family;
    msg.hammingDistance=tag.hamming;
    P = reshape([tag.p...;],2,4)'
    msg.p=P
    msg.cxy= tag.c;
    msg.homography= tag.H;
    msg.pose = homography_to_pose(tag.H, -80.,80.,320.,240.)
    return msg
end

@show parsed_args = commands()

# This colour mapping may be wrong for the PS3eye.
ycrcb = Video4Linux.YUYV(640,480)
vidchan = Channel((c::Channel) -> videoproducer(c, ycrcb, devicename = parsed_args["device"],
                                         iomethod = Video4Linux.IO_METHOD_MMAP, N=parsed_args["frames"] ))

##
#capture one frame to create im1 and canvas needed to diplay

A =  take!(vidchan)
im1 = RGB.(YCbCr.(view(A,:,:,1), view(A,:,:,2), view(A,:,:,3)))
# canvas = imshow(im1)

# Create default detector
detector = AprilTagDetector()

# tags = detector(im1)
lcm=LCM()

while isopen(vidchan)
    A =  take!(vidchan)
    utime = Int64(time()*1000_000)
    im1 = RGB.(YCbCr.(A[:,:,1], A[:,:,2], A[:,:,3]))
    image = Gray{N0f8}.(im1)

    # image = reinterpret.(RGB{N0f8},im1) #ImageMagick function
    tags = detector(image)
    msgs=apriltag_converter.(tags, utime)
    @show msgs
    data=encode.(msgs)

    for da in data
        publish(lcm,"TEST_CHANNEL",da)
    end

    #im2=showImage(image, tags)
    #imshow(canvas["gui"]["canvas"], im2)
end

close(vidchan)


text:
"""
tobdd

Create BDD
"""

export bdd!, bdd

using DD

function bdd(top::AbstractFaultTreeNode)
    forest = BDDForest{Int,Int,Int}(FullyReduced())
    node = bdd!(forest, top)
    return node, forest
end

function bdd!(forest::BDDForest{Tv,Ti,Tl}, top::AbstractFaultTreeNode) where {Tv,Ti,Tl}
    if !haskey(forest.vals, Tv(0))
        defval!(forest, Tv(0))
    end
    if !haskey(forest.vals, Tv(1))
        defval!(forest, Tv(1))
    end
    for (i,x) = enumerate(sort(collect(top.params)))
        if !haskey(forest.vars, x)
            defvar!(forest, x, Tl(i), domain([Ti(0), Ti(1)]))
        end
    end
    nodes = Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}}()
    _tobdd!(top, nodes, forest)
end

function _tobdd!(f::FaultTreeEvent, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    get(nodes, f) do
        v0 = ddval!(forest, Tv(0))
        v1 = ddval!(forest, Tv(1))
        nodes[f] = ddvar!(forest, f.var, v0, v1)
    end
end

function _tobdd!(f::AbstractFaultTreeOperation, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    get(nodes, f) do
        _tobdd!(Val(f.op), f, nodes, forest)
    end
end

function _tobdd!(::Val{:NOT}, f::AbstractFaultTreeOperation, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    bargs = [_tobdd!(x, nodes, forest) for x = f.args]
    @assert length(bargs) == 1
    bddnot!(forest, bargs[1])
end

function _tobdd!(::Val{:AND}, f::AbstractFaultTreeOperation, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    bargs = [_tobdd!(x, nodes, forest) for x = f.args]
    _createAndGate(forest, bargs)
end

function _tobdd!(::Val{:OR}, f::AbstractFaultTreeOperation, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    bargs = [_tobdd!(x, nodes, forest) for x = f.args]
    _createOrGate(forest, bargs)
end

function _tobdd!(::Val{:KofN}, f::AbstractFaultTreeOperation, nodes::Dict{AbstractFaultTreeNode,AbstractDDNode{Tv,Ti}},
    forest::BDDForest{Tv,Ti,Tl})::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    bargs = [_tobdd!(x, nodes, forest) for x = f.args]
    _createKofNGate(forest, f.k, bargs)
end

"""
"""

function _createOrGate(forest::BDDForest{Tv,Ti,Tl}, args)::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    x = args[1]
    for y = args[2:end]
        x = bddor!(forest, x, y)
    end
    x
end

function _createAndGate(forest::BDDForest{Tv,Ti,Tl}, args)::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    x = args[1]
    for y = args[2:end]
        x = bddand!(forest, x, y)
    end
    x
end

function _createKofNGate(forest::BDDForest{Tv,Ti,Tl}, k, args)::AbstractDDNode{Tv,Ti} where {Tv,Ti,Tl}
    n = length(args)
    (k == 1) && return _createOrGate(forest, args)
    (k == n) && return _createAndGate(forest, args)
    x = args[1]
    xs = args[2:end]
    bddite!(forest, x, _createKofNGate(forest, k-1, xs), _createKofNGate(forest, k, xs))
end


text:
enrichr_genesets() = ["Gene-set Library",
"Genes_Associated_with_NIH_Grants",
"Cancer_Cell_Line_Encyclopedia",
"Achilles_fitness_decrease",
"Achilles_fitness_increase",
"Aging_Perturbations_from_GEO_down",
"Aging_Perturbations_from_GEO_up",
"Allen_Brain_Atlas_10x_scRNA_2021",
"Allen_Brain_Atlas_down",
"Allen_Brain_Atlas_up",
"ARCHS4_Cell-lines",
"ARCHS4_IDG_Coexp",
"ARCHS4_Kinases_Coexp",
"ARCHS4_TFs_Coexp",
"ARCHS4_Tissues",
"BioCarta_2013",
"BioCarta_2015",
"BioCarta_2016",
"BioPlanet_2019",
"BioPlex_2017",
"CCLE_Proteomics_2020",
"ChEA_2013",
"ChEA_2015",
"ChEA_2016",
"Chromosome_Location",
"Chromosome_Location_hg19",
"ClinVar_2019",
"CORUM",
"COVID-19_Related_Gene_Sets",
"Data_Acquisition_Method_Most_Popular_Genes",
"dbGaP",
"DepMap_WG_CRISPR_Screens_Broad_CellLines_2019",
"DepMap_WG_CRISPR_Screens_Sanger_CellLines_2019",
"Disease_Perturbations_from_GEO_down",
"Disease_Perturbations_from_GEO_up",
"Disease_Signatures_from_GEO_down_2014",
"Disease_Signatures_from_GEO_up_2014",
"DisGeNET",
"Drug_Perturbations_from_GEO_2014",
"Drug_Perturbations_from_GEO_down",
"Drug_Perturbations_from_GEO_up",
"DrugMatrix",
"DSigDB",
"Elsevier_Pathway_Collection",
"ENCODE_and_ChEA_Consensus_TFs_from_ChIP-X",
"ENCODE_Histone_Modifications_2013",
"ENCODE_Histone_Modifications_2015",
"ENCODE_TF_ChIP-seq_2014",
"ENCODE_TF_ChIP-seq_2015",
"Enrichr_Libraries_Most_Popular_Genes",
"Enrichr_Submissions_TF-Gene_Coocurrence",
"Enrichr_Users_Contributed_Lists_2020",
"Epigenomics_Roadmap_HM_ChIP-seq",
"ESCAPE",
"Gene_Perturbations_from_GEO_down",
"Gene_Perturbations_from_GEO_up",
"GeneSigDB",
"Genome_Browser_PWMs",
"GO_Biological_Process_2013",
"GO_Biological_Process_2015",
"GO_Biological_Process_2017",
"GO_Biological_Process_2017b",
"GO_Biological_Process_2018",
"GO_Cellular_Component_2013",
"GO_Cellular_Component_2015",
"GO_Cellular_Component_2017",
"GO_Cellular_Component_2017b",
"GO_Cellular_Component_2018",
"GO_Molecular_Function_2013",
"GO_Molecular_Function_2015",
"GO_Molecular_Function_2017",
"GO_Molecular_Function_2017b",
"GO_Molecular_Function_2018",
"GTEx_Tissue_Sample_Gene_Expression_Profiles_down",
"GTEx_Tissue_Sample_Gene_Expression_Profiles_up",
"GWAS_Catalog_2019",
"HMDB_Metabolites",
"HMS_LINCS_KinomeScan",
"HomoloGene",
"Human_Gene_Atlas",
"Human_Phenotype_Ontology",
"HumanCyc_2015",
"HumanCyc_2016",
"huMAP",
"InterPro_Domains_2019",
"Jensen_COMPARTMENTS",
"Jensen_DISEASES",
"Jensen_TISSUES",
"KEA_2013",
"KEA_2015",
"KEGG_2013",
"KEGG_2015",
"KEGG_2016",
"KEGG_2019_Human",
"KEGG_2019_Mouse",
"Kinase_Perturbations_from_GEO_down",
"Kinase_Perturbations_from_GEO_up",
"L1000_Kinase_and_GPCR_Perturbations_down",
"L1000_Kinase_and_GPCR_Perturbations_up",
"Ligand_Perturbations_from_GEO_down",
"Ligand_Perturbations_from_GEO_up",
"LINCS_L1000_Chem_Pert_down",
"LINCS_L1000_Chem_Pert_up",
"LINCS_L1000_Ligand_Perturbations_down",
"LINCS_L1000_Ligand_Perturbations_up",
"lncHUB_lncRNA_Co-Expression",
"MCF7_Perturbations_from_GEO_down",
"MCF7_Perturbations_from_GEO_up",
"MGI_Mammalian_Phenotype_2013",
"MGI_Mammalian_Phenotype_2017",
"MGI_Mammalian_Phenotype_Level_3",
"MGI_Mammalian_Phenotype_Level_4",
"MGI_Mammalian_Phenotype_Level_4_2019",
"Microbe_Perturbations_from_GEO_down",
"Microbe_Perturbations_from_GEO_up",
"miRTarBase_2017",
"Mouse_Gene_Atlas",
"MSigDB_Computational",
"MSigDB_Hallmark_2020",
"MSigDB_Oncogenic_Signatures",
"NCI-60_Cancer_Cell_Lines",
"NCI-Nature_2015",
"NCI-Nature_2016",
"NIH_Funded_PIs_2017_AutoRIF_ARCHS4_Predictions",
"NIH_Funded_PIs_2017_GeneRIF_ARCHS4_Predictions",
"NIH_Funded_PIs_2017_Human_AutoRIF",
"NIH_Funded_PIs_2017_Human_GeneRIF",
"NURSA_Human_Endogenous_Complexome",
"Old_CMAP_down",
"Old_CMAP_up",
"OMIM_Disease",
"OMIM_Expanded",
"Panther_2015",
"Panther_2016",
"Pfam_Domains_2019",
"Pfam_InterPro_Domains",
"PheWeb_2019",
"Phosphatase_Substrates_from_DEPOD",
"PPI_Hub_Proteins",
"ProteomicsDB_2020",
"Rare_Diseases_AutoRIF_ARCHS4_Predictions",
"Rare_Diseases_AutoRIF_Gene_Lists",
"Rare_Diseases_GeneRIF_ARCHS4_Predictions",
"Rare_Diseases_GeneRIF_Gene_Lists",
"Reactome_2013",
"Reactome_2015",
"Reactome_2016",
"RNA-Seq_Disease_Gene_and_Drug_Signatures_from_GEO",
"SILAC_Phosphoproteomics",
"SubCell_BarCode",
"SysMyo_Muscle_Gene_Sets",
"Table_Mining_of_CRISPR_Studies",
"TargetScan_microRNA",
"TargetScan_microRNA_2017",
"TF-LOF_Expression_from_GEO",
"TF_Perturbations_Followed_by_Expression",
"TG_GATES_2020",
"Tissue_Protein_Expression_from_Human_Proteome_Map",
"Tissue_Protein_Expression_from_ProteomicsDB",
"Transcription_Factor_PPIs",
"TRANSFAC_and_JASPAR_PWMs",
"TRRUST_Transcription_Factors_2019",
"UK_Biobank_GWAS_v1",
"Virus-Host_PPI_P-HIPSTer_2020",
"Virus_Perturbations_from_GEO_down",
"Virus_Perturbations_from_GEO_up",
"VirusMINT",
"WikiPathways_2013",
"WikiPathways_2015",
"WikiPathways_2016",
"WikiPathways_2019_Human",
"WikiPathways_2019_Mouse"]