text:
# structure for iterating over a csv file
# no automatic type inference is done, but types are allowed to be passed
# for as many columns as desired; `CSV.detect(row, i)` can also be used to
# use the same inference logic used in `CSV.File` for determing a cell's typed value
struct Rows{transpose, O, O2, IO, T, V}
    name::String
    names::Vector{Symbol} # only includes "select"ed columns
    finaltypes::Vector{Type} # only includes "select"ed columns
    columnmap::Vector{Int} # maps "select"ed column index to actual file column index
    types::Vector{Type} # includes *all* columns (whether or not selected)
    flags::Vector{UInt8}
    cols::Int64
    e::UInt8
    buf::IO
    datapos::Int64
    datarow::Int
    len::Int
    limit::Int64
    options::O # Parsers.Options
    coloptions::O2 # Union{Nothing, Vector{Parsers.Options}}
    customtypes::T
    positions::Vector{Int64}
    reusebuffer::Bool
    columns::Vector{AbstractVector} # for parsing, allocated once and used for each iteration
    values::Vector{V} # once values are parsed, put in values; allocated on each iteration if reusebuffer=false
    lookup::Dict{Symbol, Int}
    numwarnings::Base.RefValue{Int}
    maxwarnings::Int
end

function Base.show(io::IO, r::Rows)
    println(io, "CSV.Rows(\"$(r.name)\"):")
    println(io, "Size: $(r.cols)")
    show(io, Tables.schema(r))
end

"""
    CSV.Rows(source; kwargs...) => CSV.Rows

Read a csv input returning a `CSV.Rows` object.

The `source` argument can be one of:
  * filename given as a string or FilePaths.jl type
  * an `AbstractVector{UInt8}` like a byte buffer or `codeunits(string)`
  * an `IOBuffer`

To read a csv file from a url, use the HTTP.jl package, where the `HTTP.Response` body can be passed like:
```julia
f = CSV.Rows(HTTP.get(url).body)
```

For other `IO` or `Cmd` inputs, you can pass them like: `f = CSV.Rows(read(obj))`.

While similar to [`CSV.File`](@ref), `CSV.Rows` provides a slightly different interface, the tradeoffs including:
  * Very minimal memory footprint; while iterating, only the current row values are buffered
  * Only provides row access via iteration; to access columns, one can stream the rows into a table type
  * Performs no type inference; each column/cell is essentially treated as `Union{String, Missing}`, users can utilize the performant `Parsers.parse(T, str)` to convert values to a more specific type if needed, or pass types upon construction using the `type` or `types` keyword arguments

Opens the file and uses passed arguments to detect the number of columns, ***but not*** column types (column types default to `String` unless otherwise manually provided).
The returned `CSV.Rows` object supports the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface
and can iterate rows. Each row object supports `propertynames`, `getproperty`, and `getindex` to access individual row values.
Note that duplicate column names will be detected and adjusted to ensure uniqueness (duplicate column name `a` will become `a_1`).
For example, one could iterate over a csv file with column names `a`, `b`, and `c` by doing:

```julia
for row in CSV.Rows(file)
    println("a=\$(row.a), b=\$(row.b), c=\$(row.c)")
end
```

Supported keyword arguments include:
* File layout options:
  * `header=1`: the `header` argument can be an `Int`, indicating the row to parse for column names; or a `Range`, indicating a span of rows to be concatenated together as column names; or an entire `Vector{Symbol}` or `Vector{String}` to use as column names; if a file doesn't have column names, either provide them as a `Vector`, or set `header=0` or `header=false` and column names will be auto-generated (`Column1`, `Column2`, etc.)
  * `normalizenames=false`: whether column names should be "normalized" into valid Julia identifier symbols; useful when iterating rows and accessing column values of a row via `getproperty` (e.g. `row.col1`)
  * `datarow`: an `Int` argument to specify the row where the data starts in the csv file; by default, the next row after the `header` row is used. If `header=0`, then the 1st row is assumed to be the start of data
  * `skipto::Int`: similar to `datarow`, specifies the number of rows to skip before starting to read data
  * `limit`: an `Int` to indicate a limited number of rows to parse in a csv file; use in combination with `skipto` to read a specific, contiguous chunk within a file
  * `transpose::Bool`: read a csv file "transposed", i.e. each column is parsed as a row
  * `comment`: rows that begin with this `String` will be skipped while parsing
  * `ignoreemptylines::Bool=true`: whether empty rows/lines in a file should be ignored (if `false`, each column will be assigned `missing` for that empty row)
* Parsing options:
  * `missingstrings`, `missingstring`: either a `String`, or `Vector{String}` to use as sentinel values that will be parsed as `missing`; by default, only an empty field (two consecutive delimiters) is considered `missing`
  * `delim=','`: a `Char` or `String` that indicates how columns are delimited in a file; if no argument is provided, parsing will try to detect the most consistent delimiter on the first 10 rows of the file
  * `ignorerepeated::Bool=false`: whether repeated (consecutive) delimiters should be ignored while parsing; useful for fixed-width files with delimiter padding between cells
  * `quotechar='"'`, `openquotechar`, `closequotechar`: a `Char` (or different start and end characters) that indicate a quoted field which may contain textual delimiters or newline characters
  * `escapechar='"'`: the `Char` used to escape quote characters in a quoted field
  * `dateformat::Union{String, Dates.DateFormat, Nothing}`: a date format string to indicate how Date/DateTime columns are formatted for the entire file
  * `decimal='.'`: a `Char` indicating how decimals are separated in floats, i.e. `3.14` used '.', or `3,14` uses a comma ','
  * `truestrings`, `falsestrings`: `Vectors of Strings` that indicate how `true` or `false` values are represented; by default only `true` and `false` are treated as `Bool`
* Column Type Options:
  * `type`: a single type to use for parsing an entire file; i.e. all columns will be treated as the same type; useful for matrix-like data files
  * `types`: a Vector or Dict of types to be used for column types; a Dict can map column index `Int`, or name `Symbol` or `String` to type for a column, i.e. Dict(1=>Float64) will set the first column as a Float64, Dict(:column1=>Float64) will set the column named column1 to Float64 and, Dict("column1"=>Float64) will set the column1 to Float64; if a `Vector` if provided, it must match the # of columns provided or detected in `header`
  * `typemap::Dict{Type, Type}`: a mapping of a type that should be replaced in every instance with another type, i.e. `Dict(Float64=>String)` would change every detected `Float64` column to be parsed as `String`
  * `lazystrings::Bool=true`: avoid allocating full strings while parsing; accessing a string column will materialize the full `String`
  * `strict::Bool=false`: whether invalid values should throw a parsing error or be replaced with `missing`
  * `silencewarnings::Bool=false`: if `strict=false`, whether invalid value warnings should be silenced
  * `maxwarnings::Int=100`: if more than `maxwarnings` number of warnings are printed while parsing, further warnings will be silenced by default
* Iteration options:
  * `reusebuffer=false`: while iterating, whether a single row buffer should be allocated and reused on each iteration; only use if each row will be iterated once and not re-used (e.g. it's not safe to use this option if doing `collect(CSV.Rows(file))` because only current iterated row is "valid")
"""
function Rows(source;
    # file options
    # header can be a row number, range of rows, or actual string vector
    header::Union{Integer, Vector{Symbol}, Vector{String}, AbstractVector{<:Integer}}=1,
    normalizenames::Bool=false,
    # by default, data starts immediately after header or start of file
    datarow::Integer=-1,
    skipto::Union{Nothing, Integer}=nothing,
    footerskip::Integer=0,
    limit::Integer=typemax(Int64),
    transpose::Bool=false,
    comment::Union{String, Nothing}=nothing,
    use_mmap=nothing,
    ignoreemptylines::Bool=true,
    select=nothing,
    drop=nothing,
    # parsing options
    missingstrings=String[],
    missingstring="",
    delim::Union{Nothing, Char, String}=nothing,
    ignorerepeated::Bool=false,
    quotechar::Union{UInt8, Char}='"',
    openquotechar::Union{UInt8, Char, Nothing}=nothing,
    closequotechar::Union{UInt8, Char, Nothing}=nothing,
    escapechar::Union{UInt8, Char}='"',
    dateformat::Union{String, Dates.DateFormat, Nothing}=nothing,
    dateformats::Union{AbstractDict, Nothing}=nothing,
    decimal::Union{UInt8, Char}=UInt8('.'),
    truestrings::Union{Vector{String}, Nothing}=["true", "True", "TRUE"],
    falsestrings::Union{Vector{String}, Nothing}=["false", "False", "FALSE"],
    # type options
    type=nothing,
    types=nothing,
    typemap::Dict=Dict{Type, Type}(),
    pool::Union{Bool, Real}=0.1,
    lazystrings::Bool=true,
    strict::Bool=false,
    silencewarnings::Bool=false,
    debug::Bool=false,
    parsingdebug::Bool=false,
    reusebuffer::Bool=false,
    maxwarnings::Int=100,
    kw...)

    h = Header(source, header, normalizenames, datarow, skipto, footerskip, transpose, comment, use_mmap, ignoreemptylines, select, drop, missingstrings, missingstring, delim, ignorerepeated, quotechar, openquotechar, closequotechar, escapechar, dateformat, dateformats, decimal, truestrings, falsestrings, type, types, typemap, pool, lazystrings, strict, silencewarnings, debug, parsingdebug, true)
    columns = allocate(1, h.cols, h.types, h.flags, nothing)
    values = all(x->x == Union{String, Missing}, h.types) && lazystrings ? Vector{PosLen}(undef, h.cols) : Vector{Any}(undef, h.cols)
    finaltypes = copy(h.types)
    columnmap = [i for i = 1:h.cols]
    deleteat!(h.names, h.todrop)
    deleteat!(finaltypes, h.todrop)
    deleteat!(columnmap, h.todrop)
    lookup = Dict(nm=>i for (i, nm) in enumerate(h.names))
    return Rows{transpose, typeof(h.options), typeof(h.coloptions), typeof(h.buf), typeof(h.customtypes), eltype(values)}(
        h.name,
        h.names,
        finaltypes,
        columnmap,
        h.types,
        h.flags,
        h.cols,
        h.e,
        h.buf,
        h.datapos,
        h.datarow,
        h.len,
        limit,
        h.options,
        h.coloptions,
        h.customtypes,
        h.positions,
        reusebuffer,
        columns,
        values,
        lookup,
        Ref(0),
        maxwarnings
    )
end

Tables.rowtable(::Type{<:Rows}) = true
Tables.rows(r::Rows) = r
Tables.schema(r::Rows) = Tables.Schema(r.names, r.finaltypes)
Base.eltype(r::Rows) = Row2
Base.IteratorSize(::Type{<:Rows}) = Base.SizeUnknown()

const EMPTY_TYPEMAP = Dict{Type, Type}()
const EMPTY_REFS = RefPool[]

@inline function setcustom!(::Type{T}, values, columns, i) where {T}
    if @generated
        block = Expr(:block)
        push!(block.args, quote
            error("CSV.jl code-generation error, unexpected column type: $(typeof(column))")
        end)
        for i = 1:fieldcount(T)
            vec = fieldtype(T, i)
            pushfirst!(block.args, quote
                if column isa $(fieldtype(vec, 1))
                    @inbounds values[i] = column[1]
                    return
                end
            end)
        end
        pushfirst!(block.args, quote
            @inbounds column = columns[col]
        end)
        pushfirst!(block.args, Expr(:meta, :inline))
        # @show block
        return block
    else
        # println("generated function failed")
        @inbounds column = columns[i]
        @inbounds values[i] = column[1]
        return
    end
end

@inline function Base.iterate(r::Rows{transpose, O, O2, IO, T, V}, (pos, len, row)=(r.datapos, r.len, 1)) where {transpose, O, O2, IO, T, V}
    (pos > len || row > r.limit) && return nothing
    pos > len && return nothing
    pos = parserow(1, Val(transpose), r.cols, EMPTY_TYPEMAP, r.columns, r.datapos, r.buf, pos, len, r.positions, 0.0, EMPTY_REFS, 1, r.datarow + row - 2, r.types, r.flags, false, r.options, r.coloptions, T, r.numwarnings, r.maxwarnings)
    cols = r.cols
    values = r.reusebuffer ? r.values : Vector{V}(undef, cols)
    columns = r.columns
    for i = 1:cols
        @inbounds column = columns[i]
        if column isa Vector{PosLen}
            @inbounds values[i] = column[1]
        elseif column isa SVec{Int64}
            @inbounds values[i] = column[1]
        elseif column isa SVec{Float64}
            @inbounds values[i] = column[1]
        elseif column isa SVec2{String}
            @inbounds values[i] = column[1]
        elseif column isa SVec{Date}
            @inbounds values[i] = column[1]
        elseif column isa SVec{DateTime}
            @inbounds values[i] = column[1]
        elseif column isa SVec{Time}
            @inbounds values[i] = column[1]
        elseif column isa Vector{Union{Missing, Bool}}
            @inbounds values[i] = column[1]
        elseif column isa Vector{UInt32}
            @inbounds values[i] = column[1]
        elseif T !== Tuple{}
            setcustom!(T, values, columns, i)
        else
            error("bad array type: $(typeof(column))")
        end
    end
    return Row2{O, O2, V}(r.names, r.finaltypes, r.columnmap, r.types, r.lookup, values, r.buf, r.e, r.options, r.coloptions), (pos, len, row + 1)
end

struct Row2{O, O2, V} <: Tables.AbstractRow
    names::Vector{Symbol}
    finaltypes::Vector{Type}
    columnmap::Vector{Int}
    types::Vector{Type}
    lookup::Dict{Symbol, Int}
    values::Vector{V}
    buf::Vector{UInt8}
    e::UInt8
    options::O
    coloptions::O2
end

getnames(r::Row2) = getfield(r, :names)
getfinaltypes(r::Row2) = getfield(r, :finaltypes)
getcolumnmap(r::Row2) = getfield(r, :columnmap)
gettypes(r::Row2) = getfield(r, :types)
getlookup(r::Row2) = getfield(r, :lookup)
getvalues(r::Row2) = getfield(r, :values)
getbuf(r::Row2) = getfield(r, :buf)
gete(r::Row2) = getfield(r, :e)
getoptions(r::Row2) = getfield(r, :options)
getcoloptions(r::Row2) = getfield(r, :coloptions)

Tables.columnnames(r::Row2) = getnames(r)

Base.checkbounds(r::Row2, i) = 0 < i < length(r)

Tables.getcolumn(r::Row2, nm::Symbol) = Tables.getcolumn(r, getlookup(r)[nm])
Tables.getcolumn(r::Row2, i::Int) = Tables.getcolumn(r, gettypes(r)[i], i, getnames(r)[i])

Tables.getcolumn(r::Row2{O, O2, PosLen}, nm::Symbol) where {O, O2} = @inbounds Tables.getcolumn(r, getlookup(r)[nm])

Base.@propagate_inbounds function Tables.getcolumn(r::Row2, ::Type{Missing}, i::Int, nm::Symbol)
    @boundscheck checkbounds(r, i)
    return missing
end

Base.@propagate_inbounds function Tables.getcolumn(r::Row2, ::Type{T}, i::Int, nm::Symbol) where {T}
    @boundscheck checkbounds(r, i)
    j = getcolumnmap(r)[i]
    @inbounds x = getvalues(r)[j]
    return x
end

Base.@propagate_inbounds Tables.getcolumn(r::Row2{O, O2, PosLen}, ::Type{T}, i::Int, nm::Symbol) where {T, O, O2} = error("row values are string only; requested type $T not supported; see `Parsers.parse(row, $T, $i)`")

Base.@propagate_inbounds function Tables.getcolumn(r::Row2, ::Union{Type{Union{Missing, String}}, Type{String}}, i::Int, nm::Symbol)
    @boundscheck checkbounds(r, i)
    j = getcolumnmap(r)[i]
    @inbounds poslen = getvalues(r)[j]
    if poslen isa Missing
        return missing
    elseif poslen isa String
        return poslen
    else
        return str(getbuf(r), gete(r), poslen)
    end
end

Tables.getcolumn(r::Row2{O, O2, PosLen}, ::Union{Type{Union{Missing, String}}, Type{String}}, i::Int, nm::Symbol) where {O, O2} = Tables.getcolumn(r, i)
Base.@propagate_inbounds function Tables.getcolumn(r::Row2{O, O2, PosLen}, i::Int) where {O, O2}
    @boundscheck checkbounds(r, i)
    @inbounds j = getcolumnmap(r)[i]
    @inbounds poslen = getvalues(r)[j]
    if poslen isa Missing
        return missing
    else
        return @inbounds str(getbuf(r), gete(r), poslen)
    end
end

@noinline stringsonly() = error("Parsers.parse only allowed on String column types")

Base.@propagate_inbounds function Parsers.parse(::Type{T}, r::Row2, i::Int) where {T}
    @boundscheck checkbounds(r, i)
    j = getcolumnmap(r)[i]
    type = gettypes(r)[j]
    (type == String || type == Union{String, Missing}) || stringsonly()
    @inbounds poslen = getvalues(r)[j]
    missingvalue(poslen) && return missing
    pos = getpos(poslen)
    colopts = getcoloptions(r)
    opts = colopts === nothing ? getoptions(r) : colopts[j]
    x, code, vpos, vlen, tlen = Parsers.xparse(T, getbuf(r), pos, pos + getlen(poslen), opts)
    return Parsers.ok(code) ? x : missing
end

Base.@propagate_inbounds function detect(r::Row2, i::Int)
    @boundscheck checkbounds(r, i)
    j = getcolumnmap(r)[i]
    T = gettypes(r)[j]
    (T == String || T == Union{String, Missing}) || stringsonly()
    @inbounds offlen = getvalues(r)[j]
    missingvalue(offlen) && return missing
    pos = getpos(offlen)
    colopts = getcoloptions(r)
    opts = colopts === nothing ? getoptions(r) : colopts[j]
    x = detect(getbuf(r), pos, pos + getlen(offlen) - 1, opts)
    return x === nothing ? r[i] : x
end

function Parsers.parse(::Type{T}, r::Row2, nm::Symbol) where {T}
    @inbounds x = Parsers.parse(T, r, getlookup(r)[nm])
    return x
end

function detect(r::Row2, nm::Symbol)
    @inbounds x = detect(r, getlookup(r)[nm])
    return x
end


text:
"""
Although the BSE exposes a special SOAP-based API for programmatic access to its database, I had too much trouble getting the desired results from it. Therefore the solution in this module is based around regexing of the
human readable HTML output.
"""
module BasisSetExchangeModule

export obtainBasisSetExchangeEntries, downloadBasisSetBasisSetExchange, computeBasisSetExchangeEntry, BasisSetExchangeEntry
using Requests, ..BasisSetModule
import Base.display, ..BasisSetModule.BasisSet

type BasisSetExchangeEntry
  url::String
  name::String
  availtype::String
  elts::String
  status::String
  hasEcp::String
  hasSpin::String
  lastModifiedDate::String
  contributionPI::String
  contributorName::String
  bsAbstract::String
end

function display(bseEntry::BasisSetExchangeEntry)
  @printf("%40s    |    %s\n", bseEntry.name, bseEntry.bsAbstract)
end

function display(arr::Array{BasisSetExchangeEntry,1})
  for entry in arr
    display(entry)
  end
end

function obtainBasisSetExchangeEntries()
  retval = Array{BasisSetExchangeEntry,1}()
  main = get("https://bse.pnl.gov:443/bse/portal/user/anon/js_peid/11543880926284/action/portlets.BasisSetAction/template/courier_content/panel/Main")
  for bsline in matchall(r"basisSets\[\d*\].*",Requests.text(main))
    regmatch = match(r"basisSet\(\"(?P<url>.*)\", \"(?P<name>.*)\", \"(?P<type>.*)\", \"\[(?P<elts>.*)\]\", \"(?P<status>.*)\", \"(?P<hasEcp>.*)\", \"(?P<hasSpin>.*)\", \"(?P<lastModifiedDate>.*)\", \"(?P<contributionPI>.*)\", \"(?P<contributorName>.*)\", \"(?P<bsAbstract>.*)\"\)",bsline)
    eltsList = replace(regmatch[:elts],",","")
    push!(retval,BasisSetExchangeEntry(regmatch[:url],regmatch[:name],regmatch[:type],eltsList,regmatch[:status],regmatch[:hasEcp],regmatch[:hasSpin],regmatch[:lastModifiedDate],regmatch[:contributionPI],regmatch[:contributorName],regmatch[:bsAbstract]))
  end
  return retval
end

function downloadBasisSetBasisSetExchange(entry::BasisSetExchangeEntry, filename::AbstractString, format::AbstractString="TX93")
  requestresponse = post("https://bse.pnl.gov:443/bse/portal/user/anon/js_peid/11543880926284/action/portlets.BasisSetAction/template/courier_content/panel/Main/eventSubmit_doDownload/true";
			data=Dict("bsurl" => entry.url, "bsname" => entry.name, "elts" => entry.elts, "format" => format, "minimize" => "true"), allow_redirects=false)
  sessioncookie = requestresponse.cookies["JSESSIONID"].value
  finalresponse = post("https://bse.pnl.gov/bse/portal/user/anon/panel/Main/template/courier_content/js_peid/11543880926284"; cookies = Dict("JSESSIONID" => sessioncookie))
  basSetDef = replace(Requests.text(finalresponse),r".*<pre style.*>\n*(.*)</pre>.*"s,s"\1")
  outfd = open(filename, "w")
  write(outfd, basSetDef)
  close(outfd)
end

function computeBasisSetExchangeEntry(name::AbstractString, arr::Array{BasisSetExchangeEntry,1}; exactmatching::Bool=false)
  result = Void
  for entry in arr
    if (!exactmatching && ismatch(Regex(name,"i"),entry.name) || name == entry.name)
      if result == Void
        result = entry
      elseif isa(result,BasisSetExchangeEntry)
        result = [result,entry]
      else #isa(result,Array{BasisSetExchangeEntry})
        push!(result,entry)
      end
    end
  end
  if result==Void
    error("$name is not contained within the given BasisSetExchangeEntry list.")
  end
  return result
end

"""
    BasisSet(name)

creates a BasisSet object from the specified file (possibly with a corresponding file extension).
If no such file can be found the argument is interpreted as the name of a basis set and that basis
set obtained from Basis Set Exchange.
"""
function BasisSet(name::AbstractString)
  if isfile(name)
    return readBasisSetTX93(name)
  end

  for ext in ("tx93", "TX93", "Tx93")
    if isfile("$name.$ext")
      return readBasisSetTX93("$name.$ext")
    end
  end

  info("BasisSet file not found locally. Obtaining from BasisSetExchange...")
  bseentries = obtainBasisSetExchangeEntries()
  entry = computeBasisSetExchangeEntry(name,bseentries;exactmatching=true)
  downloadBasisSetBasisSetExchange(entry,"$name.tx93","TX93")
  return readBasisSetTX93("$name.tx93")
end

end # module

ext: jl


text:
# This file was generated, do not modify it. # hide
iris = dataset("datasets", "iris")
first(iris, 3)


text:
@testset "version" begin
    @test isa(LibSymspg.spg_get_major_version(), Int64)
    @test isa(LibSymspg.spg_get_minor_version(), Int64)
    @test isa(LibSymspg.spg_get_micro_version(), Int64)
end

@testset "SpglibSpacegroupType" begin
    spacegroup_type = LibSymspg.spg_get_spacegroup_type(101)
    @test spacegroup_type.number == 15
    @test LibSymspg.char2Str(spacegroup_type.hall_symbol) == "-I 2a"
    @test LibSymspg.char2Str(spacegroup_type.arithmetic_crystal_class_symbol) == "2/mC"
end

@testset "SpglibDataset" begin
    latt = [4.0 0.0 0.0; 2.0 3.4641 0.0; 0.0 0.0 12.0]
    positions = [0.0 1/3; 0.0 1/3; 0.0 1/3]
    types = [1, 1]
    num_atom = 2
    db = LibSymspg.spg_get_dataset(latt, positions, types, num_atom, 1e-3)
    @test LibSymspg.char2Str(db.international_symbol) == "P-3m1"

    nop = db.n_operations
    @test nop == 12
    r_ = unsafe_wrap(Array{NTuple{9, Cint}}, db.rotations, nop)
    r = LibSymspg.rotsFromTuple(r_, nop)
    @test size(r) == (3, 3, 12)
    t_ = unsafe_wrap(Array{NTuple{3, Float64}}, db.translations, nop)
    t = LibSymspg.transFromTuple(t_, nop)
    @test size(t) == (3, 12)

    @test LibSymspg.char2Str(db.pointgroup_symbol) == "-3m"
end

@testset "Get Symmetry Ops" begin
    latt = [4.0 0.0 0.0;
            0.0 4.0 0.0;
            0.0 0.0 4.0]
    positions = [0.0 0.5; 0.0 0.5; 0.0 0.5]
    types = [1, 1]
    num_atom = 2

    # will be used after
    max_size = num_atom*48
    rots = Array{Cint, 3}(undef, 3, 3, max_size)
    trans = Array{Float64, 2}(undef, 3, max_size)

    _ = LibSymspg.spg_get_symmetry!(rots, trans, max_size, latt, positions, types, num_atom, 1e-5)
    @test size(rots) == (3, 3, 96)
    @test size(trans) == (3, 96)

    hall_number = LibSymspg.spg_get_hall_number_from_symmetry(rots, trans, max_size, 1e-5)
    @test hall_number == 529

    latt = [4.0 0.0 0.0;
            0.0 4.0 0.0;
            0.0 0.0 4.0]
    positions = [0.0 0.5; 0.0 0.5; 0.0 0.5]
    types = [1, 1]
    equivalent_atoms = Base.convert(Array{Cint, 1}, [0, 0])
    spins = [1.0, -2.0]
    num_atom = 2

    max_size = num_atom*48
    rots = Array{Cint, 3}(undef, 3, 3, max_size)
    trans = Array{Float64, 2}(undef, 3, max_size)
    _ = LibSymspg.spg_get_symmetry_with_collinear_spin!(rots, trans, equivalent_atoms, max_size,
                                                        latt, positions, types, spins, num_atom, 1e-5)
    @test equivalent_atoms == [0, 1]

    latt = [4.0 0.0 0.0;
            0.0 4.0 0.0;
            0.0 0.0 4.0]
    positions = [0.0 0.5; 0.0 0.5; 0.0 0.5]
    types = [1, 1]
    num_atom = 2
    @test LibSymspg.spg_get_multiplicity(latt, positions, types, num_atom, 1e-5) == 96
end

@testset "lattice reduce" begin
    latt = [4.0 20.0 0.0; 0.0 2.0 0.0; 0.0 0.0 12.0]
    LibSymspg.spg_niggli_reduce!(latt, 1e-3)
    @test latt ≈ [0.0 -2.0 0.0; 4.0 0.0 0.0; 0.0 0.0 12.0]

    latt = [4.0 20.0 0.0; 0.0 2.0 0.0; 0.0 0.0 12.0]
    LibSymspg.spg_delaunay_reduce!(latt, 1e-3)
    @test latt ≈ [0.0 2.0 0.0; -4.0 -0.0 0.0; -0.0 -0.0 12.0]
end

@testset "reciprocal mesh" begin
    latt = [-2.0 2.0 2.0; 2.0 -2.0 2.0; 2.0 2.0 -2.0]
    positions = Array{Float64, 2}([0.0 0.0 0.0]')
    types = [1]
    na = 1
    mesh = [4, 4, 4]
    is_shift = [0, 0, 0]
    nir, grid_address, ir_mapping_table =
        LibSymspg.spg_get_ir_reciprocal_mesh(mesh, is_shift, 1, latt, positions, types, na, 1e-5)
    @test nir == 8
end

# @testset "reciprocal mesh" begin
#     latt = [0.0 5.0 5.0; 5.0 0.0 5.0; 5.0 5.0 0.0]
#     positions = Array{Float64, 2}([0.0 0.25; 0.0 0.25; 0.0 0.25])
#     types = [1, 1]
#     na = 2
#     mesh = [3,3,3]
#     is_shift = [1,1,1]
#     nir, grid_address, ir_mapping_table =
#         LibSymspg.spg_get_ir_reciprocal_mesh(mesh, is_shift, 1, latt, positions, types, na, 1e-5)
#     @test nir == 4
# end

@testset "Cell reduce standardize" begin
    latt = [4.0 0.0 0.0;
            0.0 4.0 0.0;
            0.0 0.0 4.0]
    positions = [0.0 0.5; 0.0 0.5; 0.0 0.5]
    types = [1, 1]
    new_latt, new_positions, new_types = LibSymspg.spg_find_primitive(latt, positions, types, 1e-5)
    # test arguments not modified
    @test latt ≈ [4.0 0.0 0.0; 0.0 4.0 0.0; 0.0 0.0 4.0]
    @test positions ≈ [0.0 0.5; 0.0 0.5; 0.0 0.5]
    @test types == [1, 1]

    @test new_latt ≈ [-2.0 2.0 2.0; 2.0 -2.0 2.0; 2.0 2.0 -2.0]
    @test new_positions ≈ [0.0, 0.0, 0.0]
    @test new_types == [1]

    latt = [-2.0 2.0 2.0; 2.0 -2.0 2.0; 2.0 2.0 -2.0]
    positions = Array{Float64, 2}([0.0 0.0 0.0]')
    types = [1]
    latt, positions, types = LibSymspg.spg_refine_cell(latt, positions, types, 1e-5)
    @test latt ≈ [4.0 0.0 0.0; 0.0 4.0 0.0; 0.0 0.0 4.0]
    @test positions ≈ [0.0 0.5; 0.0 0.5; 0.0 0.5]
    @test types == [1, 1]
end


text:
module SymbolicUtils

export @syms, term, @fun, showraw

# Sym, Term and other types
include("types.jl")

# Methods on symbolic objects
using SpecialFunctions, NaNMath
export cond
include("methods.jl")

# LinkedList, simplification utilities
include("utils.jl")

export Rewriters

# A library for composing together expr -> expr functions
include("rewriters.jl")

using .Rewriters

using Combinatorics: permutations, combinations
export @rule, @acrule, @arule, RuleSet

# Rule type and @rule macro
include("rule.jl")

# Matching a Rule
include("matchers.jl")

# Convert to an efficient multi-variate polynomial representation
import AbstractAlgebra.Generic: MPoly, PolynomialRing, ZZ, exponent_vector
using AbstractAlgebra: ismonomial, symbols
using DataStructures
include("abstractalgebra.jl")

# Term ordering
include("ordering.jl")

# Default rules for expression simplification
include("simplify_rules.jl")

# API = simplify + substitute
export simplify, substitute
include("api.jl")

end # module


text:
using CmdStan

N1 = 10
data1 = Dict(
  "Scalar" => rand(),
  "Single_element_vector" => [rand()],
  "Vector" => rand(N1),
  "Array" => rand(N1,3)
)

N = 100000
data = Dict(
  "Scalar" => rand(),
  "Single_element_vector" => [rand()],
  "Vector" => rand(N),
  "Array" => rand(N,3)
)

ProjDir = dirname(@__FILE__)
cd(ProjDir) do

  Stan.update_R_file("testdata0", data1)
  Stan.update_R_file_jon("testdata1", data1)

  @time Stan.update_R_file("testdata0", data)
  @time Stan.update_R_file_jon("testdata1", data)

  isfile("testdata0") && rm("testdata0");
  isfile("testdata1") && rm("testdata1");

end # cd


text:
using Waterfall
import Luxor
import Base

include(joinpath(WATERFALL_DIR,"bin","io.jl"))
# include(joinpath(WATERFALL_DIR,"src","utils","correlate.jl"))
# include(joinpath(WATERFALL_DIR,"src","utils","draw.jl"))

nstep = 3
nsample = 1
nperm = 5
ncor = 1000
N = 13
permute = false
permutations = [[collect(1:N)]; random_permutation(1:N, min(factorial(N),nperm))]

cascades = Dict()
heights = Dict()
plots = Dict()

for k in permutations
# k = permutations[1]
    x = Cascade(df; permutation=k, nsample=nsample, ncor=ncor, kwargs...)
    push!(cascades, k => x)

    # Calculate the value we will want to add to a NEW cascade. This will come from the
    # HEIGHT of the current cascade.
    v1, v2 = Waterfall.cumulative_v!(x; permute=permute)
    push!(heights, k => v2 .- v1)
end

# Make dictionary of heights into one cascade.
vals = convert(Matrix, [heights[k] for k in permutations])
data = Waterfall.collect_data(cascades[permutations[1]])

data = [Data( ; value=vals[:,ii], label=data[ii].label, sublabel=data[ii].sublabel) for ii in 1:length(data)]
cascade = Cascade( ; start=first(data), stop=last(data), steps=data[2:end-1], ncor=0)

pdata = Plot(copy(cascade); ylabel="Efficiency (%)")
# p = convert(Plot{Violin}, pdata, 1.0; permute=false, vmin=15., vmax=20.5)

# calculate_kde(v::Vector{T}) where T <: Real = KernelDensity.kde(v)

# Luxor.@png begin
#     Luxor.fontsize(18)
#     Luxor.setmatrix([1 0 0 1 LEFT_BORDER TOP_BORDER])

#     draw(p; distribution=distribution, samples=nsample)
# end WIDTH+LEFT_BORDER+RIGHT_BORDER HEIGHT+TOP_BORDER+BOTTOM_BORDER "many-cascades.png"

# df0 = copy(df)
# df0 = DataFrames.transform(Waterfall.fuzzify(df0; nsample=1, kwargs...))

# # df = DataFrames.hcat(df, df; makeunique=true)
# df = convert(DataFrames.DataFrame, cascade)[:,2:end-1]
# # df = DataFrames.rename.(df, Pair.(DataFrames.propertynames(df), Symbol.(df0[2:end-1,:Label])))
# df[!,:permutation] .= permutations
# df = DataFrames.stack(df, 1:size(df,2)-1)
# # df = df[df[:,:variable] .== "2",:]
# StatsPlots.@df df StatsPlots.violin(:variable, :value, linewidth=0)
# # @df iris groupedhist(:value, group = :variable, bar_position = :dodge)
# # # DataFrames.stack(df)


text:
#
# Copyright (c) 2021 Tobias Thummerer, Lars Mikelsons
# Licensed under the MIT license. See LICENSE file in the project root for details.
#

using FMI
using Flux
using DifferentialEquations: Tsit5

import Random
Random.seed!(5678);

t_start = 0.0
t_step = 0.1
t_stop = 3.0
tData = t_start:t_step:t_stop

# generate training data
realFMU = fmiLoad("BouncingBall1D", ENV["EXPORTINGTOOL"], ENV["EXPORTINGVERSION"])
realSimData = fmiSimulate(realFMU, t_start, t_stop; recordValues=["mass_s", "mass_v"], saveat=tData)
x0 = collect(realSimData.values.saveval[1])
@test x0 == [1.0, 0.0]

# setup traing data
posData = fmi2GetSolutionValue(realSimData, "mass_s")
velData = fmi2GetSolutionValue(realSimData, "mass_v")

# loss function for training
function losssum()
    global problem, x0, posData
    solution = problem(x0)

    posNet = fmi2GetSolutionState(solution, 1; isIndex=true)
    velNet = fmi2GetSolutionState(solution, 2; isIndex=true)

    Flux.Losses.mse(posNet, posData) + Flux.Losses.mse(velNet, velData)
end

# callback function for training
global iterCB = 0
global lastLoss = 0.0
function callb()
    global iterCB += 1
    global lastLoss

    if iterCB % 1 == 0
        loss = losssum()
        @info "Loss: $loss"

        # This test condition is weak, because when the FMU passes an event, the error might increase.
        # ToDo: More intelligent testing condition.
        @test (loss < lastLoss*2.0) && (loss != lastLoss)
        lastLoss = loss
    end
end

vr = fmi2StringToValueReference(realFMU, "mass_m")

numStates = fmiGetNumberOfStates(realFMU)

# some NeuralFMU setups
nets = []

# 1. default ME-NeuralFMU (learn dynamics and states, almost-neutral setup, parameter count << 100)
net = Chain(Dense( [1.0 0.0; 0.0 1.0] + rand(numStates,numStates)*0.01, zeros(numStates), identity),
            states ->  fmiEvaluateME(realFMU, states),
            Dense( [1.0 0.0; 0.0 1.0] + rand(numStates,numStates)*0.01, zeros(numStates), identity))
push!(nets, net)

# 2. default ME-NeuralFMU (learn dynamics)
net = Chain(states ->  fmiEvaluateME(realFMU, states),
            Dense(numStates, 16, tanh),
            Dense(16, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 3. default ME-NeuralFMU (learn states)
net = Chain(Dense(numStates, 16, identity),
            Dense(16, 16, identity),
            Dense(16, numStates),
            states -> fmiEvaluateME(realFMU, states))
push!(nets, net)

# 4. default ME-NeuralFMU (learn dynamics and states)
net = Chain(Dense(numStates, 16, leakyrelu),
            Dense(16, 16, leakyrelu),
            Dense(16, numStates),
            states -> fmiEvaluateME(realFMU, states),
            Dense(numStates, 16, tanh),
            Dense(16, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 5. NeuralFMU with hard setting time to 0.0
net = Chain(states ->  fmiEvaluateME(realFMU, states), # not supported by this FMU:   states ->  fmiEvaluateME(realFMU, states, 0.0),
            Dense(numStates, 8, tanh),
            Dense(8, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 6. NeuralFMU with additional getter
getVRs = [fmi2StringToValueReference(realFMU, "mass_m")]
numGetVRs = length(getVRs)
net = Chain(states ->  fmiEvaluateME(realFMU, states, realFMU.components[end].t, fmi2ValueReference[], Real[], getVRs),
            Dense(numStates+numGetVRs, 8, tanh),
            Dense(8, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 7. NeuralFMU with additional setter
setVRs = [fmi2StringToValueReference(realFMU, "mass_m")]
numSetVRs = length(setVRs)
net = Chain(states ->  fmiEvaluateME(realFMU, states, realFMU.components[end].t, setVRs, [1.1]),
            Dense(numStates, 8, tanh),
            Dense(8, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 8. NeuralFMU with additional setter and getter
net = Chain(states ->  fmiEvaluateME(realFMU, states, realFMU.components[end].t, setVRs, [1.1], getVRs),
            Dense(numStates+numGetVRs, 8, tanh),
            Dense(8, 16, tanh),
            Dense(16, numStates))
push!(nets, net)

# 9. Empty NeuralFMU
net = Chain(states ->  fmiEvaluateME(realFMU, states),
            Dense(ones(numStates, numStates), false,  identity))
push!(nets, net)

optim = ADAM(1e-4)
for i in 1:length(nets)
    @testset "Net setup #$i" begin
        global nets, problem, lastLoss, iterCB

        net = nets[i]
        problem = ME_NeuralFMU(realFMU, net, (t_start, t_stop), Tsit5(); saveat=tData)

        @test problem !== nothing

        solutionBefore = problem(x0)
        if solutionBefore.success
            @test length(solutionBefore.states.t) == length(tData)
            @test solutionBefore.states.t[1] == t_start
            @test solutionBefore.states.t[end] == t_stop
        end

        # train it ...
        p_net = Flux.params(problem)

        iterCB = 0
        lastLoss = losssum()
        @info "Start-Loss for net #$i: $lastLoss"

        # check results
        solutionAfter = problem(x0)
        if solutionAfter.success
            @test length(solutionAfter.states.t) == length(tData)
            @test solutionAfter.states.t[1] == t_start
            @test solutionAfter.states.t[end] == t_stop
        end
    end
end

fmiUnload(realFMU)


text:
using Base.Test
include("../src/calculations.jl")

@test one_prop_z_int(0.5, 100, 0.95) == ConfidenceInterval(0.402, 0.598)

@test one_prop_z_test(0.45, 0.5, 100, 0.95, "!=") == 0.3173
@test one_prop_z_test(0.5, 0.45, 100, 0.95, "!=") == 0.3149
@test one_prop_z_test(0.45, 0.5, 100, 0.95, "<") == 0.1587
@test one_prop_z_test(0.45, 0.5, 100, 0.95, ">") == 0.8413
@test one_prop_z_test(0.45, 0.5, 100, 0.95, "asdf") == -1

@test two_prop_z_int(0.5, 100, 0.7, 200, 0.95) == ConfidenceInterval(-0.3168, -0.0832)

@test two_prop_z_test(0.5, 100, 0.6, 200, 0.95, "!=") == 0.0994
@test two_prop_z_test(0.6, 200, 0.5, 100, 0.95, "!=") == 0.0994
@test two_prop_z_test(0.5, 100, 0.6, 200, 0.95, ">") == 0.9503
@test two_prop_z_test(0.5, 100, 0.6, 200, 0.95, "<") == 0.0497

@test z_interval(50, 5, 100, 0.95) == ConfidenceInterval(49.02, 50.98)
# n < 30, t-distribution should be used
@test z_interval(50, 5, 20, 0.95) == ConfidenceInterval(47.6599, 52.3401)

@test z_test(49, 50, 5, 100, 0.95, "!=") == 0.0455
@test z_test(50, 49, 5, 100, 0.95, "!=") == 0.0455
@test z_test(49, 50, 5, 100, 0.95, "<") == 0.0228
@test z_test(51, 50, 5, 100, 0.95, ">") == 0.0228

@test two_sample_z_interval(60, 5, 100, 50, 5, 100, 0.95) == ConfidenceInterval(8.6141, 11.3859)
# n1 < 30, t-distribution should be used
@test two_sample_z_interval(6, 1, 10, 50, 5, 100, 0.95) == ConfidenceInterval(-47.1528, -40.8472)

@test two_sample_z_test(51, 5, 100, 50, 5, 100, 0.95, ">") == 0.0786
@test two_sample_z_test(51, 5, 100, 50, 5, 100, 0.95, "<") == 0.9214
@test two_sample_z_test(51, 5, 100, 50, 5, 100, 0.95, "!=") == 0.1573
@test two_sample_z_test(50, 5, 100, 51, 5, 100, 0.95, "!=") == 0.1573

max_stars_repo_name: jackcook/statulator

max_issues_repo_name: jackcook/statulator

max_forks_repo_name: jackcook/statulator


text:
using KroneckerMatrices
using Base.Test

#square matrix nonsquare factors
Q,R = 3,4
S,T = 4,3
Aouter = rand(Q,R)
Ainner = ones(S,T)

#square matrix, square factors
Q,R = 3,3
S,T = 4,4
Bouter = rand(Q,R)
Binner = ones(S,T)

#tests for square Kronecker Products
for (outer,inner) in zip(Any[Aouter Bouter],Any[Ainner Binner])
    M = KronProdMat(outer,inner)
    Mfull = kronprod(outer,inner)
    @test size(M) == size(Mfull)
    K,L = size(M)
    for i = 1:K
        for j = 1:L
            @test M[i,j] == Mfull[i,j]
        end
    end
    @test full(M') == Mfull'
    @test rank(M) == rank(Mfull)
    @test det(M) == det(Mfull)

    fullsvdvals = sort(svdvals(Mfull),by = -)
    kronsvdvalstrunc = sort(full(svdvals(M)),by = -)
    kronsvdvals = zeros(eltype(M),length(fullsvdvals))
    kronsvdvals[1:length(kronsvdvalstrunc)] = kronsvdvalstrunc
    for (σ,τ) in zip(kronsvdvals,fullsvdvals)
        @test_approx_eq_eps(σ,τ,100*eps())
    end
end

#tests for square Kronecker products with square factors
B = KronProdMat(Bouter,Binner)
Bfull = kronprod(Bouter,Binner)

#U,Ufull = convert(Matrix{eltype(B)},eigvecs(B)),eigvecs(Bfull)
#Bsim,Bsimfull = U*U',Ufull*Ufull'
#N = length(B)
#for i = 1:N, j = 1:N
#    @test_approx_eq_eps(Bsim[i,j],Bsimfull[i,j],10*eps())
#end

#for now test sorted eigs, ordering is different for different situation in the full problem
#not sure what the right tolerance should be here (probably should try to compute)

#for (λ1,λ2) in zip(sort(vec(convert(Matrix{eltype(B)},eigvals(B))),by=real),sort(eigvals(Bfull),by=real))
#    @test_approx_eq_eps(λ1,λ2,10*eps())
#end


text:
module Visualization

export
    Widget, # from InteractBase
    PushRecoveryVisualizer

using MeshCat
using MeshCatMechanisms
using CoordinateTransformations
using NamedColors

import Observables
import InteractBase
import WebIO

using InteractBase: button, slider, vbox, pad, px, Widget, style, container
using Observables: Observable, observe
using ..QPWalkingControl: PushApplier, icp
using RigidBodyDynamics: FreeVector3D, MechanismState
using MeshCat: Visualizer
using GeometryTypes: HyperSphere, Point

function InteractBase.Widget(controller::PushApplier; max_force::Number, max_Δt::Number)
    pushbutton = button("Apply push")
    angleslider = slider(range(Float64(-π), stop=Float64(π), length=51), label="θ")
    magnitudeslider = slider(range(0.0, stop=max_force, length=51), label="mag")
    timeslider = slider(range(0, stop=max_Δt, length=51), label="Δt")

    let controller = controller
        Observables.on(pushbutton) do _
            controller.new_push = true
        end
        Observables.onany(angleslider, magnitudeslider) do θ, magnitude
            frame = controller.jacobian.frame
            s, c = sincos(θ)
            controller.force = FreeVector3D(frame, magnitude * c, magnitude * s, 0.0)
        end
        Observables.on(timeslider) do Δt
            controller.Δt = Δt
        end
    end

    angleslider[] = 0
    magnitudeslider[] = max_force / 2
    timeslider[] = max_Δt

    style(container(
        pad(10px, style(vbox(pushbutton, angleslider, magnitudeslider, timeslider), :fontFamily => "sans-serif", :fontSize => "10pt")),
    ), :width => "350px")
end

struct PushRecoveryVisualizer
    mvis::MechanismVisualizer
    icpvis::Visualizer
end

function PushRecoveryVisualizer(mvis::MechanismVisualizer)
    state = MeshCatMechanisms.state(mvis)
    vis = MeshCatMechanisms.visualizer(mvis)
    icpvis = vis[:icp]
    setobject!(icpvis, HyperSphere(Point(0., 0., 0.), 0.015), MeshPhongMaterial(color=colorant"greyish blue"))
    PushRecoveryVisualizer(mvis, icpvis)
end

function Base.copyto!(vis::PushRecoveryVisualizer, state::Union{MechanismState, AbstractVector})
    mvis = vis.mvis
    copyto!(mvis, state)
    settransform!(vis.icpvis, Translation(icp(MeshCatMechanisms.state(mvis)).v))
end

Base.wait(vis::PushRecoveryVisualizer) = Base.wait(vis.mvis)

MeshCatMechanisms.visualizer(vis::PushRecoveryVisualizer) = MeshCatMechanisms.visualizer(vis.mvis)

end # module


text:
# ---
# title: 692. Top K Frequent Words
# id: problem692
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Hash Table, Heap, Trie
# link: <https://leetcode.com/problems/top-k-frequent-words/description/>
# hidden: true
# ---
#
# Given a non-empty list of words, return the _k_ most frequent elements.
#
# Your answer should be sorted by frequency from highest to lowest. If two words
# have the same frequency, then the word with the lower alphabetical order comes
# first.
#
# **Example 1:**
#
#
#
#     Input: ["i", "love", "leetcode", "i", "love", "coding"], k = 2
#     Output: ["i", "love"]
#     Explanation: "i" and "love" are the two most frequent words.
#         Note that "i" comes before "love" due to a lower alphabetical order.
#
#
# **Example 2:**
#
#
#
#     Input: ["the", "day", "is", "sunny", "the", "the", "the", "sunny", "is", "is"], k = 4
#     Output: ["the", "is", "sunny", "day"]
#     Explanation: "the", "is", "sunny" and "day" are the four most frequent words,
#         with the number of occurrence being 4, 3, 2 and 1 respectively.
#
#
# **Note:**
#
#   1. You may assume _k_ is always valid, 1 ≤ _k_ ≤ number of unique elements.
#   2. Input words contain only lowercase letters.
#
# **Follow up:**
#
#   1. Try to solve it in _O_ ( _n_ log _k_ ) time and _O_ ( _n_ ) extra space.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
module Users

using SearchLight, SearchLight.Validation, UsersValidator
using SHA

export User

Base.@kwdef mutable struct User <: AbstractModel
  ### FIELDS
  id::DbId = DbId()
  username::String = ""
  password::String = ""
  name::String = ""
  email::String = ""
end

Validation.validator(u::Type{User}) = ModelValidator([
  ValidationRule(:username, UsersValidator.not_empty),
  ValidationRule(:username, UsersValidator.unique),
  ValidationRule(:password, UsersValidator.not_empty),
  ValidationRule(:email,    UsersValidator.not_empty),
  ValidationRule(:email,    UsersValidator.unique),
  ValidationRule(:name,     UsersValidator.not_empty)
])

function hash_password(password::String)
  sha256(password) |> bytes2hex
end

end

max_stars_repo_name: genieframework/GenieAuthentication

max_issues_repo_name: genieframework/GenieAuthentication

max_forks_repo_name: genieframework/GenieAuthentication


text:
#using PyPlot

ts_length = 10000
n = 1000
stds = []
means = []
for i in range(1, n)
  data = Array(Float64, ts_length)
  data = randn(ts_length)
  #epsilon_values = Array(Float64, ts_length)
  #plot(data, "b-")

  #println("Hello")
  #for i in 1:ts_length
    #println("Hi")
    #println(data[i])
  #end

  push!(stds, std(data))
  push!(means, mean(data))
end

print("mean in means: ")
println(mean(means))
print("mean in stds: ")
println(mean(stds))


text:
function nice_julian_type(type)
    @match t = type begin
        GuardBy(is_fn_ptr) => :FunctionPtr
        GuardBy(is_opaque_pointer) => t
        :(NTuple{$N,UInt8}) => :String
        :(NTuple{$N,$T}) => begin
            _N = @match N begin
                ::Symbol => :(Int($N))
                _ => N
            end
            :(NTuple{$_N,$(nice_julian_type(T))})
        end
        :Cstring => :String
        :VkBool32 => :Bool
        :(Ptr{$pt}) => nice_julian_type(pt)
        GuardBy(in(extension_types)) => :(vk.$t)
        GuardBy(is_vulkan_type) && GuardBy(!in(spec_enums.name)) => remove_vk_prefix(t)
        _ => @match _t = innermost_type(t) begin
            GuardBy(in(getproperty.(filter(!isnothing, spec_flags.bitmask), :name))) => begin
                bitmask_types = getproperty.(filter(!isnothing, spec_flags.bitmask), :name)
                i = findfirst(==(_t), bitmask_types)
                bitmask_flag_type(bitmask_types[i])
            end
            GuardBy(in(spec_flags.name)) && if !isnothing(flag_by_name(_t).bitmask)
            end => bitmask_flag_type(flag_by_name(_t).bitmask)
            _ => @match t begin
                GuardBy(in(spec_constants.name)) => follow_constant(t)
                _ => t
            end
        end
    end
end

"""
Return a new type easier to deal with.
"""
function nice_julian_type(spec::Spec)
    @match s = spec begin
        GuardBy(is_version) => :VersionNumber
        GuardBy(is_arr) => :(Vector{$(nice_julian_type(ptr_type(s.type)))})
        GuardBy(is_data) => :(Ptr{Cvoid})
        _ => nice_julian_type(s.type)
    end
end

function signature_type(type)
    @match type begin
        :UInt || :UInt8 || :UInt16 || :UInt32 || :UInt64 || :Int || :Int8 || :Int16 || :Int32 || :Int64 => :Integer
        :Float16 || :Float32 || :Float64 => :Real
        :String => :AbstractString
        :(Vector{$et}) => :(AbstractArray{<:$(signature_type(et))})
        t => t
    end
end

function relax_signature_type(type)
    @match type begin
        :(AbstractArray{<:$_}) => :AbstractArray
        t => t
    end
end

function relax_function_signature!(p::Dict)
    p[:args] = relax_function_signature(p[:args])
    p
end

function relax_function_signature(args::AbstractVector)
    map(args) do arg
        @match arg begin
            :($identifier::$type) => :($identifier::$(relax_signature_type(type)))
            _ => arg
        end
    end
end

bitmask_flag_type(type) = Symbol(replace(remove_vk_prefix(string(type)), "Bits" => ""))
bitmask_flag_type(spec::SpecBitmask) = bitmask_flag_type(spec.name)

is_fn_ptr(type) = startswith(string(type), "PFN")
is_opaque_pointer(type) = type == :(Ptr{Cvoid})


text:
# License for this file: MIT (expat)
# Copyright 2017-2018, DLR Institute of System Dynamics and Control
#
# This file is part of module
#   Modia3D.Composition (Modia3D/Composition/_module.jl)
#

@assembly Part(data::Modia3D.AbstractObject3Ddata, rFrames::AbstractVector; visualizeFrames::Union{Bool,Modia3D.Ternary}=Modia3D.Inherited) begin
   frame0 = Modia3D.Object3D(data)
   frames = [ Modia3D.Object3D(frame0; r = rFrame, visualizeFrame = visualizeFrames) for rFrame in rFrames ]
end

@assembly BodyWithTwoFrames(data::Modia3D.AbstractObject3Ddata, r_ab::AbstractVector; visualizeFrames::Union{Bool,Modia3D.Ternary}=Modia3D.Inherited) begin
   frame_a = Modia3D.Object3D(data, visualizeFrame = visualizeFrames)
   frame_b = Modia3D.Object3D(frame_a; r = r_ab, visualizeFrame = visualizeFrames)
end

@assembly ContactBox(parent::Object3D;
                     scale::AbstractVector = [1.0, 1.0, 1.0],
                     rsmall::Float64=0.001,
                     fixed::Bool = true,
                     r::AbstractVector = ModiaMath.ZeroVector3D,
                     R::Union{ModiaMath.RotationMatrix,NOTHING} = nothing,
                     q::Union{ModiaMath.Quaternion,NOTHING} = nothing,
                     v_start::AbstractVector = ModiaMath.ZeroVector3D,
                     w_start::AbstractVector = ModiaMath.ZeroVector3D,
                     visualizeFrame::Union{Modia3D.Ternary,Bool} = Modia3D.Inherited,
                     massProperties::Union{Solids.MassProperties,Number,AbstractString,Solids.SolidMaterial,NOTHING} = nothing,
                     material::Union{Graphics.Material,NOTHING} = Graphics.Material(),
                     contactMaterial::String = "",
                     ) begin
    #box    = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1]-2*rsmall, scale[2]-2*rsmall, scale[3]-2*rsmall; rsmall=rsmall),
    box    = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1]-2*rsmall, scale[2]-2*rsmall, scale[3]-2*rsmall; rsmall=rsmall),
                              massProperties,material; contactMaterial=contactMaterial),
                              fixed=fixed, r=r, R=R, q=q, v_start=v_start, w_start=w_start, visualizeFrame=visualizeFrame)
    sphere = Modia3D.Solid(Modia3D.SolidSphere(2*rsmall), nothing, Modia3D.Material(color="Red"); contactMaterial=contactMaterial)
    ball1  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall,  scale[2]/2-rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    ball2  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall,  scale[2]/2-rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    ball3  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall, -scale[2]/2+rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    ball4  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall, -scale[2]/2+rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    ball5  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall,  scale[2]/2-rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball6  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall,  scale[2]/2-rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball7  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall, -scale[2]/2+rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball8  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall, -scale[2]/2+rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
end

@assembly ContactBox2(parent::Object3D;
                     scale::AbstractVector = [1.0, 1.0, 1.0],
                     rsmall::Float64=0.001,
                     fixed::Bool = true,
                     r::AbstractVector = ModiaMath.ZeroVector3D,
                     R::Union{ModiaMath.RotationMatrix,NOTHING} = nothing,
                     q::Union{ModiaMath.Quaternion,NOTHING} = nothing,
                     v_start::AbstractVector = ModiaMath.ZeroVector3D,
                     w_start::AbstractVector = ModiaMath.ZeroVector3D,
                     visualizeFrame::Union{Modia3D.Ternary,Bool} = Modia3D.Inherited,
                     massProperties::Union{Solids.MassProperties,Number,AbstractString,Solids.SolidMaterial,NOTHING} = nothing,
                     material::Union{Graphics.Material,NOTHING} = Graphics.Material(),
                     contactMaterial::String = "",
                     ) begin
    #box    = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1]-2*rsmall, scale[2]-2*rsmall, scale[3]-2*rsmall; rsmall=rsmall),
    box    = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1]-2*rsmall, scale[2]-2*rsmall, scale[3]-2*rsmall; rsmall=rsmall),
                              massProperties,material; contactMaterial=contactMaterial),
                              fixed=fixed, r=r, R=R, q=q, v_start=v_start, w_start=w_start, visualizeFrame=visualizeFrame)
    sphere = Modia3D.Solid(Modia3D.SolidSphere(2*rsmall), nothing, Modia3D.Material(color="Red"); contactMaterial=contactMaterial)
    #ball1  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall,  scale[2]/2-rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    #ball2  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall,  scale[2]/2-rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    #ball3  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall, -scale[2]/2+rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    #ball4  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall, -scale[2]/2+rsmall,  scale[3]/2-rsmall], visualizeFrame=false)
    ball5  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall,  scale[2]/2-rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball6  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall,  scale[2]/2-rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball7  = Modia3D.Object3D(box, sphere, fixed=true, r=[ scale[1]/2-rsmall, -scale[2]/2+rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
    ball8  = Modia3D.Object3D(box, sphere, fixed=true, r=[-scale[1]/2+rsmall, -scale[2]/2+rsmall, -scale[3]/2+rsmall], visualizeFrame=false)
end

@assembly ContactBox3(parent::Object3D;
                     scale::AbstractVector = [1.0, 1.0, 1.0],
                     rsmall::Float64=0.001,
                     fixed::Bool = true,
                     r::AbstractVector = ModiaMath.ZeroVector3D,
                     R::Union{ModiaMath.RotationMatrix,NOTHING} = nothing,
                     q::Union{ModiaMath.Quaternion,NOTHING} = nothing,
                     v_start::AbstractVector = ModiaMath.ZeroVector3D,
                     w_start::AbstractVector = ModiaMath.ZeroVector3D,
                     visualizeFrame::Union{Modia3D.Ternary,Bool} = Modia3D.Inherited,
                     massProperties::Union{Solids.MassProperties,Number,AbstractString,Solids.SolidMaterial,NOTHING} = nothing,
                     material::Union{Graphics.Material,NOTHING} = Graphics.Material(),
                     contactMaterial::String = "",
                     ) begin
    #box    = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1]-2*rsmall, scale[2]-2*rsmall, scale[3]-2*rsmall; rsmall=rsmall),
    box = Modia3D.Object3D(parent, Modia3D.Solid(Modia3D.SolidBox(scale[1], scale[2], scale[3]),massProperties,material),
                           fixed=fixed, r=r, R=R, q=q, v_start=v_start, w_start=w_start, visualizeFrame=visualizeFrame)

    solidBox1 = Modia3D.Solid(Modia3D.SolidBox(scale[1]/2, scale[2]/2, scale[3]/2; rsmall=0.0),
                              nothing,Graphics.Material(color="Red", transparency=0.7), contactMaterial=contactMaterial)
    solidBox2 = Modia3D.Solid(Modia3D.SolidBox(scale[1]/2, scale[2]/2, scale[3]/2; rsmall=0.0),
                              nothing,Graphics.Material(color="Yellow", transparency=0.7), contactMaterial=contactMaterial)
    box1 = Modia3D.Object3D(box, solidBox1, r=[ scale[1]/4, scale[2]/4, scale[3]/4])
    box2 = Modia3D.Object3D(box, solidBox2, r=[-scale[1]/4, scale[2]/4, scale[3]/4])
    box3 = Modia3D.Object3D(box, solidBox1, r=[ scale[1]/4,-scale[2]/4, scale[3]/4])
    box4 = Modia3D.Object3D(box, solidBox2, r=[-scale[1]/4,-scale[2]/4, scale[3]/4])
    box5 = Modia3D.Object3D(box, solidBox1, r=[ scale[1]/4, scale[2]/4,-scale[3]/4])
    box6 = Modia3D.Object3D(box, solidBox2, r=[-scale[1]/4, scale[2]/4,-scale[3]/4])
    box7 = Modia3D.Object3D(box, solidBox1, r=[ scale[1]/4,-scale[2]/4,-scale[3]/4])
    box8 = Modia3D.Object3D(box, solidBox2, r=[-scale[1]/4,-scale[2]/4,-scale[3]/4])
 end


text:
using Rubin
using Tests
using Elliptic
using HypergeometricFunctions
using Polylogarithms
using SpecialFunctions

@test integrate(csc(x)^-1//2*(a+b*csc(x))*(A+B*csc(x)+C*csc(x)^2), x) == :(csc(x)^(1/2)*(-2*B*b+-2*C*a)*cos(x)+csc(x)^(1/2)*sin(x)^(1/2)*(-2*a*(A+-1C)+2*B*b)*Elliptic.E(-1//2*x+1//4*pi,2)+-1*csc(x)^(1/2)*sin(x)^(1/2)*(2*A*b+2*B*a+2//3*C*b)*Elliptic.F(-1//2*x+1//4*pi,2)+-2//3*C*b*csc(x)^3//2*cos(x))

max_stars_repo_path: test/4 Trig functions/4.6 Cosecant/4.6.4.2 (a+b csc)^m (d csc)^n (A+B csc+C csc^2).jl


text:
using LatPhysPlottingMakie
using Test

@testset "LatPhysPlottingMakie.jl" begin
    # Write your own tests here.
end


text:
"""Abstract cell type"""
abstract type AbstractCell end

"""Dimension of the cell"""
dim(c::AbstractCell) = throw(MethodError(dim,(typeof(c),)))

"""Cell comparison"""
==(a::AbstractCell, b::AbstractCell) = throw(MethodError(==,(typeof(a), typeof(b))))

"""
    faces(cell)

Get a collection of `cell` faces.
"""
faces(c::AbstractCell) = throw(MethodError(faces,(typeof(c),)))

"""
    hash(cell) -> UInt

Return a unique identifier of the `cell` of `UInt` type
"""
hash(c::AbstractCell) = throw(MethodError(hash,(typeof(c),)))

"""
    union(u, v)

Create a new cell from a combination of vertices of cells `u` and `v`.
"""
union(u::C, v::C) where {C<:AbstractCell} = throw(MethodError(union,(C,C)))

"""
    vertices(cell)

Get a collection of `cell` vertices.
"""
vertices(c::AbstractCell) = throw(MethodError(vertices,(typeof(c),)))

"""
    boundary(cell)

Calculate boundary chain of the `cell`.
"""
boundary(::Type{R}, c::AbstractCell) where {R} = throw(MethodError(boundary,(typeof(c),)))
boundary(σ::AbstractCell) = boundary(Int, σ)
(+)(ch::AbstractChain{IX,R}, e::Tuple{C,R}) where {C<:AbstractCell,IX,R} = ch + (hash(e[1]), e[2])

"""
    value(cell)

Return a value stored in the cell
"""
value(c::AbstractCell) = nothing
# valtype(c::AbstractCell) = Nothing

"""Abstract simplex type"""
abstract type AbstractSimplex <: AbstractCell end

"""
    indices(cell)

Get an array of `cell` indices.
"""
indices(c::AbstractSimplex) = throw(MethodError(indices,(typeof(c),)))


text:
import Aqua
@testset "Aqua.jl" begin
    @time Aqua.test_all(ThreadingUtilities)
end


text:
# # Introduction

# A very daunting thing to program is a feedback loop with parallelism. Parallelism introduces stochasticity and thus debugging a feedback loop, or the part which learns in such a system is painful. On the other hand, we have so many different kinds of systems which implement parallelism - processes, threads, GPUs, job schedulers, GRID, etc. And so one ends up writing non-reusable code a case by a case.

# A TaskMater is an abstraction for all of those issues through two critical concepts - Master and Learner. Master is a process which takes input values from a channel `Master.tasks` and evaluates in an arbitrary fashion/order and puts that in `Master.results` channel as a tuple (input, output). One makes a concrete implementation which uses processes, threads, GPUs, TPUs, job scheduler, etc. for evaluation. Or one can treat Master as some process which comes from a piece of experimental equipment, for example, from a multi-head scanning tunnelling microscope. Or one could try to find optimal parameters for plant growth where parallelism is very natural.

# The other concept is Learner which tries to learn from the Master by asking questions and receiving answers. Nature is that Master has received multiple questions, and he answers them in arbitrary order. Thus the Learner needs to be smart to optimize the objective. Again the Learner could be a computer program, animal (if you can teach them parallelism), or human pressing buttons.

# Particularly in the case of a computer program, there is quite a variety. There is a class of learners where the programmer had programmed all cases of how the system should behave. My Python colleagues make a very great example in the [adaptive package](https://github.com/python-adaptive/adaptive), which allows adaptive function evaluation for reducing computational needs to make a beautiful figure (see Adaptive.jl for a wrapper). Another type of learners had been brainstormed in [this reedit](https://www.reddit.com/r/dataisbeautiful/comments/b8vv2p/i_wrote_a_python_package_to_do_adaptive_sampling/). The other class which might gain traction is a machine-learned Learner, for example, a plant state recognition algorithm with some ML to optimize the growth.

# # Interface

# For what follows we need to load and execute:
using Distributed
addprocs(2)
# before loading TaskMaster. That would give us two workers to proceed.

# To load the package, install it from the Julia central registry and execute:

using TaskMaster

# further on, we assume that it is loaded.

# ## Learner

# To see how to implement Learner, let's consider IgnorantLearner from the package. To initiate it, we do:
learner = IgnorantLearner(1:4)

# The first part of the interface is asking part. To `ask!` a point we execute:

x1 = ask!(learner,4)
x2 = ask!(learner,2)

# x1 and x2 now would give us points which Learner thinks as most beneficial to reach the objective. The numbers 4 and 2 represent the input, for example, a random number in that purifying Learner from randomness. That is particularly useful when one wants to debug the Learner from output values alone (see debugging section).

# The second part of the interface is telling about masters evaluated points to Learner. Let's say that x2=3 and thus y2=9 represent the results of the evaluation. Then to tell Learner about them we do:

tell!(learner,(3,9))

# Which would affect the state of the Learner, giving us better predictions on where exciting things would happen even when x1 is still being evaluated. (That is the reason why ask is written with exclamation mark ask!). Ignorant Learner as the name implies would ignore this knowledge and would proceed with evaluating points given by iterator.

# ## Master

# To see how Master works, let's consider WorkMaster from the package. To initiate it, we need to define a function on all workers and then start Master:

@everywhere f(x) = x^2
master = WorkMaster(f)

# Now we can evaluate the function with specific values as simple as:

put!(master.tasks,3)
put!(master.tasks,5)

# And take out the results:

take!(master.results)

# As one can see, it is pretty apparent to make ThreadMaster and other different kinds of Master implementations.

# ## Loop

# The third and the final concept is the Loop which represents the execution of master on the Loop. That can be initiated as follows:

master = WorkMaster(f)
learner = IgnorantLearner(1:10)
loop = Loop(master,learner)

# Also if one wishes to follow the learning process, it is possible to pass a function `iterhook(::Loop)` is executed in every iteration with constructor `Loop(::Master,::Learner,iterhook(::Loop))`.

# Now the central part is the execution. If one knows the collection beforehand on what one wants to execute the Loop (so it is finite) one can do:

output1 = evaluate!(loop,1:4)

# Often, however, one wants to learn until some convergence criteria are being met. That one can also do by passing a stopping condition which is executed every time before a new point is asked:

output2 = evaluate!(loop,learner->learner.state==7,5:9)

# which will terminate when the Learner's state would be 7. A note is that evaluate would continue to execute the previous state of the Loop (thus exclamation mark !).

# ## Debugging Learner

# Let's imagine a situation where one had spent hours evaluating the function with a Learner. For some particular reason looking at the output, the Learner seems had misbehaved. The question then is how one could debug that?

# A way the package overcomes such pitfall is by ensuring a deterministic process of `evaluate!` function which communicates with Master. This is why it is crucial that `ask!` does not have stochasticity inside but if needed, takes that from the input. That allows us to replay the history and explore the Learner's state as it evolved.

# In TaskMaster that is implemented with `HisotyMaster` type. To see how to use it, let's apply it on the previous execution:
output = [output1...,output2...]
master = HistoryMaster(output,2)
# where 2 is the number of unresolved points which were allowed during the original Masters run. Now to repeat the history, we do:
learner = IgnorantLearner(1:10)
loop = Loop(master,learner,loop->@show loop.learner.state)
evaluate!(loop,learner->learner.state==7,1:9)
# As you can see the `iterhook(::Loop)` makes a lot of sense for debugging. Also, HistoryMaster could be useful to write tests for Learner, so when code changes, one would immediately see the effects of that.


text:
include("../src/create_hypergraph.jl")
include("../src/hypergraph_analysis.jl")

betti = [];
graphs = [];
for i = 1:20

    σA = rand()*5
    C = 0.9
    N = Int64(round(rand()*15+1))
    μ = rand()*3
    σr = rand()*1.5

    reg = 0
    max_dim  = 4

    A = random_communitymatrix(N, σA, C)
    r = random_growthvector(N, μ, σr)

    # H = assembly_hypergraph_GLV(A, r; method = "permanence", regularization = reg)
    H2 = assembly_hypergraph_GLV(A, r; method = "localstability", regularization = reg)

    # R = disassembly_hypergraph(H)

    betti_H = betti_hypergraph_ripscomplex(H; max_dim = max_dim)
    @show betti_H
    if betti_H != [1, 0, 0, 0, 0]
        @show "entered"
        graphs = [graphs, [H2]]
        betti = [betti, [betti]]
    end
end

H = read_hypergraph(joinpath(@__DIR__, "data/Empirical/Jonathan/H.dat"))
betti_hypergraph_ripscomplex_old(H)
CoexistHypergraph.betti_hypergraph_ripscomplex(H)

function betti_hypergraph_ripscomplex_old(H; max_dim = 3)
    # old function using Ripser library: https://github.com/mtsch/Ripser.jl#master
    if length(H) == 0
        return zeros(max_dim + 1)
    else

        G = hypergraph_subdivide(H; expansion = false)
        # build distance matrix to give as input to Ripser.
        M = 2.0*ones(length(H), length(H))
        [ M[g[1], g[2]] = 1 for g in G ]

        #call Ripser
        barcodes = ripser(M; dim_max = max_dim, threshold = 1.5)
        # interpret the outout. Barcodes is an array of tuples of dim = max_dim + 1.
        # All tuples where the second entry is Inf are the persistent ones.
        # Betti numbers just count the number of those persistent ones.
        betti = [ count(x -> x[2] .== Inf, barcodes[i]) for i in 1:length(barcodes)]

        return betti
    end
end

max_issues_repo_name: mtangulo/coexistenceholes

max_forks_repo_name: mtangulo/coexistenceholes


text:
abstract type Paragram{LANG} <: EmbeddingSystem{LANG} end

function init(::Type{Paragram})
    vectors = [("paragram_300_ws353",
                " Paragram-WS353 300 dimensional Paragram embeddings tuned on WordSim353 dataset. 1.7 GB download.",
                "8ed9a19f8bc400cdbca5dae7f024c0310a3e5711a46ba48036a7542614440721",
              "https://drive.google.com/uc?id=0B9w48e1rj-MOLVdZRzFfTlNsem8&export=download"),
                # "https://drive.google.com/file/d/0B9w48e1rj-MOLVdZRzFfTlNsem8/view"),
               ("paragram_300_sl999",
                "Paragram-SL999 300 dimensional Paragram embeddings tuned on SimLex999 dataset. 1.7 GB download.",
                "9a16adc7d620642f863278451db4c03a2646016440ccea7e30a37ba17868781d",
               "https://drive.google.com/uc?id=0B9w48e1rj-MOck1fRGxaZW1LU2M&export=download"),
]
#https://drive.google.com/file/d/0B9w48e1rj-MOLVdZRzFfTlNsem8/view?usp=sharing
    for (depname, description, sha, link) in vectors
        register(DataDep(depname,
                         """
                         Pretrained Paragram word embeddings.
                         Website: https://www.cs.cmu.edu/~jwieting/
                         Author: John Wieting
                         Year: 2015
                         Licence: Open Data Commons Public Domain Dedication and License (PDDL)

                         Paper:
                             John Wieting, Mohit Bansal, Kevin Gimpel, Karen Livescu, Dan Roth. From Paraphrase Database to Compositional Paraphrase Model and Back.

                         $description
                         """,
                         link,
                         sha,
                         fetch_method = google_download,
			 post_fetch_method = unpack))

            append!(language_files(Paragram{:en}), ["$(depname)/$(depname).txt"])

    end
end

_load_embeddings(::Type{<:Paragram}, embedding_file, max_vocab_size, keep_words) = _load_embeddings_csv(embedding_file, max_vocab_size, keep_words , ' ')


text:
#---------------------------------------------------------------
#---------------------------------------------------------------
# AbstractOperations
#---------------------------------------------------------------
### All T where {T <: AbstractOperation} are assumed by default to have the fields:
###     op which describes the operation being performed
###     args::Vector which contains the operands
#---------------------------------------------------------------
function Base.:(==)(x::AbstractOperation,y::AbstractOperation)
    (x.op == y.op) && (length(x.args) == length(y.args)) && all(isequal.(x.args,y.args))
end

AbstractOperation()
#---------------------------------------------------------------
#---------------------------------------------------------------
# Operations
struct Operation <: AbstractOpation
    op::Atom
    args::Vector
end

#---------------------------------------------------------------
#---------------------------------------------------------------
# SymbolicOperations
struct SymbolicOperation <: AbstractOpation
    op::Atom
    args::Vector
end


text:
export permuteloci!, permutesamples!, permutegenotypes!, permutealleles!
export strictshuffle, strictshuffle!

"""
    permuteloci!(data::PopData)
Edits `PopData` in place with loci permuted across populations within
the `.genodata` dataframe.
"""
@inline function permuteloci!(data::PopData)
    @inbounds @sync for locus in groupby(data.genodata, :locus)
        Base.Threads.@spawn begin
            shuffle!(locus.population)
        end
    end
    data
end

"""
    permutesamples!(data::PopData; meta::Bool = false)
Edits `PopData` in place with samples permuted across populations within
the `.genodata` dataframe. Since performance is important for many permutations,
the default is to only edit the `.genodata` table in place; use `meta = true`
if you also require the `.meta` dataframe edited in place.
"""
@inline function permutesamples!(data::PopData; meta::Bool = false)
    pops = shuffle(data.sampleinfo.population)

    if meta == true
        meta_pops = copy(pops)
        @inbounds for name in groupby(data.metadata, :name)
            @inbounds name.population .= pop!(meta_pops)
        end
    end
    @inbounds @sync for name in groupby(data.genodata, :name)
        Base.Threads.@spawn begin
            @inbounds name.population .= pop!(pops)
        end
    end
    data
end

@inline function permutesamples!(data::AbstractDataFrame, popnames::Vector{String})
    pops = shuffle(popnames)

    @inbounds #=@sync=# for name in groupby(data, :name)
        #Base.Threads.@spawn begin
            @inbounds name.population .= pop!(pops)
        #end
    end
    data
end

"""
    permutegenotypes!(data::PopData; by::String = "locus")
Edits `PopData` in place with genotypes permuted across individuals within
the `.genodata` dataframe. Use `by = "population"` to permute genotypes
within populations.
"""
@inline function permutegenotypes!(data::PopData; by::String = "locus")
    # establish mode of operation
    groupings = by == "locus" ? :locus : [:locus, :population]
    @inbounds @sync for grp in groupby(data.genodata, groupings)
        Base.Threads.@spawn begin
            grp.genotype .= strictshuffle!(grp.genotype)
        end
    end
    data
end

"""
    permutealleles!(data::PopData; ploidy::Union{Nothing, Int} = nothing, by::String = "locus")
Edits `PopData` in place with alleles permuted and reconstructed into genotypes
for each locus within the `.genodata` dataframe. Use `by = "population"`
to permute alleles within populations. If `ploidy` is not provided (default `ploidy = nothing`),
then ploidy will be identified from the PopData. If performance is important,
it would be best to identify ploidy in advance and set it to a specific integer.
"""
@inline function permutealleles!(data::PopData; ploidy::Union{Nothing, Int} = nothing, by::String = "locus")
    if ploidy === nothing
        length(data.metadata.ploidy) > 1 && error("This permutation method is not appropriate for mixed-ploidy data")
        ploidy = first(tmp)
    end

    # establish mode of operation
    groupings = by == "locus" ? :locus : [:locus, :population]

    @inbounds @sync for grp in groupby(data.genodata, groupings)
        Base.Threads.@spawn begin
            alle = shuffle(alleles(grp.genotype))
            new_genos = Tuple.(Base.Iterators.partition(alle, ploidy))
            (@view grp.genotype[.!ismissing.(grp.genotype)]) .= new_genos
        end
    end
    data
end

"""
    strictshuffle(x::T) where T <: AbstractArray
Shuffle only the non-missing values of a Vector and return a copy of the vector,
keeping the `missing` values at their original locations.
Use `strictshuffle!` to edit in-place instead of returning a copy.
"""
@inline function strictshuffle(x::T) where T <: AbstractArray
    y = copy(x)
    @inbounds shuffle!(@view y[.!ismissing.(y)])
    return y
end

"""
    strictshuffle!(x::T)! where T <: AbstractArray
Shuffle only the non-missing values of a Vector, keeping the
`missing` values at their original locations. Use `strictshuffle`
to return a copy instead of editing in-place.
"""
@inline function strictshuffle!(x::T) where T <: AbstractArray
    @inbounds shuffle!(@view x[.!ismissing.(x)])
    return x
end


text:
mutable struct Stream
    s::String
    pos::Int64
end

# These keywords denote end of blocks
block_delimiters = ["if", "in", "using"]
function block_delimiter_symbol(c::String)
    if c == "if"
        return :block_delimiter_if
    elseif c == "in"
        return :block_delimiter_in
    elseif c == "using"
        return :block_delimiter_using
    else
        error("$c : Not a block delimiter")
    end
end

word_delimiters = [' ', '\n', ',']
function delimiter_symbol(c::Char)
    if c == ' '
        return :delimiter_whitespace
    elseif c == '\n'
        return :delimiter_nl
    elseif c == ','
        return :delimiter_comma
    else
        error("Not a delimiter character.")
    end
end
function delimiter_symbol(c::String)
    if c == "//"
        return :delimiter_comment
    elseif c == ": "
        return :delimiter_colon
    else
        error("Not a delimiter string.")
    end
end

par_openers = ['(', '[', '{']
par_closers = [')', ']', '}']
string_delimiters = ['"'] # we're not recognizing single quotes at this stage

# is `closer` a matching parentheses to `opener`?
function is_matching_closer(opener::Char, closer::Char)
    return (opener, closer) == ('(',')') ||
        (opener, closer) == ('[',']') ||
        (opener, closer) == ('{','}')
end

function is_keyword(s::AbstractString)
    return s ∈ keywords
end
function is_word_delimiter(c::Char)
    return c ∈ word_delimiters
end

# primitive parser functions
function parse_peek(s::Stream)
    return s.s[s.pos]
end
function parse_next(s::Stream)
    s.pos += 1
    return s.s[s.pos-1]
end
function parse_eof(s::Stream)
    return s.pos>length(s.s)
end
function flush_and_indicate(s::Stream)
    return "$(s.s)\n" * " "^(s.pos-1) * "^"
end

# get the string until the next delimiter
function get_word(s::Stream)
    ret_str = ""
    # open parentheses and strings
    open_parentheses = Vector{Char}()
    in_string = false
    delimiter = :delimiter_eof # change this if we hit another delimiter
    while !parse_eof(s)
        c = parse_next(s)
        if length(open_parentheses) == 0 && !in_string && is_word_delimiter(c)
            # parentheses/strings are balanced and we are facing a word delimiter
            delimiter = delimiter_symbol(c)
            break
        end
        if c == '/' && parse_peek(s) == '/'
            # comment
            delimiter = delimiter_symbol("//")
            parse_next(s)
            break
        elseif length(open_parentheses) == 0 && c == ':' && !parse_eof(s) && parse_peek(s) == ' '
            # end of the prefix
            delimiter = delimiter_symbol(": ")
            parse_next(s)
            break
        elseif c ∈ par_closers
            is_matching_closer(open_parentheses[end], c) || error("Douglass: parse error: unbalanced parentheses.\n$(flush_and_indicate(s))")
            pop!(open_parentheses)
        elseif c ∈ par_openers
            push!(open_parentheses, c)
        elseif c ∈ string_delimiters
            in_string = !in_string
        end
        # append it
        ret_str = ret_str * c
    end
    if length(open_parentheses) > 0
        error("Douglass: parse error: unbalanced parentheses.\n$(flush_and_indicate(s))")
    end
    if in_string
        error("Douglass: parse error: unbalanced string quotes.\n$(flush_and_indicate(s))")
    end
    return ret_str, delimiter
end

function get_block(s::Stream)

    block_delimiter = :none
    ret_str = ""
    while block_delimiter == :none
        pos_before = s.pos
        str, word_delimiter = get_word(s)

        if str ∈ block_delimiters
            # end the block because we've hit a keyword
            block_delimiter = block_delimiter_symbol(str)
            # all these keywords expect something afterwards, so throw error if there's nothing
            if word_delimiter != :delimiter_whitespace
                error("Douglass: parse error: premature end of input after keyword `$(str)`.\n$(flush_and_indicate(s))")
            end
        elseif (word_delimiter == :delimiter_nl) || (word_delimiter == :delimiter_comment) || (word_delimiter == :delimiter_eof)
            # the following end the line
            block_delimiter = :block_delimiter_eol
            ret_str = ret_str * str
        elseif word_delimiter == :delimiter_colon
            block_delimiter = :block_delimiter_colon
            ret_str = ret_str * str
        elseif word_delimiter == :delimiter_comma
            block_delimiter = :block_delimiter_comma
            ret_str = ret_str * str
        elseif word_delimiter == :delimiter_whitespace
            ret_str = ret_str * str * " " # and keep going
        end
    end

    return ret_str, block_delimiter

end

function parse(s::Stream)

    delimiter = :none
    level = 0 # this makes sure that the order of blocks is ok

    # first block is either the prefix or the command
    str, delimiter = get_block(s)
    if delimiter == :block_delimiter_colon
        # it's the prefix
        prefix = parse_prefix(strip(str))
        # next one must be the main command
        str, delimiter = get_block(s)
        command, arguments = parse_main(strip(str))
    else
        # it's the main part of the command
        # no prefix
        prefix = Prefix(nothing, nothing)
        command, arguments = parse_main(strip(str))
    end

    if delimiter != :block_delimiter_eol
        # next one is either if/in/using or go to options
        if delimiter == :block_delimiter_comma
            # options
            str, delimiter = get_block(s)
            filter = nothing
            use = nothing
            options = parse_options(str)
            if delimiter != :block_delimiter_eol
                # we don't allow any extra block after the options
                error("Douglass: parse error: unexpected symbol.\n$(flush_and_indicate(s))")
            end
        else
            # if/in/using
            old_delimiter = delimiter
            str, delimiter = get_block(s)
            if old_delimiter == :block_delimiter_if
                filter = Meta.parse(str)
                use = nothing
            elseif old_delimiter == :block_delimiter_in
                error("Douglass: parse error: `in` block is currently not supported.")
            elseif old_delimiter == :block_delimiter_using
                filter = nothing
                use = str
            end
            # now the only valid delimiter is "," or EOL
            if delimiter == :block_delimiter_comma
                # options
                str, delimiter = get_block(s)
                options = parse_options(str)
                if delimiter != :block_delimiter_eol
                    # we don't allow any extra block after the options
                    error("Douglass: parse error: unexpected symbol.\n$(flush_and_indicate(s))")
                end
            elseif delimiter != :block_delimiter_eol
                error("Douglass: parse error: unexpected keyword.\n$(flush_and_indicate(s))")
            else
                # no options
                options = nothing
            end
        end
    else # no if/in/using or options after the main command
        filter = nothing
        use = nothing
        options = nothing
    end

    # println("***************************************************************")
    # println("Parser debug output:")
    # println("By:")
    # @show prefix.by
    # println("Sort:")
    # @show prefix.sort
    # println("Command:")
    # @show command
    # println("Arguments:")
    # @show arguments
    # println("Filter:")
    # @show filter
    # println("Using:")
    # @show use
    # println("Options:")
    # @show options
    # println("***************************************************************")

    return Command(prefix.by, prefix.sort, command, arguments, filter, use, options)

end

function parse_prefix(str::AbstractString)
    s = Stream(strip(str),1)

    byvars = Vector{Symbol}()
    sortvars = Vector{Symbol}()

    str, delimiter = get_word(s)
    if str == "bysort"
        while delimiter == :delimiter_whitespace
            str, delimiter = get_word(s)
            if str[1] == '('
                # we're entering the sort part
                # assert format (<varlist>)
                (str[end] == ')') || error("Douglass: parse error: error parsing prefix. expecting ')'.\n$(flush_and_indicate(s))")
                vars = split(str[2:end-1]," ")
                sortvars = Symbol.(stripcolon.(vars[.!isempty.(vars)]))
                (delimiter == :delimiter_eof) || error("Douglass: parse error: error parsing prefix. expected end of line after ')'.\n$(flush_and_indicate(s))")
                break
            else
                # we're still in the 'by' part
                !isempty(strip(str)) && push!(byvars, Symbol(stripcolon(strip(str))))
            end
        end
        # Stata allows bysort without variables to sort by... not nice, but ok.
        # !isempty(sortvars) || error("Douglass: parse error: error parsing prefix. `bysort` expects a list of variables to sort by.\n$(flush_and_indicate(s))")

    elseif str == "by"
        (delimiter == :delimiter_whitespace) || error("Douglass: parse error: error parsing prefix. expecting list of variable names.\n$(flush_and_indicate(s))")
        while delimiter == :delimiter_whitespace
            str, delimiter = get_word(s)
            (str[1] != '(') || error("Douglass: parse error: `by` does not allow sorting. use `bysort`.\n$(flush_and_indicate(s))")
            !isempty(strip(str)) && push!(byvars, Symbol(stripcolon(strip(str))))
        end
    else
        error("Douglass: parse error: error parsing prefix. prefix must be `by` or `bysort`.\n$(flush_and_indicate(s))")
    end
    return Prefix(byvars, sortvars)
end
# This function parses the <command> <arguments> part of the string.
# if the arguments are a sequence of strings separated by whitespaces where each of the strings starts with a ':', it's interpreted
# as a varlist and returned as a Vector{Symbol}, otherwise as an Expr
function parse_main(str::AbstractString)

    s = split(strip(str), " ", limit=2)
    if length(s)==1
        # command only
        command = s[1]
        arguments = nothing
    else
        # command + arguments
        command = s[1]
        # remove all empty strings
        args = split(strip(s[2]), " ")
        args = args[.!(isempty.(args))]
        if all([args[i][1] == ':' for i=1:size(args,1)])
            # looks like it's a varlist... parse as Vector{Symbol}
            arguments = Symbol.([args[i][2:end] for i=1:size(args,1)])
        else
            # parse as Expr
            arguments = Meta.parse(s[2])
        end
    end

    return command, arguments
end

# takes an option string of the form "foo" or "foo(bar)" and parses it as a key, val pair
function parse_option(str::AbstractString)
    s = Stream(str,1)
    outkey = ""
    outval = ""
    inval = false
    while !parse_eof(s)
        c = parse_next(s)
        if c == '('
            inval = true
        elseif (inval==false) && (c == ')')
            error("Parse error: unexpected symbol encountered while parsing options.\n$(flush_and_indicate(s))")
        elseif (inval==true) && (c == ')')
            !parse_eof(s) && error("Parse error: unexpected symbol encountered while parsing options.\n$(flush_and_indicate(s))")
        elseif inval == false
            outkey = outkey * c
        elseif inval == true
            outval = outval * c
        end
    end
    if length(outkey)==0
        error("Parse error: invalid option string.\n$(flush_and_indicate(s))")
    end
    if length(outval)==0
        # no value, set to "true"
        outval = "true"
    end
    return outkey, outval
end

# options are being parsed in the following way:
#   `foo` results in an entry "foo" => true
#   `foo(bar)` results in an entry "foo" => Meta.parse(bar), unless `bar` is a sequence of
#   symbols separated by spaces, then they get parsed as `Vector{Symbol}``
function parse_options(str::AbstractString)
    s = Stream(str,1)
    out = Dict{String, Any}()
    while !parse_eof(s)
        word, del = get_word(s)
        if !isempty(strip(word))
            k, vs = parse_option(strip(word))
            # better parsing of the stuff inside the bracket: Vector{Symbol} if possible
            args = split(strip(vs), " ")
            args = args[.!(isempty.(args))]
            if all([args[i][1] == ':' for i=1:size(args,1)])
                # looks like it's a varlist... parse as Vector{Symbol}
                v = Symbol.([args[i][2:end] for i=1:size(args,1)])
            else
                # parse as Expr
                v = Meta.parse(vs)
            end
            push!(out, k => v)

        end
        if (del == :delimiter_colon) || (del == :delimiter_comma)
            error("Parse error: unexpected delimiter in options.\n$(flush_and_indicate(s))")
        end
    end
    return out
end

# This evaluates the $$ in the Stream.
# I hate this solution, but it's the best I could come up with.
function interp(s::Stream)

    ret_string = ""
    while !parse_eof(s)
        c = parse_next(s)
        if (c == '$') && !parse_eof(s) && parse_peek(s) == '$'
            !parse_eof(s) || error("Interpolation error: unexpected end of string.")
            parse_next(s)
            if (parse_peek(s) == '(')
                # go until next ')'
                parse_next(s)
                m = ""
                while true
                    c2 = parse_next(s)
                    if c2 == ')'
                        break
                    else
                        m = m * c2
                    end
                    !parse_eof(s) || error("Interpolation error: unexpected end of string.")
                end
            else
                # go until next whitespace
                m = ""
                while true
                    c2 = parse_next(s)
                    if c2 == ' '
                        break
                    else
                        m = m * c2
                    end
                    !parse_eof(s) || break
                end
            end
            ret_string = ret_string * string(Base.eval(Main,Symbol(m))) * ' '
        else
            ret_string = ret_string * c
        end
    end
    return ret_string
end

# This is supposed to be an simple parser for commands.
#
# The standard form is
#   [prefix]: <cmd> [expr] [if] [in] [using <filename>] [= <expr>] [, <options>]
#
#
#  Standard Stata syntax is
# cmd [varlist | namelist | anything]
# [if]
# [in]
# [using filename]
# [= exp]
# [weight]
# [, options]
#
# Stuff before the first ": " is considered the prefix
# Stuff after the first
function parse(str::AbstractString)

    # First interpolate the $ (or $() ) expressions
    # I hate it, but I have not found a solution other than eval (or globals)
    stream = Stream(str,1)
    s_after_interpolation = interp(stream)
    #println("After \$ parsing: $s_after_interpolation")

    stream = Stream(s_after_interpolation, 1)
    cmd = parse(stream)
end

"""
parse_varlist()

Parses an `AbstractString` into a `Vector{Symbol}`. The variables are assumed to be prefixed by colons and separated by whitespaces,
e.g. in the format ":variable1 :variable2 :variable3".

Throws an error if that's not the format

Arguments:
    str::AbstractString: string to be parsed
"""
function parse_varlist(str::AbstractString)

    s = strip(str)
    vars = split(s," ")
    vars = vars[.!isempty.(vars)]
    # check that all whitespace-separated strings start with a colon
    all([vars[i][1] for i=1:length(vars)] .== ':') || error("Parse error: invalid varlist.")
    out = Symbol.(stripcolon.(vars))
    return out

end

# macro d_str(ex)
#     println("Entering d_str:")
#     println("$(typeof(ex))")

#     #return Expr(:macrocall, Symbol("@$(ex)"), "")
# end

# d"replace x = 1 if y == 2"

# d"""

# """

# s = Stream("bysort mygroup (mysort): gen x = 5",1)

# # Stata prefix syntax
# struct Prefix
#     prefix::Symbol
#     varlist::Vector{Symbol}

# end

# # Stata command format:
# #  [prefix :] command [varlist] [=exp] [if] [in] [weight] [using filename] [, options]
# # to this we add [frame] at the start, so that it becomes
# # [frame] [prefix :] command [varlist] [=exp] [if] [in] [weight] [using filename] [, options]
# struct Command
#     frame::DataFrame
#     prefix::Prefix
#     command::Function
#     varlist::Vector{Symbol}

#     end
# end


text:
using GalacticOptim, GalacticOptimJL, GalacticFlux, Test
using ForwardDiff, Zygote, ReverseDiff, FiniteDiff, Tracker
using ModelingToolkit
x0 = zeros(2)
rosenbrock(x, p=nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
l1 = rosenbrock(x0)

function g!(G, x)
    G[1] = -2.0 * (1.0 - x[1]) - 400.0 * (x[2] - x[1]^2) * x[1]
    G[2] = 200.0 * (x[2] - x[1]^2)
end

function h!(H, x)
    H[1, 1] = 2.0 - 400.0 * x[2] + 1200.0 * x[1]^2
    H[1, 2] = -400.0 * x[1]
    H[2, 1] = -400.0 * x[1]
    H[2, 2] = 200.0
end

G1 = Array{Float64}(undef, 2)
G2 = Array{Float64}(undef, 2)
H1 = Array{Float64}(undef, 2, 2)
H2 = Array{Float64}(undef, 2, 2)

g!(G1, x0)
h!(H1, x0)

cons = (x, p) -> [x[1]^2 + x[2]^2]
optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoModelingToolkit(), cons=cons)
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoModelingToolkit(), nothing, 1)
optprob.grad(G2, x0)
@test G1 == G2
optprob.hess(H2, x0)
@test H1 == H2
@test optprob.cons(x0) == [0.0]
J = Array{Float64}(undef, 2)
optprob.cons_j(J, [5.0, 3.0])
@test J == [10.0, 6.0]
H3 = [Array{Float64}(undef, 2, 2)]
optprob.cons_h(H3, x0)
@test H3 == [[2.0 0.0; 0.0 2.0]]

function con2_c(x, p)
    [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]]
end
optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoModelingToolkit(), cons=con2_c)
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoModelingToolkit(), nothing, 2)
optprob.grad(G2, x0)
@test G1 == G2
optprob.hess(H2, x0)
@test H1 == H2
@test optprob.cons(x0) == [0.0, 0.0]
J = Array{Float64}(undef, 2, 2)
optprob.cons_j(J, [5.0, 3.0])
@test all(isapprox(J, [10.0 6.0; -0.149013 -0.958924]; rtol=1e-3))
H3 = [Array{Float64}(undef, 2, 2), Array{Float64}(undef, 2, 2)]
optprob.cons_h(H3, x0)
@test H3 == [[2.0 0.0; 0.0 2.0], [-0.0 1.0; 1.0 0.0]]

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoModelingToolkit(true, true), cons=con2_c)
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoModelingToolkit(true, true), nothing, 2)
using SparseArrays
sH = sparse([1, 1, 2, 2], [1, 2, 1, 2], zeros(4))
optprob.hess(sH, x0)
@test sH == H2
@test optprob.cons(x0) == [0.0, 0.0]
sJ = sparse([1, 1, 2, 2], [1, 2, 1, 2], zeros(4))
optprob.cons_j(sJ, [5.0, 3.0])
@test all(isapprox(sJ, [10.0 6.0; -0.149013 -0.958924]; rtol=1e-3))
sH3 = [sparse([1,2], [1, 2], zeros(2)), sparse([1, 1, 2], [1, 2, 1], zeros(3))]
optprob.cons_h(sH3, x0)
@test Array.(sH3) == [[2.0 0.0; 0.0 2.0], [-0.0 1.0; 1.0 0.0]]

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoForwardDiff())
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoForwardDiff(), nothing)
optprob.grad(G2, x0)
@test G1 == G2
optprob.hess(H2, x0)
@test H1 == H2

prob = OptimizationProblem(optprob, x0)

sol = solve(prob, Optim.BFGS())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.Newton())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.KrylovTrustRegion())
@test 10 * sol.minimum < l1

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoZygote())
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoZygote(), nothing)
optprob.grad(G2, x0)
@test G1 == G2
optprob.hess(H2, x0)
@test H1 == H2

prob = OptimizationProblem(optprob, x0)

sol = solve(prob, Optim.BFGS())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.Newton())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.KrylovTrustRegion())
@test 10 * sol.minimum < l1

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoReverseDiff())
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoReverseDiff(), nothing)
optprob.grad(G2, x0)
@test G1 == G2
optprob.hess(H2, x0)
@test H1 == H2

prob = OptimizationProblem(optprob, x0)
sol = solve(prob, Optim.BFGS())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.Newton())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.KrylovTrustRegion())
@test 10 * sol.minimum < l1

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoTracker())
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoTracker(), nothing)
optprob.grad(G2, x0)
@test G1 == G2
@test_throws ErrorException optprob.hess(H2, x0)

prob = OptimizationProblem(optprob, x0)

sol = solve(prob, Optim.BFGS())
@test 10 * sol.minimum < l1

@test_throws ErrorException solve(prob, Newton())

optf = OptimizationFunction(rosenbrock, GalacticOptim.AutoFiniteDiff())
optprob = GalacticOptim.instantiate_function(optf, x0, GalacticOptim.AutoFiniteDiff(), nothing)
optprob.grad(G2, x0)
@test G1 ≈ G2 rtol = 1e-6
optprob.hess(H2, x0)
@test H1 ≈ H2 rtol = 1e-6

prob = OptimizationProblem(optprob, x0)
sol = solve(prob, Optim.BFGS())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.Newton())
@test 10 * sol.minimum < l1

sol = solve(prob, Optim.KrylovTrustRegion())
@test sol.minimum < l1 #the loss doesn't go below 5e-1 here

sol = solve(prob, Flux.ADAM(0.1), maxiters=1000)
@test 10 * sol.minimum < l1


text:
using BinaryBuilder

name = "LinuxKernelHeaders"
version = v"4.12"

# sources to build, such as glibc, linux kernel headers, our patches, etc....
sources = [
	"https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.12.tar.xz" =>
	"a45c3becd4d08ce411c14628a949d08e2433d8cdeca92036c7013980e93858ab",
]

# Bash recipe for building across all platforms
script = raw"""
## Function to take in a target such as `aarch64-linux-gnu`` and spit out a
## linux kernel arch like "arm64".
target_to_linux_arch()
{
    case "$1" in
        arm*)
            echo "arm"
            ;;
        aarch64*)
            echo "arm64"
            ;;
        powerpc*)
            echo "powerpc"
            ;;
        i686*)
            echo "x86"
            ;;
        x86*)
            echo "x86"
            ;;
    esac
}

## sysroot is where most of this stuff gets plopped
sysroot=${prefix}/${target}/sys-root

# First, install kernel headers
cd $WORKSPACE/srcdir/linux-*/

# The kernel make system can't deail
KERNEL_FLAGS="ARCH=\\\"$(target_to_linux_arch ${target})\\\" CROSS_COMPILE=\\\"/opt/${target}/bin/${target}-\\\" HOSTCC=\\\"${HOSTCC}\\\""
echo $KERNEL_FLAGS

eval make ${KERNEL_FLAGS} mrproper V=1
eval make ${KERNEL_FLAGS} headers_check V=1
eval make ${KERNEL_FLAGS} INSTALL_HDR_PATH=${sysroot}/usr V=1 headers_install
"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line
platforms = [p for p in supported_platforms() if Sys.islinux(p)]

# The products that we will ensure are always built
products(prefix) = [
    FileProduct(prefix, "\${target}/sys-root/usr/include/linux/types.h", :types_h),
]

# Dependencies that must be installed before this package can be built
dependencies = [
]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies)