text:
abstract type StructType end

"Default `JSON3.StructType` for types that don't have a `StructType` defined; this ensures any object going in/out of JSON3 has an explicit `StructType`"
struct NoStructType <: StructType end

# "Data" type: fields are json keys, field values are json values; use names, omitempties, excludes
abstract type DataType <: StructType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.Struct()

This `StructType` is less flexible, yet more performant than `JSON3.Mutable`. For reading a `JSON3.Struct()` from a JSON string input, each key-value pair is read in the order it is encountered in the JSON input, the keys are ignored, and the values are directly passed to the type at the end of the object parsing like `MyType(val1, val2, val3)`. Yes, the JSON specification says that Objects are specifically ***un-ordered*** collections of key-value pairs, but the truth is that many JSON libraries provide ways to maintain JSON Object key-value pair order when reading/writing. Because of the minimal processing done while parsing, and the "trusting" that the Julia type constructor will be able to handle fields being present, missing, or even extra fields that should be ignored, this is the fastest possible method for mapping a JSON input to a Julia structure. If your workflow interacts with non-Julia APIs for sending/receiving JSON, you should take care to test and confirm the use of `JSON3.Struct()` in the cases mentioned above: what if a field is missing when parsing? what if the key-value pairs are out of order? what if there extra fields get included that weren't anticipated? If your workflow is questionable on these points, or it would be too difficult to account for these scenarios in your type constructor, it would be better to consider the `JSON3.Mutable()` option.
"""
struct Struct <: DataType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.Mutable()

The slightly less performant, yet much more robust method for directly mapping Julia struct fields to JSON objects is via `JSON3.Mutable()`. This technique requires your Julia type to be defined, ***at a minimum***, like:
```julia
mutable struct MyType
    field1
    field2
    field3
    # etc.

    MyType() = new()
end
```
Note specifically that we're defining a `mutable struct` to allow field mutation, and providing a `MyType() = new()` inner constructor which constructs an "empty" `MyType` where isbits fields will be randomly initialized, and reference fields will be `#undef`. (Note that the inner constructor doesn't need to be ***exactly*** this, but at least needs to be callable like `MyType()`. If certain fields need to be intialized or zeroed out for security, then this should be accounted for in the inner constructor). For these mutable types, the type will first be initizlied like `MyType()`, then JSON parsing will parse each key-value pair in a JSON object, setting the field as the key is encountered, and converting the JSON value to the appropriate field value. This flow has the nice properties of: allowing parsing success even if fields are missing in the JSON structure, and if "extra" fields exist in the JSON structure that aren't apart of the Julia struct's fields, it will automatically be ignored. This allows for maximum robustness when mapping Julia types to arbitrary JSON objects that may be generated via web services, other language JSON libraries, etc.

There are a few additional helper methods that can be utilized by `JSON3.Mutable()` types to hand-tune field reading/writing behavior:

* `JSON3.names(::Type{MyType}) = ((:field1, :json1), (:field2, :json2))`: provides a mapping of Julia field name to expected JSON object key name. This affects both reading and writing. When reading the `json1` key, the `field1` field of `MyType` will be set. When writing the `field2` field of `MyType`, the JSON key will be `json2`.
* `JSON3.excludes(::Type{MyType}) = (:field1, :field2)`: specify fields of `MyType` to ignore when reading and writing, provided as a `Tuple` of `Symbol`s. When reading, if `field1` is encountered as a JSON key, it's value will be read, but the field will not be set in `MyType`. When writing, `field1` will be skipped when writing out `MyType` fields as key-value pairs.
* `JSON3.omitempties(::Type{MyType}) = (:field1, :field2)`: specify fields of `MyType` that shouldn't be written if they are "empty", provided as a `Tuple` of `Symbol`s. This only affects writing. If a field is a collection (AbstractDict, AbstractArray, etc.) and `isempty(x) === true`, then it will not be written. If a field is `#undef`, it will not be written. If a field is `nothing`, it will not be written.
"""
struct Mutable <: DataType end

StructType(u::Union) = Struct()
StructType(::Type{Any}) = Struct()
StructType(::Type{T}) where {T} = NoStructType()
StructType(x::T) where {T} = StructType(T)

"""
    JSON3.names(::Type{MyType}) = ((:field1, :json1), (:field2, :json2))

Provides a mapping of Julia field name to expected JSON object key name.
This affects both reading and writing.
When reading the `json1` key, the `field1` field of `MyType` will be set.
When writing the `field2` field of `MyType`, the JSON key will be `json2`.
"""
function names end

# maps Julia struct field name to json key name: ((:field1, :json1), (:field2, :json2))
names(x::T) where {T} = names(T)
names(::Type{T}) where {T} = ()

Base.@pure function julianame(names::Tuple{Vararg{Tuple{Symbol, Symbol}}}, jsonname::Symbol)
    for nm in names
        nm[2] === jsonname && return nm[1]
    end
    return jsonname
end

Base.@pure function jsonname(names::Tuple{Vararg{Tuple{Symbol, Symbol}}}, julianame::Symbol)
    for nm in names
        nm[1] === julianame && return nm[2]
    end
    return julianame
end

"""
    JSON3.excludes(::Type{MyType}) = (:field1, :field2)

Specify for a `JSON3.Mutable` `StructType` the fields, given as a `Tuple` of `Symbol`s, that should be ignored when reading, and excluded from writing.
"""
function excludes end
# Julia struct field names as symbols that will be ignored when reading, and never written
excludes(x::T) where {T} = excludes(T)
excludes(::Type{T}) where {T} = ()

"""
    JSON3.omitempties(::Type{MyType}) = (:field1, :field2)

Specify for a `JSON3.Mutable` `StructType` the fields, given as a `Tuple` of `Symbol`s, that should not be written if they're considered "empty".
If a field is a collection (AbstractDict, AbstractArray, etc.) and `isempty(x) === true`, then it will not be written. If a field is `#undef`, it will not be written. If a field is `nothing`, it will not be written.
"""
function omitempties end

# Julia struct field names as symbols
omitempties(x::T) where {T} = omitempties(T)
omitempties(::Type{T}) where {T} = ()

"""
    JSON3.keywordargs(::Type{MyType}) = (field1=(dateformat=dateformat"mm/dd/yyyy",), field2=(dateformat=dateformat"HH MM SS",))

Specify for a `JSON3.Mutable` `StructType` the keyword arguments by field, given as a `NamedTuple` of `NamedTuple`s, that should be passed
to the `JSON3.construct` method when deserializing `MyType`. This essentially allows defining specific keyword arguments you'd like to be passed for each field
in your struct. Note that keyword arguments can be passed when reading, like `JSON3.read(source, MyType; dateformat=...)` and they will be passed down to each `JSON3.construct` method.
`JSON3.keywordargs` just allows the defining of specific keyword arguments per field.
"""
function keywordargs end

keywordargs(x::T) where {T} = keywordargs(T)
keywordargs(::Type{T}) where {T} = NamedTuple()

"""
    JSON3.JSONType

An abstract type used in the API for "interface types" to map Julia types to a specific JSON type. See docs for the following for more details:

    * JSON3.ObjectType
    * JSON3.ArrayType
    * JSON3.StringType
    * JSON3.NumberType
    * JSON3.BoolType
    * JSON3.NullType
"""
abstract type JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.ObjectType()

Declaring my type is `JSON3.ObjectType()` means it should map to a JSON object of unordered key-value pairs, where keys are `Symbol` or `String`, and values are any other type (or `Any`).

Types already declared as `JSON3.ObjectType()` include:
  * Any subtype of `AbstractDict`
  * Any `NamedTuple` type
  * Any `Pair` type

So if your type subtypes `AbstractDict` and implements its interface, then JSON reading/writing should just work!

Otherwise, the interface to satisfy `JSON3.ObjectType()` for reading is:

  * `MyType(x::Dict{Symbol, Any})`: implement a constructor that takes a `Dict{Symbol, Any}` of key-value pairs parsed from JSON
  * `JSON3.construct(::Type{MyType}, x::Dict; kw...)`: alternatively, you may overload the `JSON3.construct` method for your type if defining a constructor is undesirable (or would cause other clashes or ambiguities)

The interface to satisfy for writing is:

  * `pairs(x)`: implement the `pairs` iteration function (from Base) to iterate key-value pairs to be written out to JSON
  * `JSON3.keyvaluepairs(x::MyType)`: alternatively, you can overload the `JSON3.keyvaluepairs` function if overloading `pairs` isn't possible for whatever reason
"""
struct ObjectType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.ArrayType()

Declaring my type is `JSON3.ArrayType()` means it should map to a JSON array of ordered elements, homogenous or otherwise.

Types already declared as `JSON3.ArrayType()` include:
  * Any subtype of `AbstractArray`
  * Any subtype of `AbstractSet`
  * Any `Tuple` type

So if your type already subtypes these and satifies the interface, things should just work.

Otherwise, the interface to satisfy `JSON3.ArrayType()` for reading is:

  * `MyType(x::Vector)`: implement a constructo that takes a `Vector` argument of values and constructs a `MyType`
  * `JSON3.construct(::Type{MyType}, x::Vector; kw...)`: alternatively, you may overload the `JSON3.construct` method for your type if defining a constructor isn't possible
  * Optional: `Base.IteratorEltype(::Type{MyType})` and `Base.eltype(x::MyType)`: this can be used to signal to JSON3 that elements for your type are expected to be a single type and JSON3 will attempt to parse as such

The interface to satisfy for writing is:

  * `iterate(x::MyType)`: just iteration over each element is required; note if you subtype `AbstractArray` and define `getindex(x::MyType, i::Int)`, then iteration is already defined for your type
"""
struct ArrayType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.StringType()

Declaring my type is `JSON3.StringType()` means it should map to a JSON string value.

Types already declared as `JSON3.StringType()` include:
  * Any subtype of `AbstractString`
  * The `Symbol` type
  * Any subtype of `Enum` (values are written with their symbolic name)
  * The `Char` type

So if your type is an `AbstractString` or `Enum`, then things should already work.

Otherwise, the interface to satisfy `JSON3.StringType()` for reading is:

  * `MyType(x::String)`: define a constructor for your type that takes a single String argument
  * `JSON3.construct(::Type{MyType}, x::String; kw...)`: alternatively, you may overload `JSON3.construct` for your type
  * `JSON3.construct(::Type{MyType}, ptr::Ptr{UInt8}, len::Int; kw...)`: another option is to overload `JSON3.construct` with pointer and length arguments, if it's possible for your custom type to take advantage of avoiding the full string materialization; note that your type should implement both `JSON3.construct` methods, since JSON strings with escape characters in them will be fully unescaped before calling `JSON3.construct(::Type{MyType}, x::String)`, i.e. there is no direct pointer/length method for escaped strings

The interface to satisfy for writing is:

  * `Base.string(x::MyType)`: overload `Base.string` for your type to return a "stringified" value
"""
struct StringType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.NumberType()

Declaring my type is `JSON3.NumberType()` means it should map to a JSON number value.

Types already declared as `JSON3.NumberType()` include:
  * Any subtype of `Signed`
  * Any subtype of `Unsigned`
  * Any subtype of `AbstractFloat`

In addition to declaring `JSON3.NumberType()`, custom types can also specify a specific, ***existing*** number type it should map to. It does this like:
```julia
JSON3.numbertype(::Type{MyType}) = Float64
```

In this case, I'm declaring the `MyType` should map to an already-supported number type `Float64`. This means that when reading, JSON3 will first parse a `Float64` value, and then call `MyType(x::Float64)`. Note that custom types may also overload `JSON3.construct(::Type{MyType}, x::Float64)` if using a constructor isn't possible. Also note that the default for any type declared as `JSON3.NumberType()` is `Float64`.

Similarly for writing, JSON3 will first call `Float64(x::MyType)` before writing the resulting `Float64` value out as a JSON number.
"""
struct NumberType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.BoolType()

Declaring my type is `JSON3.BoolType()` means it should map to a JSON boolean value.

Types already declared as `JSON3.BoolType()` include:
  * `Bool`

The interface to satisfy for reading is:
  * `MyType(x::Bool)`: define a constructor that takes a single `Bool` value
  * `JSON3.construct(::Type{MyType}, x::Bool; kw...)`: alternatively, you may overload `JSON3.construct`

The interface to satisfy for writing is:
  * `Bool(x::MyType)`: define a conversion to `Bool` method
"""
struct BoolType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.NullType()

Declaring my type is `JSON3.NullType()` means it should map to the JSON value `null`.

Types already declared as `JSON3.NullType()` include:
  * `nothing`
  * `missing`

The interface to satisfy for reading is:
  * `MyType()`: an empty constructor for `MyType`
  * `JSON3.construct(::Type{MyType}, x::Nothing; kw...)`: alternatively, you may overload `JSON3.construct`

There is no interface for writing; if a custom type is declared as `JSON3.NullType()`, then the JSON value `null` will be written.
"""
struct NullType <: JSONType end

"""
    JSON3.StructType(::Type{MyType}) = JSON3.AbstractType()

When declaring my type as `JSON3.AbstractType()`, you must also define `JSON3.subtypes`, which should be a NamedTuple with subtype keys mapping to Julia subtype Type values. You may optionally define `JSON3.subtypekey` that indicates which JSON key should be used for identifying the appropriate concrete subtype. A quick example should help illustrate proper use of this `StructType`:
```julia
abstract type Vehicle end

struct Car <: Vehicle
    type::String
    make::String
    model::String
    seatingCapacity::Int
    topSpeed::Float64
end

struct Truck <: Vehicle
    type::String
    make::String
    model::String
    payloadCapacity::Float64
end

JSON3.StructType(::Type{Vehicle}) = JSON3.AbstractType()
JSON3.subtypekey(::Type{Vehicle}) = :type
JSON3.subtypes(::Type{Vehicle}) = (car=Car, truck=Truck)

car = JSON3.read(\"\"\"
{
    "type": "car",
    "make": "Mercedes-Benz",
    "model": "S500",
    "seatingCapacity": 5,
    "topSpeed": 250.1
}\"\"\", Vehicle)
```
Here we have a `Vehicle` type that is defined as a `JSON3.AbstractType()`. We also have two concrete subtypes, `Car` and `Truck`. In addition to the `StructType` definition, we also define `JSON3.subtypekey(::Type{Vehicle}) = :type`, which signals to JSON3 that, when parsing a JSON structure, when it encounters the `type` key, it should use the value, in our example it's `car`, to discover the appropriate concrete subtype to parse the structure as, in this case `Car`. The mapping of JSON subtype key value to Julia Type is defined in our example via `JSON3.subtypes(::Type{Vehicle}) = (car=Car, truck=Truck)`. Thus, `JSON3.AbstractType` is useful when the JSON structure to read includes a "subtype" key-value pair that can be used to parse a specific, concrete type; in our example, parsing the structure as a `Car` instead of a `Truck`.
"""
struct AbstractType <: StructType end

subtypekey(x::T) where {T} = subtypekey(T)
subtypekey(::Type{T}) where {T} = :type
subtypes(x::T) where {T} = subtypes(T)
subtypes(::Type{T}) where {T} = NamedTuple()

read(io::IO, ::Type{T}; kw...) where {T} = read(Base.read(io, String), T; kw...)
read(bytes::AbstractVector{UInt8}, ::Type{T}; kw...) where {T} = read(VectorString(bytes), T; kw...)

function read(str::AbstractString, ::Type{T}; kw...) where {T}
    buf = codeunits(str)
    len = length(buf)
    if len == 0
        error = UnexpectedEOF
        pos = 0
        @goto invalid
    end
    pos = 1
    b = getbyte(buf, pos)
    @wh
    pos, x = read(StructType(T), buf, pos, len, b, T; kw...)
    return x
@label invalid
    invalid(error, buf, pos, T)
end

read(::NoStructType, buf, pos, len, b, ::Type{T}; kw...) where {T} = throw(ArgumentError("$T doesn't have a defined `JSON3.StructType`"))

function read(::Struct, buf, pos, len, b, U::Union; kw...)
    # Julia implementation detail: Unions are sorted :)
    # This lets us avoid the below try-catch when U <: Union{Missing,T}
    if U.a === Nothing || U.a === Missing
        if buf[pos] == UInt8('n')
            return read(StructType(U.a), buf, pos, len, b, U.a)
        else
            return read(StructType(U.b), buf, pos, len, b, U.b; kw...)
        end
    end
    try
        return read(StructType(U.a), buf, pos, len, b, U.a; kw...)
    catch e
        return read(StructType(U.b), buf, pos, len, b, U.b; kw...)
    end
end

@inline function read(::Struct, buf, pos, len, b, ::Type{Any}; kw...)
    if b == UInt8('{')
        return read(ObjectType(), buf, pos, len, b, Dict{String, Any})
    elseif b == UInt8('[')
        return read(ArrayType(), buf, pos, len, b, Base.Array{Any})
    elseif b == UInt8('"')
        return read(StringType(), buf, pos, len, b, String)
    elseif b == UInt8('n')
        return read(NullType(), buf, pos, len, b, Nothing)
    elseif b == UInt8('t')
        return read(BoolType(), buf, pos, len, b, Bool)
    elseif b == UInt8('f')
        return read(BoolType(), buf, pos, len, b, Bool)
    elseif (UInt8('0') <= b <= UInt8('9')) || b == UInt8('-') || b == UInt8('+')
        float, code, pos = Parsers.typeparser(Float64, buf, pos, len, b, Int16(0), Parsers.OPTIONS)
        if code > 0
            int = unsafe_trunc(Int64, float)
            if int == float
                return pos, int
            else
                return pos, float
            end
        end
    end
@label invalid
    invalid(InvalidChar, buf, pos, Any)
end

StructType(::Type{<:AbstractString}) = StringType()
StructType(::Type{Symbol}) = StringType()
StructType(::Type{<:Enum}) = StringType()
StructType(::Type{Char}) = StringType()
StructType(::Type{UUID}) = StringType()
StructType(::Type{T}) where {T <: Dates.TimeType} = StringType()

function construct(::Type{Char}, str::String; kw...)
    if length(str) == 1
        return Char(str[1])
    else
        throw(ArgumentError("invalid conversion from json string to Char: '$str'"))
    end
end

function construct(::Type{E}, ptr::Ptr{UInt8}, len::Int; kw...) where {E <: Enum}
    @static if VERSION < v"1.2.0-DEV.272"
        Core.eval(parentmodule(E), _symbol(ptr, len))
    else
        sym = _symbol(ptr, len)
        for (k, v) in Base.Enums.namemap(E)
            sym == v && return E(k)
        end
        throw(ArgumentError("invalid $E string value: \"$(unsafe_string(ptr, len))\""))
    end
end

construct(T, str::String; kw...) = T(str)
construct(T, ptr::Ptr{UInt8}, len::Int; kw...) = construct(T, unsafe_string(ptr, len); kw...)
construct(::Type{Symbol}, ptr::Ptr{UInt8}, len::Int; kw...) = _symbol(ptr, len)
construct(::Type{T}, str::String; dateformat::Dates.DateFormat=Dates.default_format(T), kw...) where {T <: Dates.TimeType} = T(str, dateformat)

@inline function read(::StringType, buf, pos, len, b, ::Type{T}; kw...) where {T}
    if b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    pos += 1
    @eof
    strpos = pos
    strlen = 0
    escaped = false
    b = getbyte(buf, pos)
    while b != UInt8('"')
        if b == UInt8('\\')
            escaped = true
            # skip next character
            pos += 2
            strlen += 2
        else
            pos += 1
            strlen += 1
        end
        @eof
        b = getbyte(buf, pos)
    end
    ptr = pointer(buf, strpos)
    return pos + 1, escaped ? construct(T, unescape(PointerString(ptr, strlen)); kw...) : construct(T, ptr, strlen; kw...)

@label invalid
    invalid(error, buf, pos, T)
end

StructType(::Type{Bool}) = BoolType()

construct(T, bool::Bool; kw...) = T(bool)

@inline function read(::BoolType, buf, pos, len, b, ::Type{T}; kw...) where {T}
    if pos + 3 <= len &&
        b            == UInt8('t') &&
        buf[pos + 1] == UInt8('r') &&
        buf[pos + 2] == UInt8('u') &&
        buf[pos + 3] == UInt8('e')
        return pos + 4, construct(T, true; kw...)
    elseif pos + 4 <= len &&
        b            == UInt8('f') &&
        buf[pos + 1] == UInt8('a') &&
        buf[pos + 2] == UInt8('l') &&
        buf[pos + 3] == UInt8('s') &&
        buf[pos + 4] == UInt8('e')
        return pos + 5, construct(T, false; kw...)
    else
        invalid(InvalidChar, buf, pos, Bool)
    end
end

StructType(::Type{Nothing}) = NullType()
StructType(::Type{Missing}) = NullType()

construct(T, ::Nothing; kw...) = T()

@inline function read(::NullType, buf, pos, len, b, ::Type{T}; kw...) where {T}
    if pos + 3 <= len &&
        b            == UInt8('n') &&
        buf[pos + 1] == UInt8('u') &&
        buf[pos + 2] == UInt8('l') &&
        buf[pos + 3] == UInt8('l')
        return pos + 4, construct(T, nothing; kw...)
    else
        invalid(InvalidChar, buf, pos, T)
    end
end

StructType(::Type{<:Unsigned}) = NumberType()
StructType(::Type{<:Signed}) = NumberType()
StructType(::Type{<:AbstractFloat}) = NumberType()
numbertype(::Type{T}) where {T <: Real} = T
numbertype(x) = Float64
construct(T, x::Real; kw...) = T(x)

@inline function read(::NumberType, buf, pos, len, b, ::Type{T}; kw...) where {T}
    x, code, pos = Parsers.typeparser(numbertype(T), buf, pos, len, b, Int16(0), Parsers.OPTIONS)
    if code > 0
        return pos, construct(T, x; kw...)
    end
    invalid(InvalidChar, buf, pos, T)
end

StructType(::Type{<:AbstractArray}) = ArrayType()
StructType(::Type{<:AbstractSet}) = ArrayType()
StructType(::Type{<:Tuple}) = ArrayType()

construct(T, x::Vector{S}; kw...) where {S} = T(x)

@inline read(::ArrayType, buf, pos, len, b, ::Type{T}; kw...) where {T} = read(ArrayType(), buf, pos, len, b, T, Base.IteratorEltype(T) == Base.HasEltype() ? eltype(T) : Any; kw...)
@inline read(::ArrayType, buf, pos, len, b, ::Type{T}, ::Type{eT}; kw...) where {T, eT} = readarray(buf, pos, len, b, T, eT; kw...)
read(::ArrayType, buf, pos, len, b, ::Type{Tuple}, ::Type{eT}; kw...) where {eT} = readarray(buf, pos, len, b, Tuple, eT; kw...)

@inline function readarray(buf, pos, len, b, ::Type{T}, ::Type{eT}; kw...) where {T, eT}
    if b != UInt8('[')
        error = ExpectedOpeningArrayChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    vals = Vector{eT}(undef, 0)
    if b == UInt8(']')
        return pos + 1, construct(T, vals; kw...)
    end
    while true
        # positioned at start of value
        pos, y = read(StructType(eT), buf, pos, len, b, eT; kw...)
        push!(vals, y)
        @eof
        b = getbyte(buf, pos)
        @wh
        if b == UInt8(']')
            return pos + 1, construct(T, vals; kw...)
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
    end

@label invalid
    invalid(error, buf, pos, T)
end

@inline function read(::ArrayType, buf, pos, len, b, ::Type{T}, ::Type{eT}; kw...) where {T <: Tuple, eT}
    if b != UInt8('[')
        error = ExpectedOpeningArrayChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    if b == UInt8(']')
        pos += 1
        return pos, T()
    end
    N = fieldcount(T)
    Base.@nexprs 32 i -> begin
        # positioned at start of value
        eT_i = fieldtype(T, i)
        pos, x_i = read(StructType(eT_i), buf, pos, len, b, eT_i; kw...)
        @eof
        b = getbyte(buf, pos)
        @wh
        if N == i
            if b == UInt8(']')
                return pos + 1, Base.@ncall i tuple x
            else
                error = ExpectedClosingArrayChar
                @goto invalid
            end
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
    end
    vals = []
    for i = 33:N
        eT_i = fieldtype(T, i)
        pos, y = read(StructType(eT_i), buf, pos, len, b, eT_i; kw...)
        push!(vals, y)
        @eof
        b = getbyte(buf, pos)
        @wh
        if b == UInt8(']')
            pos += 1
            break
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
    end
    return pos, (x_1, x_2, x_3, x_4, x_5, x_6, x_7, x_8, x_9, x_10, x_11, x_12, x_13, x_14, x_15, x_16,
                  x_17, x_18, x_19, x_20, x_21, x_22, x_23, x_24, x_25, x_26, x_27, x_28, x_29, x_30, x_31, x_32, vals...)

@label invalid
    invalid(error, buf, pos, T)
end

StructType(::Type{<:AbstractDict}) = ObjectType()
StructType(::Type{<:NamedTuple}) = ObjectType()
StructType(::Type{<:Pair}) = ObjectType()

keyvaluepairs(x) = pairs(x)
keyvaluepairs(x::Pair) = (x,)

construct(::Type{Dict{K, V}}, x::Dict{K, V}; kw...) where {K, V} = x
construct(T, x::Dict{K, V}; kw...) where {K, V} = T(x)

construct(::Type{NamedTuple}, x::Dict; kw...) = NamedTuple{Tuple(keys(x))}(values(x))
construct(::Type{NamedTuple{names}}, x::Dict; kw...) where {names} = NamedTuple{names}(Tuple(x[nm] for nm in names))
construct(::Type{NamedTuple{names, types}}, x::Dict; kw...) where {names, types} = NamedTuple{names, types}(Tuple(x[nm] for nm in names))

keyvalue(::Type{Symbol}, escaped, ptr, len) = escaped ? Symbol(unescape(PointerString(ptr, len))) : _symbol(ptr, len)
keyvalue(::Type{T}, escaped, ptr, len) where {T} = escaped ? construct(T, unescape(PointerString(ptr, len))) : construct(T, unsafe_string(ptr, len))

@inline read(::ObjectType, buf, pos, len, b, ::Type{T}; kw...) where {T} = read(ObjectType(), buf, pos, len, b, T, Symbol, Any; kw...)
@inline read(::ObjectType, buf, pos, len, b, ::Type{T}; kw...) where {T <: NamedTuple} = read(ObjectType(), buf, pos, len, b, T, Symbol, Any; kw...)
@inline read(::ObjectType, buf, pos, len, b, ::Type{Dict}; kw...) = read(ObjectType(), buf, pos, len, b, Dict, String, Any; kw...)
@inline read(::ObjectType, buf, pos, len, b, ::Type{T}; kw...) where {T <: AbstractDict} = read(ObjectType(), buf, pos, len, b, T, keytype(T), valtype(T); kw...)

@inline function read(::ObjectType, buf, pos, len, b, ::Type{T}, ::Type{K}, ::Type{V}; kw...) where {T, K, V}
    if b != UInt8('{')
        error = ExpectedOpeningObjectChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    x = Dict{K, V}()
    if b == UInt8('}')
        return pos + 1, construct(T, x; kw...)
    elseif b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    pos += 1
    @eof
    while true
        keypos = pos
        keylen = 0
        escaped = false
        # read first key character
        b = getbyte(buf, pos)
        # positioned at first character of object key
        while b != UInt8('"')
            if b == UInt8('\\')
                escaped = true
                # skip next character
                pos += 2
                keylen += 2
            else
                pos += 1
                keylen += 1
            end
            @eof
            b = getbyte(buf, pos)
        end
        key = keyvalue(K, escaped, pointer(buf, keypos), keylen)
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8(':')
            error = ExpectedSemiColon
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        # now positioned at start of value
        pos, y = read(StructType(V), buf, pos, len, b, V; kw...)
        x[key] = y
        @eof
        b = getbyte(buf, pos)
        @wh
        if b == UInt8('}')
            return pos + 1, construct(T, x; kw...)
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8('"')
            error = ExpectedOpeningQuoteChar
            @goto invalid
        end
        pos += 1
        @eof
    end

@label invalid
    invalid(error, buf, pos, Object)
end

@inline function read(::Mutable, buf, pos, len, b, ::Type{T}; kw...) where {T}
    if b != UInt8('{')
        error = ExpectedOpeningObjectChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    x = T()
    if b == UInt8('}')
        pos += 1
        return pos, x
    elseif b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    N = fieldcount(T)
    nms = names(T)
    excl = excludes(T)
    kwargs = keywordargs(T)
    pos += 1
    @eof
    while true
        keypos = pos
        keylen = 0
        escaped = false
        b = getbyte(buf, pos)
        while b != UInt8('"')
            if b == UInt8('\\')
                escaped = true
                # skip next character
                pos += 2
                keylen += 2
            else
                pos += 1
                keylen += 1
            end
            @eof
            b = getbyte(buf, pos)
        end
        key = keyvalue(Symbol, escaped, pointer(buf, keypos), keylen)
        key = julianame(nms, key)
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8(':')
            error = ExpectedSemiColon
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        is_included = !symbolin(excl, key)
        # unroll the first 32 field checks to avoid dynamic dispatch if possible
        Base.@nif(
            32,
            i -> (i <= N && fieldname(T, i) === key),
            i -> begin
                FT_i = fieldtype(T, i)
                if isempty(kwargs)
                    pos, y_i = read(StructType(FT_i), buf, pos, len, b, FT_i; kw...)
                else
                    pos, y_i = read(StructType(FT_i), buf, pos, len, b, FT_i; kwargs[fieldname(T, i)]...)
                end
                is_included && setfield!(x, i, y_i)
            end,
            i -> begin
                is_field_still_unread = true
                for j in 33:N
                    fieldname(T, j) === key || continue
                    FT_j = fieldtype(T, j)
                    if isempty(kwargs)
                        pos, y_j = read(StructType(FT_j), buf, pos, len, b, FT_j; kw...)
                    else
                        pos, y_j = read(StructType(FT_j), buf, pos, len, b, FT_j; kwargs[fieldname(T, j)]...)
                    end
                    is_included && setfield!(x, j, y_j)
                    is_field_still_unread = false
                    break
                end
                if is_field_still_unread
                    pos, _ = read(Struct(), buf, pos, len, b, Any)
                end
            end
        )
        @eof
        b = getbyte(buf, pos)
        @wh
        if b == UInt8('}')
            pos += 1
            return pos, x
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8('"')
            error = ExpectedOpeningQuoteChar
            @goto invalid
        end
        pos += 1
        @eof
    end

@label invalid
    invalid(error, buf, pos, T)
end

@inline function read(::Struct, buf, pos, len, b, ::Type{T}; kw...) where {T}
    if b != UInt8('{')
        error = ExpectedOpeningObjectChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    if b == UInt8('}')
        pos += 1
        return pos, T()
    elseif b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    pos += 1
    @eof
    N = fieldcount(T)
    Base.@nexprs 32 i -> begin
        pos, x_i = readvalue(buf, pos, len, fieldtype(T, i); kw...)
        if N == i
            return pos, Base.@ncall i T x
        end
    end
    vals = []
    for i = 33:N
        pos, y = readvalue(buf, pos, len, fieldtype(T, i); kw...)
        push!(vals, y)
    end
    return pos, T(x_1, x_2, x_3, x_4, x_5, x_6, x_7, x_8, x_9, x_10, x_11, x_12, x_13, x_14, x_15, x_16,
                  x_17, x_18, x_19, x_20, x_21, x_22, x_23, x_24, x_25, x_26, x_27, x_28, x_29, x_30, x_31, x_32, vals...)

@label invalid
    invalid(error, buf, pos, T)
end

@inline function readvalue(buf, pos, len, ::Type{T}; kw...) where {T}
    b = getbyte(buf, pos)
    while b != UInt8('"')
        pos += ifelse(b == UInt8('\\'), 2, 1)
        @eof
        b = getbyte(buf, pos)
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    if b != UInt8(':')
        error = ExpectedSemiColon
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    # read value
    pos, y = read(StructType(T), buf, pos, len, b, T; kw...)
    @eof
    b = getbyte(buf, pos)
    @wh
    if b == UInt8('}')
        pos += 1
        return pos, y
    elseif b != UInt8(',')
        error = ExpectedComma
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    if b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    pos += 1
    @eof
    return pos, y
@label invalid
    invalid(error, buf, pos, T)
end

@inline function read(::AbstractType, buf, pos, len, b, ::Type{T}; kw...) where {T}
    startpos = pos
    startb = b
    if b != UInt8('{')
        error = ExpectedOpeningObjectChar
        @goto invalid
    end
    pos += 1
    @eof
    b = getbyte(buf, pos)
    @wh
    if b == UInt8('}')
        throw(ArgumentError("invalid json abstract type"))
    elseif b != UInt8('"')
        error = ExpectedOpeningQuoteChar
        @goto invalid
    end
    pos += 1
    @eof
    types = subtypes(T)
    skey = subtypekey(T)
    while true
        keypos = pos
        keylen = 0
        escaped = false
        b = getbyte(buf, pos)
        while b != UInt8('"')
            if b == UInt8('\\')
                escaped = true
                # skip next character
                pos += 2
                keylen += 2
            else
                pos += 1
                keylen += 1
            end
            @eof
            b = getbyte(buf, pos)
        end
        key = keyvalue(Symbol, escaped, pointer(buf, keypos), keylen)
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8(':')
            error = ExpectedSemiColon
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        # read value
        pos, val = read(Struct(), buf, pos, len, b, Any)
        if key == skey
            TT = types[Symbol(val)]
            return read(StructType(TT), buf, startpos, len, startb, TT; kw...)
        end
        @eof
        b = getbyte(buf, pos)
        @wh
        if b == UInt8('}')
            pos += 1
            throw(ArgumentError("invalid json abstract type: didn't find subtypekey"))
        elseif b != UInt8(',')
            error = ExpectedComma
            @goto invalid
        end
        pos += 1
        @eof
        b = getbyte(buf, pos)
        @wh
        if b != UInt8('"')
            error = ExpectedOpeningQuoteChar
            @goto invalid
        end
        pos += 1
        @eof
    end

@label invalid
    invalid(error, buf, pos, T)
end


text:
Transducers.executor_type(::DArray) = DaggerEx

struct DAZip{A,M}
    arrays::A  # a tuple of arrays of chunks
    mappers::M  # a tuple functions
end

preprocess_darray(rf, xs::AbstractArray) = preprocess_darray(Map(first)'(rf), zip(xs))

function preprocess_darray(rf, xs::Iterators.Zip)
    da = nothing
    for it in xs.is
        if it isa ReferenceableArray
            it = it.x
        end
        if it isa DArray
            da = it
            break
        end
    end
    da === nothing && return nothing, nothing
    blocks = map(last, first(da.subdomains).indexes) # find a better way
    arrays = map(xs.is) do it
        if it isa DArray
            it
        elseif it isa ReferenceableArray
            it.x isa DArray || error("`ReferenceableArray` must wrap `DArray`")
            it.x
        else
            distribute(it, blocks)
        end
    end
    all(it.subdomains == da.subdomains for it in arrays) ||
        error("unequal chunking not implemented yet")

    mappers = map(xs.is) do it
        if it isa ReferenceableArray{<:Any,<:Any,<:DArray}
            referenceable
        else
            identity
        end
    end

    return (rf, DAZip(arrays, mappers))
end

function _reduce_basecase_zip(rf, init, mappers, chunks...)
    iters = map(|>, chunks, mappers)
    return _reduce_basecase(rf, init, zip(iters...))
end

function _transduce_darray(rf, init, xs::DAZip, _basesize)
    # TODO: further divide each chunk into `basesize`
    mappers = xs.mappers
    f(chunks) = delayed(_reduce_basecase_zip)(rf, init, mappers, chunks...)
    op(a, b) = delayed(_combine)(rf, a, b)
    return _mapreduce(f, op, init, zip((A.chunks for A in xs.arrays)...))
end

ext: jl


text:
importall Base
read(s::IO, t::DataType) = t(ntuple(length(t.types), x->read(s, t.types[x]))...)

const Entities       = 1

const Magic = "IBSP"

type Lump
	offset::Uint32
	length::Uint32
end

type Header
	signature::Uint32
	version::Uint32
	#lumps::Array{Lump,1}(19)
end

function bspPrintEntities(io::IO)
	hdr = read(io, Header)
	lumps = read(io, Lump, 19)

	seek(io, lumps[Entities].offset)
	entityLump = bytestring(read(io, Uint8, lumps[Entities].length-1))
	entities = Array(Dict{String, String}, 0)
	for entity = split(entityLump, ['{', '}'])
		entity = strip(entity)
		if length(entity) < 1 # TODO: remove this w/ regex
			continue
		end
		dict = Dict{String, String}()
		for field = split(entity, '\n')
			fieldStr = split(strip(field), "\" \"")
			name = lstrip(strip(fieldStr[1], "\""), "_")
			value = strip(fieldStr[2], "\"")
			dict[name] = value
		end
		if length(dict) > 0
			push!(entities, dict)
		end
	end
	for entity = entities
		println(entity)
	end
end

bspPrintEntities(open(ARGS[1]))


text:
using Transformers
using Transformers.Basic
using Transformers.Pretrain
using Transformers.Datasets
using Transformers.BidirectionalEncoder

using Flux
using Flux: onehotbatch, gradient
import Flux.Optimise: update!
using WordTokenizers

include("./model.jl")

const Epoch = 20
const Batch = 4
const FromScratch = false

#use wordpiece and tokenizer from pretrain
const wordpiece = pretrain"bert-uncased_L-12_H-768_A-12:wordpiece"
const tokenizer = pretrain"bert-uncased_L-12_H-768_A-12:tokenizer"
const vocab = Vocabulary(wordpiece)

#see model.jl
const bert_model = gpu(
  FromScratch ? create_bert() : pretrain"bert-uncased_L-12_H-768_A-12:bert_model"
)
const ps = params(bert_model)
const opt = ADAM(1e-4)

#read sentence line by line and wrap in channel
#for working with files, read the data into a channel
function prepare_data()
  Datasets.reader("./sample_text.txt")
end

function preprocess(batch)
  mask = getmask(batch[1])
  tok = vocab(batch[1])
  segment = fill!(similar(tok), 1.0)

  for (i, sentence) ∈ enumerate(batch[1])
    j = findfirst(isequal("[SEP]"), sentence)
    if j !== nothing
      @view(segment[j+1:end, i]) .= 2.0
    end
  end

  ind = vcat(
    map(enumerate(batch[2])) do (i, x)
     map(j->(j,i), x)
    end...)

  masklabel = onehotbatch(vocab(vcat(batch[3]...)), 1:length(vocab))
  nextlabel = onehotbatch(batch[4], (true, false))

  return (tok=tok, segment=segment), ind, masklabel, nextlabel, mask
end

function loss(data, ind, masklabel, nextlabel, mask = nothing)
  e = bert_model.embed(data)
  t = bert_model.transformers(e, mask)
  nextloss = Basic.logcrossentropy(
    bert_model.classifier.nextsentence(
      bert_model.classifier.pooler(
        t[:,1,:]
      )
    ),
    nextlabel
  )
  mkloss = masklmloss(bert_model.embed.embeddings.tok,
                      bert_model.classifier.masklm.transform,
                      bert_model.classifier.masklm.output_bias,
                      t,
                      ind,
                      masklabel
                      )
  return nextloss + mkloss
end

function train!()
  global Batch
  global Epoch
  @info "start training"
  for e = 1:Epoch
    @info "epoch: $e"
    #prepare training data
    pretrain_data = prepare_data()

    #pretrain_task function help you to create mask lm and next sentence prediction data
    datas = bert_pretrain_task(pretrain_data, wordpiece; tokenizer = tokenizer) #or set_tokenizer(tokenizer)

    i = 1
    while(batch = get_batch(datas, Batch)) !== nothing
      data, ind, masklabel, nextlabel, mask = todevice(preprocess(batch))
      l = loss(data, ind, masklabel, nextlabel, mask)
      @show l
      grad = gradient(()->l, ps)
      update!(opt, ps, grad)
    end
  end
end

train!()


text:
# This file is a part of SimilaritySearch.jl

import Base: push!

export ExhaustiveSearch, search, push!

"""
    ExhaustiveSearch(dist::PreMetric, db::AbstractVector)

Solves queries evaluating `dist` for the query and all elements in the dataset
"""
@with_kw struct ExhaustiveSearch{DistanceType<:PreMetric, DataType<:AbstractVector} <: AbstractSearchContext
    dist::DistanceType = SqL2Distance()
    db::DataType = Vector{Int32}[]
end

Base.copy(seq::ExhaustiveSearch; dist=seq.dist, db=seq.db) = ExhaustiveSearch(dist, db)

"""
    search(seq::ExhaustiveSearch, q, res::KnnResult)

Solves the query evaluating all items in the given query.

By default, it uses an internal result buffer;
multithreading applications must duplicate specify another `res` object.
"""
function search(seq::ExhaustiveSearch, q, res::KnnResult)
    db = seq.db

    @inbounds for i in eachindex(db)
        push!(res, i, evaluate(seq.dist, db[i], q))
    end

    res
end


text:
function package_version(name)
    gitdir = Pkg.dir(name) * "/.git"
    gitcmd = `git --git-dir $gitdir log -1 --format="%H"`
    gitver = chomp(readstring(gitcmd))
    string(Pkg.installed(name)) * " ($gitver)"
end


text:
function ALexport(x; verbose = true)
   y = replace(x, r"^export( as){0,1} " => "")
   Core.eval(Main, Meta.parse("$y = replset"))
   verbose && printmarkdown("`julia>` $y = `replset`")
   verbose && println()
end

function ALimport(userinput; verbose = true)

    imports = Symbol(strip(replace(userinput, r"^import "=>"")))

    !(imports ∈ names(Main)) && (println("$imports not found!"); return)

    imported = getfield(Main, imports)
    !isa(imported, LogicalCombo) && (println("$imports not a LogicalCombo!"); return)

    setreplset!(imported)

    setactivehistory!(History())
    pushactivehistory!(replset)

    pushsessionhistory!(imported)

    verbose && println("Importing $imports - " * reportfeasible(replset))
end


text:
module Hyperopt

export Hyperoptimizer, @hyperopt, @phyperopt, printmin, printmax
export RandomSampler, BlueNoiseSampler, LHSampler, CLHSampler, Continuous, Categorical, GPSampler, Max, Min

using LinearAlgebra, Statistics
using Juno
using Lazy
using MacroTools
using MacroTools: postwalk, prewalk
using Parameters
using RecipesBase
using Distributed
using LatinHypercubeSampling
using BayesianOptimization, GaussianProcesses

abstract type Sampler end
@with_kw struct Hyperoptimizer
    iterations::Int
    params
    candidates
    history = []
    results = []
    sampler::Sampler = RandomSampler()
end

function Base.getproperty(ho::Hyperoptimizer, s::Symbol)
    s == :minimum && (return minimum(replace(ho.results, NaN => Inf)))
    s == :minimizer && (return minimum(ho)[1])
    return getfield(ho,s)
end

include("samplers.jl")

function Hyperoptimizer(iterations::Int, sampler::Sampler = RandomSampler(); kwargs...)
    params = keys(kwargs)
    candidates = []
    for kw in kwargs
        push!(candidates, kw[2])
    end
    Hyperoptimizer(iterations=iterations, params=params, candidates=candidates, sampler=sampler)
end

Lazy.@forward Hyperoptimizer.history Base.length, Base.getindex

function Base.iterate(ho::Hyperoptimizer, state=1)
    state > ho.iterations && return nothing
    samples = ho.sampler(ho)
    push!(ho.history, samples)
    [state;samples], state+1
end

function preprocess_expression(ex)
    ex.head == :for || error("Wrong syntax, Use For-loop syntax, ex: @hyperopt for i=100, param=LinRange(1,10,100) ...")
    params     = []
    candidates = []
    sampler_ = :(RandomSampler())
    loop = ex.args[1].args # ex.args[1] = the arguments to the For loop
    i = 1
    while i <= length(loop)
        if @capture(loop[i], sampler = sam_) # A sampler was provided
            sampler_ = sam
            deleteat!(loop,i) # Remove the sampler from the args
            continue
        end
        @capture(loop[i], param_ = list_) || error("Wrong syntax, Use For-loop syntax, ex: @hyperopt for i=100, param=LinRange(1,10,100) ...")
        push!(params, param)
        push!(candidates, list)
        i += 1
    end
    params = ntuple(i->params[i], length(params))

    params, candidates, sampler_
end

function macrobody(ex, params, candidates, sampler)
    quote
        ho = Hyperoptimizer(iterations = $(esc(candidates[1])), params = $(esc(params[2:end])), candidates = $(Expr(:tuple, esc.(candidates[2:end])...)), sampler=$(esc(sampler)))
        Juno.progress() do id
            for $(Expr(:tuple, esc.(params)...)) = ho
                res = $(esc(ex.args[2])) # ex.args[2] = Body of the For loop
                push!(ho.results, res)
                Base.CoreLogging.@logmsg -1 "Hyperopt" progress=$(esc(params[1]))/ho.iterations  _id=id
            end
        end
        ho
    end
end

macro hyperopt(ex)
    params, candidates, sampler_ = preprocess_expression(ex)
    macrobody(ex, params, candidates, eval(sampler_))
end

function pmacrobody(ex, params, candidates, sampler_)
    quote
        function workaround_function()
            ho = Hyperoptimizer(iterations = $(esc(candidates[1])), params = $(esc(params[2:end])), candidates = $(Expr(:tuple, esc.(candidates[2:end])...)), sampler=$(esc(sampler_)))
            ho.sampler isa GPSampler && error("We currently do not support running the GPSampler in parallel. If this is an issue, open an issue ;)")
            res = pmap(1:ho.iterations) do i
                $(Expr(:tuple, esc.(params)...)),_ = iterate(ho,i)
                res = $(esc(ex.args[2])) # ex.args[2] = Body of the For loop

                res, $(Expr(:tuple, esc.(params[2:end])...))
            end
            append!(ho.results, getindex.(res,1))
            empty!(ho.history) # The call to iterate(ho) populates history, but only on host process
            append!(ho.history, getindex.(res,2))
            ho
        end
        workaround_function()
    end
end

macro phyperopt(ex)
    params, candidates, sampler_ = preprocess_expression(ex)
    pmacrobody(ex, params, candidates, sampler_)
end

function Base.minimum(ho::Hyperoptimizer)
    m,i = findmin(replace(ho.results, NaN => Inf))
    ho.history[i], m
end
function Base.maximum(ho::Hyperoptimizer)
    m,i = findmax(replace(ho.results, NaN => Inf))
    ho.history[i], m
end

function printmin(ho::Hyperoptimizer)
    m,i = minimum(ho)
    for (param, value) in zip(ho.params, m)
        println(param, " = ", value)
    end
end

function printmax(ho::Hyperoptimizer)
    m,i = maximum(ho)
    for (param, value) in zip(ho.params, m)
        println(param, " = ", value)
    end
end

include("plotting.jl")
end # module


text:
struct BamReadChunk
	e::CIGARElement
	readPos::Int
	refPos::Int
end

BamReadChunk() = BamReadChunk(0,0,0)

op(c::BamReadChunk) = op(c.e)
op_char(c::BamReadChunk) = op_char(c.e)
len(c::BamReadChunk) = len(c.e)

readpos(c::BamReadChunk) = c.readPos
refpos(c::BamReadChunk) = c.refPos

print(io::Main.Base.IO, c::BamReadChunk) = print(io, "Chunk: ", c.e, ", readPos: ", c.readPos, ", refPos: ", c.refPos)

struct BamReadChunkState
	cig::CIGAR
	cigState::Int
	readPos::Int
	refPos::Int
end
BamReadChunkState(r::BamRead) = BamReadChunkState(cigar(r),1,1,pos(r))

function iterate(r::BamRead, s::BamReadChunkState=BamReadChunkState(r))
	cigIter = iterate(s.cig, s.cigState)
	cigIter == nothing && return nothing
	(e,cigState) = cigIter

	readPos = s.readPos
	refPos = s.refPos

	o = op(e)
	l = len(e)
	if o==CIGAR_M || o==CIGAR_S || o==CIGAR_I || o==CIGAR_Eq || o==CIGAR_X
		readPos += l
	end

	if o==CIGAR_M || o == CIGAR_D || o==CIGAR_N || o==CIGAR_Eq || o==CIGAR_X
		refPos += l
	end
	# Do nothing for CIGAR_H and CIGAR_P

	# TODO: check for unsupported/unknown CIGAR ops

	(BamReadChunk(e,s.readPos,s.refPos),BamReadChunkState(s.cig,cigState,readPos,refPos))
end

eltype(::Type{BamRead}) = BamReadChunk

function readrange(c::BamReadChunk)
	p = readpos(c)
	l = len(c)
	p:p+l-1
end

seq(r::BamRead,c::BamReadChunk) = view(seq(r),readrange(c))
qual(r::BamRead,c::BamReadChunk) = view(qual(r),readrange(c))


text:
function screeplot(S)
    hold(false)
    colorscheme("light")
    plot(1:length(S),S, title="Scree Plot",
        xlabel = "Singular Value IDs",
        ylabel = "Singular Values", grid=false)
    hold(true)
    aspectratio(24/16)
    display(scatter(1:length(S),S,grid=false))
end


text:
## common.jl: Core types and shared functions ##

## Core Types ##

type Scene{T<:Real}
    boundaries::Vector{T}              # Scene boundaries
    waypoints::Matrix{T}               # Waypoint positions
    landmarks::Matrix{T}               # True (simulated) feature positions
    true_track::Array{T}               # Ideal pose history
    slam_track::Array{T}               # SLAM pose history
    nsteps::Integer                    # Timestep counter
end

type Particle{T<:Real}
    pose::Vector{T}                    # Inferred vehicle pose e.g. [x, y, phi]
    features::Matrix{T}                # Feature positions (as columns)
    pcov::Matrix{T}                    # Covariance matrices 3 x 3 x nparticles
    fcov::Matrix{T}                    # Feature/landmark cov matrix
    weight::T
end

abstract SlamState

# Extended Kalman Filter SLAM state and covariance
type EKFSlamState{T<:Real} <: SlamState
    x::Vector{T}                       # Joint pose + feature state
    cov::Matrix{T}                     # Joint pose + feature cov matrix
end

# Particle Filter SLAM state
type PFSlamState{T<:Real} <: SlamState
    n::Int
    particles::Vector{Particle{T}}
end

type Vehicle{T<:Real}

    ## Intrinsic properties ##
    wheelbase::T     # Front-to-rear wheel separation [m]
    max_gamma::T     # Max steering angle [rad] (-max_g < g < max_g)
    steer_rate::T    # Maximum rate of change in steer angle [rad/s]
    sensor_range::T  # [m] Landmark detection radius
    shape::Matrix{T} # Set of x,y points defining a polygon for visualization

    ## (Potentially) time-varying properties ##
    pose::Vector{T}  # [x; y; phi]

    target_speed::T  # Velocity magnitude (control setpoint) [m/s]
    measured_speed::T  # Measured velocity magnitude [m/s]

    target_gamma::T  # Target steering angle [rad]
    measured_gamma::T  # Measured steering angle [rad]

    waypoint_id::Int  # Index of current target waypoint
end

Vehicle() = Vehicle(0., 0., 0., 0., zeros(2,0), zeros(3), 0., 0., 0., 0., 0)

type SimData{T<:Real}
    scene::Scene{T}
    vehicle::Vehicle{T}
    state::SlamState
    z::Matrix{T}             # Recent [range, bearing] feature observations
    nz::Int                  # Current number of observations
    state_updated::Bool      # If SLAM state occured in this step
    paused::Bool             # Paused or running
    nlaps::Int               # Number of loops through the waypoint list
end

## Functions ##

"""
Returns true if (x,y) falls within scene.boundaries
"""
function inbounds(x::Real, y::Real, scene::Scene)
    # Scene boundaries: xmin, xmax, ymin, ymax
    xmin, xmax, ymin, ymax = scene.boundaries
    return xmin <= x <= xmax && ymin <= y <= ymax
end

"""
Read x,y positions from 2-column text file and return as 2 x N array
"""
function get_waypoints(txtfile::AbstractString)
    wp, _ = readdlm(txtfile, header=true)
    return wp'
end

"""
Initial [x, y, phi] at first waypoint, heading for second waypoint
"""
function initial_pose(scene::Scene)
    wp = scene.waypoints
    return [wp[1,1]; wp[2,1]; atan2(wp[2,2] - wp[2,1], wp[1,2] - wp[1,1])]
end

"""
Wrap phi into -pi <= phi <= pi
"""
function mpi_to_pi(phi::AbstractFloat)
    if phi > pi
        return phi - 2*pi
    end
    if phi < -pi
        return phi + 2*pi
    end
    return phi
end

"""
Given a 2-by-n or 3-by-n matrix `l` whose columns are positions (x,y) or
poses (x,y,phi) in the local vehicle coordinate system, and a vehicle pose `g`
in the global coordinate system, return `l` transformed to global coordinates.
"""
function local_to_global(l, g)

    q = zeros(l)
    phi = g[3]

    # Rotate and translate
    R = [cos(phi) -sin(phi); sin(phi) cos(phi)]
    q[1:2,:] = R*l[1:2,:] .+ g[1:2]

    # if l is a pose (includes an azimuthal angle)
    if size(l,1)==3
        q[3,:] = mpi_to_pi(l[3,:] .+ phi)
    end
    return q
end

"""
Predict the expected range-bearing observation z (and Jacobian H) of
feature/landmark idf in joint pose + landmark state vector x
"""
function predict_observation(x::AbstractVector, idf::Int)

    # Map feature index idf to position fpos in x
    pose_length = 3
    fpos = pose_length + idf*2 - 1

    # Distance from vehicle to landmark
    dx = x[fpos] - x[1]
    dy = x[fpos + 1] - x[2]
    d2 = dx^2 + dy^2
    d = sqrt(d2)

    # Predict [range, bearing] observation z
    z = [d; atan2(dy,dx) - x[3]]

    # Calculate Jacobian H for pose and z
    xd = dx/d
    yd = dy/d
    xd2 = dx/d2
    yd2 = dy/d2

    H = zeros(2, length(x))
    H[:,1:3]         = [-xd -yd 0; yd2 -xd2 -1]
    H[:,fpos:fpos+1] = [ xd  yd;  -yd2  xd2]

    return z, H
end

"""
Advance equations of motion for vehicle by dt and update vehicle pose in-place
This uses the ideal/target speed and steering angle
"""
function step_vehicle!(vehicle::Vehicle, dt::AbstractFloat)

    x, y, phi = vehicle.pose
    v = vehicle.target_speed
    g = vehicle.target_gamma

    vehicle.pose = [x + v*dt*cos(g + phi);
                    y + v*dt*sin(g + phi);
                    mpi_to_pi(phi + v*dt*sin(g)/vehicle.wheelbase)]
end

"""
Find the next target waypoint given the current position and steer for it.
d_min is minimum distance to current waypoint before switching to next
This uses the ideal/target steering angle
"""
function steer!(vehicle::Vehicle, waypoints::AbstractArray, d_min::AbstractFloat, dt::AbstractFloat)

    # Get current vehicle steering angle and waypoint index
    g = vehicle.target_gamma
    iwp = vehicle.waypoint_id
    x, y, phi = vehicle.pose

    # Get coords of current waypoint
    cwp = waypoints[:, iwp]

    # Determine if current waypoint has been reached
    d2 = (cwp[1] - x)^2 + (cwp[2] - y)^2
    if d2 < d_min^2
        iwp += 1             # Advance to next waypoint
        if iwp > size(waypoints, 2) # Final waypoint reached, flag and return
            iwp = 0
            vehicle.waypoint_id = iwp
            return
        end
        cwp = waypoints[:, iwp] # Coords of next waypoint
    end

    # Change in steering angle required to point towards current waypoint
    dg = mpi_to_pi(atan2(cwp[2] - y, cwp[1] - x) - phi - g)

    # Constrain steering angle by max steering rate
    dgmax = vehicle.steer_rate*dt
    if abs(dg) > dgmax
        dg = sign(dg)*dgmax
    end

    # Limit steering angle to max steering angle
    g += dg
    if abs(g) > vehicle.max_gamma
        g = sign(g)*vehicle.max_gamma
    end

    vehicle.target_gamma = g
    vehicle.waypoint_id = iwp

    return
end

"""
Make a single 2D n-sigma Gaussian contour centered at x with axes given
by covariance matrix P
"""
function ellipse(x, P, nsigma, nsegs)

    # Approximate smooth ellipse with nsegs line segments (nsegs + 1 points)
    phi = 0:2*pi/nsegs:2*pi

    return nsigma*sqrtm(P)*[cos(phi)'; sin(phi)'] .+ x[1:2]
end

function compute_landmark_ellipses(x, P)
    nsigma = 2 # Size of ellipse
    nsegs = 12 # Number of line segments to use

    # Number of observed features/landmarks
    nf = floor(Int, (length(x) - 3)/2)

    # Allocate rows for x,y and columns for npoints/ellipse times # ellipses
    ellipses = Array{Float64}(2*nf, nsegs + 1)

    for i = 1:nf
        j = (2*i + 2):(2*i + 3)

        ellipses[2i-1:2i, :] = ellipse(x[j], P[j,j], nsigma, nsegs)
    end
    return ellipses
end

"""
Return matrix of line segments for laser range-bearing measurements.
Each column contains vehicle and feature positions: [vx vy fx fy]'
"""
function laser_lines(z, vehicle_pose)
    nlines = size(z, 2)
    lines = Array{Float64}(4, nlines)
    lines[1,:] = zeros(1, nlines) .+ vehicle_pose[1]
    lines[2,:] = zeros(1, nlines) .+ vehicle_pose[2]

    # Range and bearing from vehicle to observations
    r, b = z[1,:], z[2,:]

    # x,y position of observations in local (vehicle) coordinate frame
    xy_local = [r.*cos(b) r.*sin(b)]'

    lines[3:4,:] = local_to_global(xy_local, vehicle_pose)
    return lines
end


text:
using PiGPIO

red_pin1 = 18
red_pin2 = 23

pinMode(red_pin1, OUTPUT)
pinMode(red_pin2, OUTPUT)

try
	for i in 1:10
		digitalWrite(red_pin1, HIGH)
        digitalWrite(red_pin2, LOW)
		sleep(0.5)
        digitalWrite(red_pin1, LOW)
		digitalWrite(red_pin2, HIGH)
		sleep(0.5)
	end
finally
    println("Cleaning up!")
	pinMode(red_pin1, INPUT)
    pinMode(red_pin2, INPUT)
end


text:
# syntax: proto3
using ProtoBuf
import ProtoBuf.meta

mutable struct NoneValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function NoneValue(; kwargs...)
        obj = new(meta(NoneValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct NoneValue
const __meta_NoneValue = Ref{ProtoMeta}()
function meta(::Type{NoneValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_NoneValue)
            __meta_NoneValue[] = target = ProtoMeta(NoneValue)
            allflds = Pair{Symbol,Union{Type,String}}[]
            meta(target, NoneValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_NoneValue[]
    end
end

mutable struct TensorSpecProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function TensorSpecProto(; kwargs...)
        obj = new(meta(TensorSpecProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct TensorSpecProto
const __meta_TensorSpecProto = Ref{ProtoMeta}()
function meta(::Type{TensorSpecProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_TensorSpecProto)
            __meta_TensorSpecProto[] = target = ProtoMeta(TensorSpecProto)
            allflds = Pair{Symbol,Union{Type,String}}[:name => AbstractString, :shape => TensorShapeProto, :dtype => Int32]
            meta(target, TensorSpecProto, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_TensorSpecProto[]
    end
end
function Base.getproperty(obj::TensorSpecProto, name::Symbol)
    if name === :name
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :shape
        return (obj.__protobuf_jl_internal_values[name])::TensorShapeProto
    elseif name === :dtype
        return (obj.__protobuf_jl_internal_values[name])::Int32
    else
        getfield(obj, name)
    end
end

mutable struct BoundedTensorSpecProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function BoundedTensorSpecProto(; kwargs...)
        obj = new(meta(BoundedTensorSpecProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct BoundedTensorSpecProto
const __meta_BoundedTensorSpecProto = Ref{ProtoMeta}()
function meta(::Type{BoundedTensorSpecProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_BoundedTensorSpecProto)
            __meta_BoundedTensorSpecProto[] = target = ProtoMeta(BoundedTensorSpecProto)
            allflds = Pair{Symbol,Union{Type,String}}[:name => AbstractString, :shape => TensorShapeProto, :dtype => Int32, :minimum => TensorProto, :maximum => TensorProto]
            meta(target, BoundedTensorSpecProto, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_BoundedTensorSpecProto[]
    end
end
function Base.getproperty(obj::BoundedTensorSpecProto, name::Symbol)
    if name === :name
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :shape
        return (obj.__protobuf_jl_internal_values[name])::TensorShapeProto
    elseif name === :dtype
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :minimum
        return (obj.__protobuf_jl_internal_values[name])::TensorProto
    elseif name === :maximum
        return (obj.__protobuf_jl_internal_values[name])::TensorProto
    else
        getfield(obj, name)
    end
end

mutable struct StructuredValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function StructuredValue(; kwargs...)
        obj = new(meta(StructuredValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct StructuredValue (has cyclic type dependency)
const __meta_StructuredValue = Ref{ProtoMeta}()
function meta(::Type{StructuredValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_StructuredValue)
            __meta_StructuredValue[] = target = ProtoMeta(StructuredValue)
            fnum = Int[1,11,12,13,14,31,32,33,34,35,51,52,53,54]
            wtype = Dict(:int64_value => :sint64)
            allflds = Pair{Symbol,Union{Type,String}}[:none_value => NoneValue, :float64_value => Float64, :int64_value => Int64, :string_value => AbstractString, :bool_value => Bool, :tensor_shape_value => TensorShapeProto, :tensor_dtype_value => Int32, :tensor_spec_value => TensorSpecProto, :type_spec_value => "TypeSpecProto", :bounded_tensor_spec_value => BoundedTensorSpecProto, :list_value => "ListValue", :tuple_value => "TupleValue", :dict_value => "DictValue", :named_tuple_value => "NamedTupleValue"]
            oneofs = Int[1,1,1,1,1,1,1,1,1,1,1,1,1,1]
            oneof_names = Symbol[Symbol("kind")]
            meta(target, StructuredValue, allflds, ProtoBuf.DEF_REQ, fnum, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, wtype, oneofs, oneof_names)
        end
        __meta_StructuredValue[]
    end
end
function Base.getproperty(obj::StructuredValue, name::Symbol)
    if name === :none_value
        return (obj.__protobuf_jl_internal_values[name])::NoneValue
    elseif name === :float64_value
        return (obj.__protobuf_jl_internal_values[name])::Float64
    elseif name === :int64_value
        return (obj.__protobuf_jl_internal_values[name])::Int64
    elseif name === :string_value
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :bool_value
        return (obj.__protobuf_jl_internal_values[name])::Bool
    elseif name === :tensor_shape_value
        return (obj.__protobuf_jl_internal_values[name])::TensorShapeProto
    elseif name === :tensor_dtype_value
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :tensor_spec_value
        return (obj.__protobuf_jl_internal_values[name])::TensorSpecProto
    elseif name === :type_spec_value
        return (obj.__protobuf_jl_internal_values[name])::Any
    elseif name === :bounded_tensor_spec_value
        return (obj.__protobuf_jl_internal_values[name])::BoundedTensorSpecProto
    elseif name === :list_value
        return (obj.__protobuf_jl_internal_values[name])::Any
    elseif name === :tuple_value
        return (obj.__protobuf_jl_internal_values[name])::Any
    elseif name === :dict_value
        return (obj.__protobuf_jl_internal_values[name])::Any
    elseif name === :named_tuple_value
        return (obj.__protobuf_jl_internal_values[name])::Any
    else
        getfield(obj, name)
    end
end

mutable struct ListValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function ListValue(; kwargs...)
        obj = new(meta(ListValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct ListValue (has cyclic type dependency)
const __meta_ListValue = Ref{ProtoMeta}()
function meta(::Type{ListValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_ListValue)
            __meta_ListValue[] = target = ProtoMeta(ListValue)
            allflds = Pair{Symbol,Union{Type,String}}[:values => Base.Vector{StructuredValue}]
            meta(target, ListValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_ListValue[]
    end
end
function Base.getproperty(obj::ListValue, name::Symbol)
    if name === :values
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{StructuredValue}
    else
        getfield(obj, name)
    end
end

mutable struct TupleValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function TupleValue(; kwargs...)
        obj = new(meta(TupleValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct TupleValue (has cyclic type dependency)
const __meta_TupleValue = Ref{ProtoMeta}()
function meta(::Type{TupleValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_TupleValue)
            __meta_TupleValue[] = target = ProtoMeta(TupleValue)
            allflds = Pair{Symbol,Union{Type,String}}[:values => Base.Vector{StructuredValue}]
            meta(target, TupleValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_TupleValue[]
    end
end
function Base.getproperty(obj::TupleValue, name::Symbol)
    if name === :values
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{StructuredValue}
    else
        getfield(obj, name)
    end
end

mutable struct DictValue_FieldsEntry <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function DictValue_FieldsEntry(; kwargs...)
        obj = new(meta(DictValue_FieldsEntry), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct DictValue_FieldsEntry (mapentry) (has cyclic type dependency)
const __meta_DictValue_FieldsEntry = Ref{ProtoMeta}()
function meta(::Type{DictValue_FieldsEntry})
    ProtoBuf.metalock() do
        if !isassigned(__meta_DictValue_FieldsEntry)
            __meta_DictValue_FieldsEntry[] = target = ProtoMeta(DictValue_FieldsEntry)
            allflds = Pair{Symbol,Union{Type,String}}[:key => AbstractString, :value => StructuredValue]
            meta(target, DictValue_FieldsEntry, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_DictValue_FieldsEntry[]
    end
end
function Base.getproperty(obj::DictValue_FieldsEntry, name::Symbol)
    if name === :key
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :value
        return (obj.__protobuf_jl_internal_values[name])::StructuredValue
    else
        getfield(obj, name)
    end
end

mutable struct DictValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function DictValue(; kwargs...)
        obj = new(meta(DictValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct DictValue (has cyclic type dependency)
const __meta_DictValue = Ref{ProtoMeta}()
function meta(::Type{DictValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_DictValue)
            __meta_DictValue[] = target = ProtoMeta(DictValue)
            allflds = Pair{Symbol,Union{Type,String}}[:fields => "Base.Dict{AbstractString,StructuredValue}"]
            meta(target, DictValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_DictValue[]
    end
end
function Base.getproperty(obj::DictValue, name::Symbol)
    if name === :fields
        return (obj.__protobuf_jl_internal_values[name])::Any
    else
        getfield(obj, name)
    end
end

mutable struct PairValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function PairValue(; kwargs...)
        obj = new(meta(PairValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct PairValue (has cyclic type dependency)
const __meta_PairValue = Ref{ProtoMeta}()
function meta(::Type{PairValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_PairValue)
            __meta_PairValue[] = target = ProtoMeta(PairValue)
            allflds = Pair{Symbol,Union{Type,String}}[:key => AbstractString, :value => StructuredValue]
            meta(target, PairValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_PairValue[]
    end
end
function Base.getproperty(obj::PairValue, name::Symbol)
    if name === :key
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :value
        return (obj.__protobuf_jl_internal_values[name])::StructuredValue
    else
        getfield(obj, name)
    end
end

mutable struct NamedTupleValue <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function NamedTupleValue(; kwargs...)
        obj = new(meta(NamedTupleValue), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct NamedTupleValue (has cyclic type dependency)
const __meta_NamedTupleValue = Ref{ProtoMeta}()
function meta(::Type{NamedTupleValue})
    ProtoBuf.metalock() do
        if !isassigned(__meta_NamedTupleValue)
            __meta_NamedTupleValue[] = target = ProtoMeta(NamedTupleValue)
            allflds = Pair{Symbol,Union{Type,String}}[:name => AbstractString, :values => Base.Vector{PairValue}]
            meta(target, NamedTupleValue, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_NamedTupleValue[]
    end
end
function Base.getproperty(obj::NamedTupleValue, name::Symbol)
    if name === :name
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    elseif name === :values
        return (obj.__protobuf_jl_internal_values[name])::Base.Vector{PairValue}
    else
        getfield(obj, name)
    end
end

const TypeSpecProto_TypeSpecClass = (;[
    Symbol("UNKNOWN") => Int32(0),
    Symbol("SPARSE_TENSOR_SPEC") => Int32(1),
    Symbol("INDEXED_SLICES_SPEC") => Int32(2),
    Symbol("RAGGED_TENSOR_SPEC") => Int32(3),
    Symbol("TENSOR_ARRAY_SPEC") => Int32(4),
    Symbol("DATA_DATASET_SPEC") => Int32(5),
    Symbol("DATA_ITERATOR_SPEC") => Int32(6),
    Symbol("OPTIONAL_SPEC") => Int32(7),
    Symbol("PER_REPLICA_SPEC") => Int32(8),
    Symbol("VARIABLE_SPEC") => Int32(9),
    Symbol("ROW_PARTITION_SPEC") => Int32(10),
    Symbol("NDARRAY_SPEC") => Int32(11),
]...)

mutable struct TypeSpecProto <: ProtoType
    __protobuf_jl_internal_meta::ProtoMeta
    __protobuf_jl_internal_values::Dict{Symbol,Any}
    __protobuf_jl_internal_defaultset::Set{Symbol}

    function TypeSpecProto(; kwargs...)
        obj = new(meta(TypeSpecProto), Dict{Symbol,Any}(), Set{Symbol}())
        values = obj.__protobuf_jl_internal_values
        symdict = obj.__protobuf_jl_internal_meta.symdict
        for nv in kwargs
            fldname, fldval = nv
            fldtype = symdict[fldname].jtyp
            (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname))
            values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval)
        end
        obj
    end
end # mutable struct TypeSpecProto (has cyclic type dependency)
const __meta_TypeSpecProto = Ref{ProtoMeta}()
function meta(::Type{TypeSpecProto})
    ProtoBuf.metalock() do
        if !isassigned(__meta_TypeSpecProto)
            __meta_TypeSpecProto[] = target = ProtoMeta(TypeSpecProto)
            allflds = Pair{Symbol,Union{Type,String}}[:type_spec_class => Int32, :type_state => StructuredValue, :type_spec_class_name => AbstractString]
            meta(target, TypeSpecProto, allflds, ProtoBuf.DEF_REQ, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES)
        end
        __meta_TypeSpecProto[]
    end
end
function Base.getproperty(obj::TypeSpecProto, name::Symbol)
    if name === :type_spec_class
        return (obj.__protobuf_jl_internal_values[name])::Int32
    elseif name === :type_state
        return (obj.__protobuf_jl_internal_values[name])::StructuredValue
    elseif name === :type_spec_class_name
        return (obj.__protobuf_jl_internal_values[name])::AbstractString
    else
        getfield(obj, name)
    end
end

export StructuredValue, NoneValue, ListValue, TupleValue, DictValue_FieldsEntry, DictValue, PairValue, NamedTupleValue, TensorSpecProto, BoundedTensorSpecProto, TypeSpecProto_TypeSpecClass, TypeSpecProto, StructuredValue, ListValue, TupleValue, DictValue_FieldsEntry, DictValue, PairValue, NamedTupleValue, TypeSpecProto_TypeSpecClass, TypeSpecProto
# mapentries: "DictValue_FieldsEntry" => ("AbstractString", "StructuredValue")


text:
using ReactiveBasics
using FactCheck

number() = round(Int, rand()*1000)

abstract Action{T}
type Update{T} <: Action{T}
    val::T
end
type Reset{T} <: Action{T}
    val::T
end

## Basics

facts("Basic checks") do

    a = Signal(number())
    b = map(x -> x*x, a)

    context("Signal") do
        as = Signal(Action, Update(1))
        @fact typeof(value(as)) --> Update{Int64}
        push!(as, Reset(1))
        @fact typeof(value(as)) --> Reset{Int64}
    end

    context("map") do

        # Lift type
        #@fact typeof(b) --> Reactive.Lift{Int}

        # type conversion
        push!(a, 1.0)
        @fact value(b) --> 1
        @fact value(b) --> 1

        push!(a, number())
        @fact value(b) --> value(a)^2

        push!(a, -number())
        @fact value(b) --> value(a)^2

        ## Multiple inputs to map
        c = map(+, a, b)
        @fact value(c) --> value(a) + value(b)

        push!(a, number())
        @fact value(c) --> value(a) + value(b)

        push!(b, number())
        @fact value(c) --> value(a) + value(b)

        as = Signal(0)
        bs = map(Update, as, typ = Action{Int64})
        @fact typeof(bs) --> Signal{Action{Int64}}
    end

    context("zipmap") do

        as = Signal(1)
        bs = map(a -> a * 0.1, as)
        cs = zipmap((a,b) -> a + b, as, bs)
        counts = foldp((cnt,_) -> cnt + 1, 0, cs)

        @fact value(counts) --> 1
        @fact value(cs) --> 1.1

        push!(as, 2)
        @fact value(counts) --> 2
        @fact value(cs) --> 2.2
    end

    context("merge") do

        ## Merge
        d = Signal(number())
        e = merge(d, b, a)

        # precedence to d
        @fact value(e) --> value(d)

        push!(a, number())
        # Note that his works differently than Reactive.jl because of the
        # way updates are pushed.
        @fact value(e) --> value(a)

        # Merge two different signal types
        as = Signal(Update(1))
        bs = Signal(Reset(2))
        cs = merge(as, bs)
        @fact typeof(value(cs)) --> Update{Int64}
    end

    context("zip") do

        ## zip
        d = Signal(number())
        b = Signal(number())
        a = Signal(number())
        e = zip(d, b, a)
        @fact value(e) --> (value(d), value(b), value(a))

        d = Signal(1)
        b = Signal(2)
        a = Signal(3)
        e = zip(d, b, a)
        @fact value(e) --> (1,2,3)

        push!(a, 6)
        @fact value(e) --> (1,2,3)
        push!(d, 4)
        @fact value(e) --> (1,2,3)
        push!(b, 5)
        @fact value(e) --> (4,5,6)

        e = zip(d, b, a, Signal(3))
        @fact value(e) --> (value(d), value(b), value(a), 3)

        as = Signal(Action{Int64}, Update(1))
        bs = Signal(Reset(2))
        cs = zip(as, bs)
        @fact typeof(cs) --> Signal{Tuple{Action{Int64}, Reset{Int64}}}

        push!(as, Reset(3))
        @fact typeof(cs) --> Signal{Tuple{Action{Int64}, Reset{Int64}}}

        d = Signal(1)
        b = Signal(2)
        a = Signal(3)
        e = zip(d, b, a, max_buffer_size = 1)
        @fact value(e) --> (1,2,3)
        push!(a, 1)
        @fact value(e) --> (1,2,3)
        error = false
        try
            push!(a, 1)
        catch
            error = true
        end
        @fact error --> true
        push!(b, 1)
        push!(d, 2)
        @fact value(e) --> (2, 1, 1)

        d = Signal(1)
        b = Signal(2)
        a = Signal(3)
        e = zip(d, b, a, max_buffer_size = 2)
        @fact value(e) --> (1,2,3)
        push!(a, 1)
        push!(a, 2)
        @fact value(e) --> (1,2,3)
        error = false
        try
            push!(a, 1)
        catch
            error = true
        end
        @fact error --> true
        push!(b, 1)
        push!(d, 2)
        @fact value(e) --> (2, 1, 1)
    end

    context("foldp") do
        a = Signal(0)
        ## foldp over time
        push!(a, 0)
        f = foldp(+, 0, a)
        nums = [6,3,1]
        map(x -> push!(a, x), nums)
        @fact sum(nums) --> value(f)

        x = Signal(ones(4,5))
        y = foldp((b,a) -> b + a, ones(4,5) * 2, x)
        @fact value(y) --> ones(4,5) * 3
        push!(x, ones(4,5))
        @fact value(y) --> ones(4,5) * 4
    end

    context("filter") do
        # filter
        g = Signal(0)
        pred = x -> x % 2 != 0
        h = filter(pred, 1, g)
        j = filter(x -> x % 2 == 0, 1, g)

        @fact value(h) --> 1
        @fact value(j) --> 0

        push!(g, 2)
        @fact value(h) --> 1

        push!(g, 3)
        @fact value(h) --> 3
    end

    context("filterwhen") do
        # filterwhen
        bs = Signal(false)
        as = Signal(1)
        cs = filterwhen(bs, 9, as)
        @fact value(cs) --> 9

        bs = Signal(true)
        as = Signal(1)
        cs = filterwhen(bs, 9, as)
        @fact value(cs) --> 1

        push!(as, 2)
        @fact value(cs) --> 2
        push!(bs, false)
        @fact value(cs) --> 2
        push!(as, 5)
        @fact value(cs) --> 2
        push!(bs, true)
        @fact value(cs) --> 5
    end

    context("push! inside push!") do
        a = Signal(0)
        b = Signal(1)
        subscribe!(x -> push!(a, x), b)
        @fact value(a) --> 0

        push!(a, 2)
        @fact value(a) --> 2
        @fact value(b) --> 1

        push!(b, 3)
        @fact value(b) --> 3
        @fact value(a) --> 3

    end

    context("jw3126") do   # https://github.com/JuliaLang/Reactive.jl/issues/101
        x1 = Signal(1)
        x2 = Signal(10)
        y1 = map(identity, x1)
        y2 = map(identity, x2)
        y12 = map(+, x1, x2)
        z = map(+, y1, y2, y12)
        y12 = map(+, x1, x2)
        z = map(+, y1, y2, y12)
        push!(x1, 3)
        @fact value(x1)  -->  3
        @fact value(x2)  --> 10
        @fact value(y1)  -->  3
        @fact value(y2)  --> 10
        @fact value(y12) --> 13
        @fact value(z)   --> 26
        zz = map(+, y1, y2, y12, Signal(3))
        push!(x1, 3)
        @fact value(zz)  --> 29
    end

    context("asyncmap") do
        x = Signal(1)
        y = asyncmap(-, 0, x)

        @sync push!(x, 2)

        @fact value(y) --> -2

        x = Signal(1)
        y = asyncmap(0, x) do z
            sleep(2)
            -z
        end

        @sync push!(x, 2)

        @fact value(y) --> -2
    end

    context("flatmap") do

        a = Signal(1)
        u = Signal(3)
        b = flatmap(a) do x
            x > 10 ? Signal(1 + x) : u
        end
        @fact value(b) --> 3
        push!(a, 5)
        @fact value(b) --> 3
        push!(u, 6)
        @fact value(b) --> 6
        push!(a, 15)
        @fact value(b) --> 16
    end

    context("sampleon") do
        # sampleon
        g = Signal(0)

        push!(g, number())
        i = Signal(true)
        j = sampleon(i, g)
        # default value
        @fact value(j) --> value(g)
        push!(g, value(g)-1)
        @fact value(j) --> value(g)+1
        push!(i, true)
        @fact value(j) --> value(g)
    end

    context("droprepeats") do
        # droprepeats
        count = s -> foldp((x, y) -> x+1, -1, s)

        k = Signal(1)
        l = droprepeats(k)

        @fact value(l) --> value(k)
        push!(k, 1)
        @fact value(l) --> value(k)
        push!(k, 0)
        #println(l.value, " ", value(k))
        @fact value(l) --> value(k)

        m = count(k)
        n = count(l)

        seq = [1, 1, 1, 0, 1, 0, 1, 0, 0]
        map(x -> push!(k, x), seq)

        @fact value(m) --> length(seq)
        @fact value(n) --> 6
    end

    context("skip") do
        x = Signal(0)
        y = skip(2, x)
        count = foldp((x, y) -> x+1, -1, y)
        @fact value(y) --> 0
        @fact value(count) --> 0

        push!(x, 1)
        @fact value(y) --> 0
        @fact value(count) --> 0

        push!(x, 2)
        @fact value(y) --> 0
        @fact value(count) --> 0

        push!(x, 3)
        @fact value(y) --> 3
        @fact value(count) --> 1

        push!(x, 4)
        @fact value(y) --> 3
        @fact value(count) --> 1

        push!(x, 5)
        @fact value(y) --> 3
        @fact value(count) --> 1

        push!(x, 6)
        @fact value(y) --> 6
        @fact value(count) --> 2
    end

    context("previous") do
        x = Signal(0)
        y = previous(x)
        @fact value(y) --> 0

        push!(x, 1)

        @fact value(y) --> 0

        push!(x, 2)

        @fact value(y) --> 1

        push!(x, 3)

        @fact value(y) --> 2

        x = Signal(zeros(2,3))
        y = previous(x)
        @fact value(y) --> zeros(2,3)

        push!(x, ones(2,3))

        @fact value(y) --> zeros(2,3)

        push!(x, ones(2,3) * 2)

        @fact value(y) --> ones(2,3)

        push!(x, ones(2,3) * 3)

        @fact value(y) --> ones(2,3) * 2
    end

    context("bind!") do
        x = Signal(1)
        y = Signal(2)
        xx = map(u -> 2u, x)
        yy = map(u -> 3u, y)
        bind!(x, y)
        @fact value(y) --> value(x)
        @fact value(y) --> 2
        push!(x, 10)
        @fact value(y) --> value(x)
        @fact value(y) --> 10
        @fact value(yy) --> 30
        push!(y, 20)
        @fact value(y) --> value(x)
        @fact value(y) --> 20
        @fact value(xx) --> 40

        x = Signal(1)
        y = Signal(2)
        xx = map(u -> 2u, x)
        yy = map(u -> 3u, y)
        bind!(x, y, false)
        @fact value(y) --> value(x)
        @fact value(y) --> 2
        push!(x, 10)
        @fact value(y) --> 2
        @fact value(x) --> 10
        push!(y, 20)
        @fact value(y) --> value(x)
        @fact value(y) --> 20
        @fact value(xx) --> 40

        x = Signal(1)
        y = Signal(2)
        xx = map(u -> 2u, x)
        yy = map(u -> 3u, y)
        bind!(x, y, initial = false)
        @fact value(x) --> 1
        @fact value(y) --> 2
        push!(x, 10)
        @fact value(y) --> 10
        @fact value(yy) --> 30
        push!(y, 20)
        @fact value(y) --> value(x)
        @fact value(y) --> 20
        @fact value(xx) --> 40
    end

    context("preserve") do
        x = Signal(1)
        z = let xx = map(u -> 2u, x),
            x2 = preserve(map(u -> 2u, x))
            map(+, xx, x2)
        end
        push!(x, 10)
        @fact value(z) --> 40
    end
end

facts("Flatten") do

    a = Signal(0)
    b = Signal(1)

    c = Signal(a)

    d = flatten(c)
    cnt = foldp((x, y) -> x+1, -1, d)

    context("Signal{Signal} -> flat Signal") do
        # Flatten implies:
        @fact value(c) --> a
        @fact value(d) --> value(a)
    end

    context("Initial update count") do

        @fact value(cnt) --> 0
    end

    context("Current signal updates") do
        push!(a, 2)

        @fact value(cnt) --> 1
        @fact value(d) --> value(a)
    end

    context("Signal swap") do
        push!(c, b)
        @fact value(cnt) --> 2
        @fact value(d) --> value(b)

        push!(a, 3)
        @fact value(cnt) --> 2
        @fact value(d) --> value(b)

        push!(b, 3)

        @fact value(cnt) --> 3
        @fact value(d) --> value(b)
    end
end