text:
# This file is a part of Julia. License is MIT: http://julialang.org/license
__precompile__()

module OctoRelocator

import Base: GMP, Bottom, unsafe_convert, uncompressed_ast
import Base: get!, getindex, setindex!
import Core: arrayref, arrayset

# a much faster ObjectIdDict, but which does not guarantee object uniqueness for isbits objects
type FuzzyDict
    dict::Dict{UInt, Int}
    FuzzyDict() = new(Dict{UInt, Int}())
end
get!(d::FuzzyDict, k::ANY, v) = get!(d.dict, UInt(pointer_from_objref(k)), v)
getindex(d::FuzzyDict, k::ANY) = getindex(d.dict, UInt(pointer_from_objref(k)))
setindex!(d::FuzzyDict, v, k::ANY) = setindex!(d.dict, v, UInt(pointer_from_objref(k)))
typealias ObjectIdDict FuzzyDict

if !isdefined(Base, :unsafe_write)
unsafe_write(s::IO, p::Ptr{UInt8}, nb::UInt) = write(s, p, Int(nb))
unsafe_read(s::IO, p::Ptr{UInt8}, nb::UInt) = unsafe_copy(p, pointer(read(s, Array{UInt8}(nb))), nb)
end

export serialize, deserialize

## serializing values ##

type FastSerializationState{I<:IO}
    io::I
    hcounter::Int
    ocounter::Int
    serialized::ObjectIdDict
    workq::Vector{Any}
    relocarrayid::Vector{Int}
    relocarrays::Vector{Vector{Int}}
    FastSerializationState(io::I) = new(io, 0, 0, ObjectIdDict(), Vector{Any}(), Vector{Int}(), Vector{Vector{Int}}())
end
FastSerializationState(io::IO) = FastSerializationState{typeof(io)}(io)
typealias SerializationState FastSerializationState

function serialize(s::IO, x)
    fastserialize(FastSerializationState(s), x)
    return s
end
serialize(x) = serialize(PipeBuffer(), x)

if VERSION < v"0.5-" || VERSION > v"0.7-"
    error("Serializer not defined for this version.")
elseif VERSION >= v"0.6-"
const TAGS = Any[
    Int8, UInt8, Int16, UInt16, Int32, UInt32,
    Int64, UInt64, Int128, UInt128, Float32, Float64, Char, Ptr,
    DataType, Union, TypeName,
    Tuple, Array,
    Bool, Any, Type, Tuple{},
    Symbol, #=LongSymbol=#Symbol, # dummy entry, intentionally shadowed by earlier ones
    LineNumberNode, Slot, LabelNode, GotoNode,
    QuoteNode, CodeInfo, TypeVar, Core.Box, Core.MethodInstance,
    Module, #=UndefRefTag=#Symbol, Task, String, Float16,
    SimpleVector, #=Any Value=#Symbol, Method, GlobalRef,

    (), :Any, Bottom,
    :Array, :TypeVar, :Tuple,
    :Core, :Base, Core.svec(),
    false, true, nothing, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
    12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
    28, 29, 30, 31, 32
]
elseif VERSION >= v"0.5-"
const TAGS = Any[
    Int8, UInt8, Int16, UInt16, Int32, UInt32,
    Int64, UInt64, Int128, UInt128, Float32, Float64, Char, Ptr,
    DataType, Union, TypeName,
    Tuple, Array,
    Bool, Any, Type, Tuple{},
    Symbol, #=LongSymbol=#Symbol, # dummy entry, intentionally shadowed by earlier ones
    LineNumberNode, Slot, LabelNode, GotoNode,
    QuoteNode, #=reserved=#Symbol, TypeVar, Core.Box, LambdaInfo,
    Module, #=UndefRefTag=#Symbol, Task, String, Float16,
    SimpleVector, #=Any Value=#Symbol, Method, GlobalRef,

    (), :Any, Bottom,
    :Array, :TypeVar, :Tuple,
    :Core, :Base, Core.svec(),
    false, true, nothing, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
    12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
    28, 29, 30, 31, 32
]
end

const ser_version = 4 # do not make changes without bumping the version #!

@inline function sertag(v::ANY)
    ptr = pointer_from_objref(v)
    ptags = convert(Ptr{Ptr{Void}}, pointer(TAGS))
    t = typeof(v)
    if is(t,DataType)
        s = 1
        e = VALUE_TAGS - 1
    else
        s = VALUE_TAGS
        e = ZERO_TAG - 1
    end
    @inbounds for i = s:e
        ptr == unsafe_load(ptags,i) && return (i+1)%Int32
    end
    return Int32(-1)
end
desertag(i::Int32) = TAGS[i-1]

# tags >= this just represent themselves, their whole representation is 1 byte
const VALUE_TAGS = (findfirst(x -> x === (), TAGS) + 1)%Int32
const ZERO_TAG = (findfirst(x -> x === 0, TAGS) + 1)%Int32
const SIMPLEVECTOR_TAG = sertag(SimpleVector)
const SYMBOL_TAG = sertag(Symbol)
const LONGSYMBOL_TAG = Int32(sertag(Symbol)+1)
const ARRAY_TAG = sertag(Array)
const UNDEFREF_TAG = Int32(sertag(Module)+1)
const VALUE_TAG = Int32(sertag(SimpleVector)+1)
const MODULE_TAG = sertag(Module)
const TASK_TAG = sertag(Task)
const DATATYPE_TAG = sertag(DataType)
const INT_TAG = sertag(Int)
const UNION_TAG = sertag(Union)

writetag(s::IO, tag) = (write(s, UInt8(tag)); nothing)

function write_as_tag(s::IO, tag)
    tag < VALUE_TAGS && write(s, UInt8(0))
    write(s, UInt8(tag))
    nothing
end

macro writefield(s, x, xptr, t, i, fldty) # precondition: isbits(fldty::DataType) && 1 <= i <= nfields(t)
    return esc(quote
        unsafe_write(s.io, convert(Ptr{UInt8}, xptr + fieldoffset(t, i)), UInt(fldty.size))
    end)
    #ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), x, i-1, read(s.io, fldty))
end

function readfield{T}(::Type{T}, s::IO, fptr)
    v = read(s, T)
    unsafe_store!(convert(Ptr{T}, fptr), v)
    nothing
end

macro readfield(s, x, xptr, t, i, fldty) # precondition: isbits(fldty::DataType) && 1 <= i <= nfields(t)
    return esc(quote
        unsafe_read(s.io, convert(Ptr{UInt8}, xptr + Base.fieldoffset(t, i)), UInt(fldty.size))
    end)
    return esc(quote
        fldsz = $fldty.size
        if fldsz == 1
            readfield(UInt8, s.io, xptr + Base.field_offset(t, i - 1))
        elseif fldsz == 2
            readfield(UInt16, s.io, xptr + Base.field_offset(t, i - 1))
        elseif fldsz == 4
            readfield(UInt32, s.io, xptr + Base.field_offset(t, i - 1))
        elseif fldsz == 8
            readfield(UInt64, s.io, xptr + Base.field_offset(t, i - 1))
        else
            readfield($fldty, s.io, xptr + Base.field_offset(t, i - 1))
        end
        #ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), x, i-1, read(s.io, fldty))
    end)
end

if VERSION < v"0.5-"
    LLT_ALIGN(v::Unsigned, typ::DataType) = (v + Base.type_alignment(typ) - 1) & ~UInt(Base.type_alignment(typ) - 1)
    const fielddesc_type_offset = LLT_ALIGN(Base.field_offset(DataType,
            findfirst(Base.fieldnames(DataType), :ninitialized) - 1) +
        sizeof(fieldtype(DataType, :ninitialized)), UInt32) + sizeof(UInt32)
    const fielddesc_offset = LLT_ALIGN(
        LLT_ALIGN(fielddesc_type_offset + sizeof(UInt32), Ptr{Void}) + 2 * sizeof(Ptr{Void}),
        DataType)
    function jl_field_isptr(t::DataType, i::Int) # precondition: 1 <= i <= nfields(t)
        tptr = pointer_from_objref(t)
        fielddesc_type = (unsafe_load(convert(Ptr{UInt32}, tptr) + fielddesc_type_offset) >> 30) & 0x2
        i = 2 * i
        if fielddesc_type == 0
            isptr = (unsafe_load(convert(Ptr{UInt8}, tptr) + fielddesc_offset, i) >> 7) != 0
        elseif fielddesc_type == 1
            isptr = (unsafe_load(convert(Ptr{UInt16}, tptr) + fielddesc_offset, i) >> 15) != 0
        else
            isptr = (unsafe_load(convert(Ptr{UInt32}, tptr) + fielddesc_offset, i) >> 31) != 0
        end
        #fldty = t.types[i]
        #return typeof(fldty) === DataType && isbits(fldty::DataType)
    end
else
    const fielddesc_type_offset = sizeof(UInt32)
    const fielddesc_offset = 2 * sizeof(UInt32)
    function jl_field_isptr(t::DataType, i::Int) # precondition: 1 <= i <= nfields(t)
        tptr = t.layout
        fielddesc_type = (unsafe_load(convert(Ptr{UInt32}, tptr) + fielddesc_type_offset) >> 30) & 0x2
        return unsafe_load(convert(Ptr{UInt8}, tptr + fielddesc_offset + ((i - 1) << (fielddesc_type + 1)))) & 1 != 0
    end
end

@inline function fastserialize_any(s::SerializationState, obj::ANY)
    t = typeof(obj)
    if is(t, DataType) || is(t, Symbol) || is(t, Module) || is(t, Union)
        return serialize_header_obj(s, obj)
    end
    xid = get!(s.serialized, obj, s.ocounter)::Int
    if xid == s.ocounter
        s.ocounter -= 1
        push!(s.workq, obj)
    end
    return xid
end

function fastserialize(s::SerializationState, obj::ANY)
    workq = s.workq
    xid = 1
    s.hcounter = 1
    s.ocounter = -1
    id = fastserialize_any(s, obj)
    relocations = Vector{Tuple{Int,Int,Int}}()
    while xid <= length(workq)
        x = workq[xid]
        tag = sertag(x)
        if tag <= 0
            t = typeof(x)::DataType
            @assert !(is(t, DataType) || is(t, Symbol) || is(t, Module) || is(t, Union))
            if is(t.name, Array.name)
                serialize_array(s, x, xid)
            elseif is(t, SimpleVector)
                #serialize_sv(s, x::SimpleVector, xid)
                sv = x::SimpleVector
                for i in 1:length(sv)
                    id = fastserialize_any(sv[i])::Int
                    push!(relocations, (xid, i, id))
                end
            else
                t_tag = sertag(t)
                t_tag <= 0 && serialize_header_obj(s, t)
                nf = nfields(t)
                for i in 1:nf
                    if jl_field_isptr(t, i)
                        if isdefined(x, i)
                            xf = getfield(x, i)
                            id = fastserialize_any(s, xf)
                            push!(relocations, (xid, i, id))
                        end
                    end
                end
            end
        end
        xid += 1
    end
    writetag(s.io, UNDEFREF_TAG) # mark the end of the header

    for xid = 1:length(workq)
        x = workq[xid]
        tag = sertag(x)
        if tag > 0
            write_as_tag(s.io, tag)
        else
            t = typeof(x)::DataType
            @assert !(is(t, DataType) || is(t, Symbol) || is(t, Module) || is(t, Union))
            if is(t.name, Array.name)
                serialize_array(s, x::Array, 0)
            elseif is(t, SimpleVector)
                #serialize_sv(s, x::SimpleVector, 0)
                sv = x::SimpleVector
                writetag(s.io, SIMPLEVECTOR_TAG)
                l = length(sv)::Int
                write(s.io, l)
            elseif isa(x, Int)
                serialize_int(s, x::Int)
            else
                t_tag = sertag(t)
                if t_tag > 0
                    writetag(s.io, t_tag)
                else
                    writetag(s.io, VALUE_TAG)
                    write(s.io, s.serialized[t]::Int)
                end

                nf = nfields(t)
                xptr = pointer_from_objref(x)
                for i in 1:nf
                    if !jl_field_isptr(t, i)
                        fldty = t.types[i]::DataType
                        @writefield(s, x, xptr, t, i, fldty)
                    end
                end
            end
        end
    end

    # write all of the pending relocations
    writetag(s.io, UNDEFREF_TAG)
    write(s.io, Int(length(relocations)))
    write(s.io, relocations)
    write(s.io, length(s.relocarrays))
    for i = 1:length(s.relocarrays)
        write(s.io, s.relocarrayid[i])
        write(s.io, s.relocarrays[i])
    end
    write(s.io, s.serialized[obj]::Int)
    nothing
end

# the header contains objects that cannot be recursive,
# esp. all types that might be needed later
function serialize_header_obj(s::SerializationState, x::ANY)
    xid = get!(s.serialized, x, 0)::Int
    xid > 0 && return xid

    tag = sertag(x)
    if tag > 0
        write_as_tag(s.io, tag)
    else
        t = typeof(x)::DataType
        if isa(x, DataType)
            serialize_datatype(s, x)
        elseif isa(x, Module)
            serialize_module(s, x)
        elseif isa(x, Symbol)
            serialize_symbol(s, x)
        elseif isa(x, Int)
            serialize_int(s, x)
        elseif isa(x, Union)
            serialize_union(s, x)
        else
            t_tag = sertag(t)
            if t_tag > 0
                writetag(s.io, t_tag)
            else
                tid = serialize_header_obj(s, t)
                writetag(s.io, VALUE_TAG)
                write(s.io, tid)
            end
            nf = nfields(t)
            if nf == 0 && t.size > 0 # bitstype
                write(s.io, x)
            else # other isbits
                @assert isbits(t)
                xptr = pointer_from_objref(x)
                for i = 1:nf
                    fldty = t.types[i]::DataType
                    @writefield(s, x, xptr, t, i, fldty)
                end
            end
        end
    end

    xid = s.hcounter
    s.hcounter += 1
    s.serialized[x] = xid
    return xid
end

## HEADER OBJECTS ##################################

function serialize_mod_names(s::SerializationState, m::Module, writeout::Bool)
    p = module_parent(m)
    if m !== p
        serialize_mod_names(s, p, writeout)
        name = module_name(m)
        if writeout
            write(s.io, s.serialized[name]::Int)
        else
            serialize_header_obj(s, name)
        end
    end
    nothing
end

function serialize_module(s::SerializationState, m::Module)
    serialize_mod_names(s, m, false)
    writetag(s.io, MODULE_TAG)
    serialize_mod_names(s, m, true)
    write(s.io, 0)
    nothing
end

function serialize_datatype(s::SerializationState, t::DataType)
    tname = t.name.name
    id_tname = serialize_header_obj(s, tname)
    mod = t.name.module
    id_mod = serialize_header_obj(s, mod)
    count = -1
    tp = t.parameters
    if !isempty(tp)
        if isdefined(mod, tname) && is(t, getfield(mod, tname))
            count = 0
        else
            count = length(tp)
            for i = 1:count
                serialize_header_obj(s, tp[i])
            end
        end
    end
    writetag(s.io, DATATYPE_TAG)
    write(s.io, id_tname)
    write(s.io, id_mod)
    if count >= 0
        write(s.io, count)
        for i = 1:count
            write(s.io, s.serialized[tp[i]]::Int)
        end
    end
    nothing
end

function serialize_int(s::SerializationState, n::Int)
    if 0 <= n <= 32
        write(s.io, UInt8(ZERO_TAG+n))
        return
    end
    writetag(s.io, INT_TAG)
    write(s.io, n)
    nothing
end

function serialize_union(s::SerializationState, u::Union)
    ut = u.types
    count = length(ut)::Int
    for i = 1:count
        serialize_header_obj(s, ut[i])
    end
    writetag(s.io, UNION_TAG)
    write(s.io, count)
    for i = 1:count
        serialize_header_obj(s, s.serialized[ut[i]]::Int)
    end
    nothing
end

function serialize_symbol(s::SerializationState, x::Symbol)
    pname = unsafe_convert(Ptr{UInt8}, x)
    ln = UInt(ccall(:strlen, Csize_t, (Ptr{UInt8},), pname))
    if ln <= 255
        writetag(s.io, SYMBOL_TAG)
        write(s.io, UInt8(ln))
    else
        writetag(s.io, LONGSYMBOL_TAG)
        write(s.io, Int32(ln))
    end
    unsafe_write(s.io, pname, ln)
    nothing
end

## MISC OBJECT SERIALIZERS ##################################

function serialize_array_bits(s::IO, a)
    elty = eltype(a)
    if elty === Bool && !isempty(a)
        last = a[1]
        count = 1
        for i = 2:length(a)
            if a[i] != last || count == 127
                write(s, UInt8((UInt8(last) << 7) | count))
                last = a[i]
                count = 1
            else
                count += 1
            end
        end
        write(s, UInt8((UInt8(last) << 7) | count))
    else
        write(s, a)
    end
    nothing
end

function serialize_array(s::SerializationState, a::Array, xid::Int)
    if xid != 0
        elty = eltype(a)
        serialize_header_obj(s, elty)
        if typeof(elty) !== DataType || !isbits(elty::DataType)
            na = length(a)::Int
            reloca = Array{Int}(na)
            for fld = 1:na
                if isassigned(a, fld)
                    id = fastserialize_any(s, a[fld])::Int
                else
                    id = 0
                end
                reloca[fld] = id
            end
            push!(s.relocarrayid, xid)
            push!(s.relocarrays, reloca)
        end
    else
        elty = eltype(a)
        writetag(s.io, ARRAY_TAG)
        write(s.io, s.serialized[elty]::Int)
        nd = ndims(a)::Int
        write(s.io, nd)
        for i = 1:nd
            write(s.io, size(a, i))
        end
        if typeof(elty) === DataType && isbits(elty::DataType)
            serialize_array_bits(s.io, a)
        end
    end
    nothing
end

function serialize_sv(s::SerializationState, v::SimpleVector, xid::Int)
    if xid != 0
        for x in v
            id = fastserialize_any(s, x)
        end
        nothing
    end
end

#
#
#function fastserialize_any(s::SerializationState, obj::ANY)
#    xid::Int = get!(s.serialized, obj, 0)::Int
#    if xid != 0
#        writetag(s.io, UNDEFREF_TAG)
#        write(s.io, xid)
#        return
#    end
#    relocations = Vector{Any}()
#    relocarrayid = Vector{Int}()
#    relocarrays = Vector{Vector{Int}}()
#    push!(relocations, obj)
#    fld::Int = 1
#    while !isempty(relocations)
#        x = relocations[1]
#        xid = s.serialized[x]::Int
#        reloc::Tuple{Int,Int,Int} = (0, 0, 0)
#        if xid == 0
#            shift!(relocations)
#        else
#            # already serialized x, now look at its fields
#            if isa(x, Array)
#                x = x::Array
#                na::Int = length(x)
#                while fld <= na
#                    if isdefined(x, fld)
#                        xf = arrayref(x, fld)
#                        id = get!(s.serialized, xf, 0)::Int
#                        if id == 0
#                            # need to serialize this value
#                            x = xf
#                            break
#                        end
#                    end
#                    fld += 1
#                end
#                if fld > na
#                    # finished with serializing this array
#                    reloca = Array{Int}(na)
#                    for fld = 1:na
#                        if isdefined(x, fld)
#                            id = s.serialized[arrayref(x, fld)]::Int
#                        else
#                            id = 0
#                        end
#                        reloca[fld] = id
#                    end
#                    push!(relocarrayid, xid)
#                    push!(relocarrays, reloca)
#                    fld = 1
#                    xid = 0
#                    shift!(relocations)
#                    continue
#                end
#                fld += 1
#            else
#                t = typeof(x)::DataType
#                nf::Int = nfields(t)
#                while fld <= nf
#                    fldty = t.types[i]
#                    if typeof(fldty) !== DataType || !isbits(fldty::DataType)
#                        if isdefined(x, fld)
#                            xf = getfield(x, fld)
#                            id = s.serialized[xf]::Int
#                            if id == 0
#                                # need to serialize this value
#                                reloc = (xid, fld, s.counter)
#                                x = xf
#                                break
#                            elseif id > xid
#                                # already got serialized, just record the relocation
#                                push!(s.relocations, (xid, fld, id))
#                            end
#                        end
#                    end
#                    fld += 1
#                end
#                if fld > nf
#                    # finished with serializing this object
#                    fld = 1
#                    xid = 0
#                    shift!(relocations)
#                    continue
#                end
#                fld += 1
#            end
#        end
#
#        # serialize x
#        xid = s.counter
#        s.counter += 1
#        s.serialized[x] = xid
#        tag = sertag(x)
#        if tag > 0
#            write_as_tag(s.io, tag)
#        else
#            t = typeof(x)::DataType
#            #if isa(x, Tuple)
#            #    serialize_tuple(s, x)
#            if isa(x, Array)
#                serialize_array(s, x, relocations)
#            elseif isa(x, DataType)
#                serialize_datatype(s, x)
#            elseif isa(x, Symbol)
#                serialize_symbol(s, x)
#            elseif isa(x, Module)
#                serialize_module(s, x)
#            elseif isa(x, Expr)
#                serialize_expr(s, x)
#            elseif isa(x, SimpleVector)
#                serialize_sv(s, x)
#            elseif isa(x, Int)
#                serialize_int(s, x)
#            elseif isa(x, Union)
#                serialize_union(s, x)
#            else
#                t_tag = sertag(t)
#                if t_tag > 0
#                    writetag(s.io, t_tag)
#                else
#                    writetag(s.io, VALUE_TAG)
#                    fastserialize_any(s, t)
#                end
#
#                nf = nfields(t)
#                if nf == 0 && t.size > 0 # bitstype
#                    write(s.io, x)
#                else
#                    needsreloc = false
#                    xptr = pointer_from_objref(x)
#                    for i in 1:nf
#                        fldty = t.types[i]
#                        wasbits = false
#                        if typeof(fldty) === DataType
#                            fldty = fldty::DataType
#                            if isbits(fldty)
#                                fldsz = fldty.size
#                                if fldsz == 1
#                                    readfield(UInt8, s, xptr + Base.field_offset(t, i))
#                                elseif fldsz == 2
#                                    readfield(UInt16, s, xptr + Base.field_offset(t, i))
#                                elseif fldsz == 4
#                                    readfield(UInt32, s, xptr + Base.field_offset(t, i))
#                                elseif fldsz == 8
#                                    readfield(UInt64, s, xptr + Base.field_offset(t, i))
#                                else
#                                    readfield(fldty, s, xptr + Base.field_offset(t, i))
#                                end
#                                #write(s.io, getfield(x, i))
#                                wasbits = true
#                            end
#                        end
#                        if !wasbits && isdefined(x, i)
#                            xf = getfield(x, i)
#                            id = get!(s.serialized, xf, 0)::Int
#                            if id == 0 || id > xid
#                                needsreloc = true
#                            else
#                                push!(s.relocations, (xid, i, id))
#                            end
#                        end
#                    end
#                    if needsreloc
#                        push!(relocations, x)
#                    end
#                end
#            end
#        end
#
#        if reloc[1] != 0
#            push!(s.relocations, reloc)
#        end
#    end
#
#    # flush all of the pending relocations
#    writetag(s.io, UNDEFREF_TAG)
#    write(s.io, Int(length(s.relocations)))
#    write(s.io, s.relocations)
#    write(s.io, length(relocarrays))
#    for i = 1:length(relocarrays)
#        write(s.io, relocarrayid[i])
#        write(s.io, relocarrays[i])
#    end
#    empty!(s.relocations)
#    nothing
#end
#
#function serialize_tuple(s::SerializationState, t::Tuple)
#    l = length(t)
#    if l == 0
#        writetag(s.io, EMPTYTUPLE_TAG)
#    elseif l <= 255
#        writetag(s.io, TUPLE_TAG)
#        write(s.io, UInt8(l))
#    else
#        writetag(s.io, LONGTUPLE_TAG)
#        write(s.io, Int32(l))
#    end
#    for i = 1:l
#        fastserialize_any(s, t[i])
#    end
#    nothing
#end
#
#function serialize_array(s::SerializationState, a::Array, relocations)
#    elty = eltype(a)
#    writetag(s.io, ARRAY_TAG)
#    if elty !== UInt8
#        fastserialize_any(s, elty)
#    end
#    if ndims(a) != 1
#        fastserialize_any(s, size(a))
#    else
#        fastserialize_any(s, length(a))
#    end
#    if isbits(elty)
#        serialize_array_bits(s.io, a)
#    else
#        isa(s.io, IOBuffer) && Base.ensureroom(s.io, length(a) * (5 * Core.sizeof(UInt) + Core.sizeof(eltype(a))))
#        hasdata = false
#        for i = 1:length(a)
#            if isdefined(a, i)
#                hasdata = true
#                break
#            end
#        end
#        if hasdata
#            push!(relocations, a)
#        end
#    end
#    nothing
#end
#
#function serialize_expr(s::SerializationState, ex::Expr)
#    l = length(ex.args)
#    if l <= 255
#        writetag(s.io, EXPR_TAG)
#        write(s.io, UInt8(l))
#    else
#        writetag(s.io, LONGEXPR_TAG)
#        write(s.io, Int32(l))
#    end
#    fastserialize_any(s, ex.head)
#    fastserialize_any(s, ex.typ)
#    fastserialize_any(s, ex.args)
#    nothing
#end
#

############

type FastDeSerializationState{I<:IO}
    io::I
    deserialized::Vector{Any}
    FastDeSerializationState(io::I) = new(io, Vector{Any}())
end
FastDeSerializationState(io::IO) = FastDeSerializationState{typeof(io)}(io)

deserialize(s::IO) = fastdeserialize_any(FastDeSerializationState(s))

function fastdeserialize_any(s::FastDeSerializationState)
    pos = -1
    while true
        b = Int32(read(s.io, UInt8)::UInt8)
        #Core.println(b, " => ", b == 0 ? "..." : desertag(b))
        if b == UNDEFREF_TAG
            if pos < 0
                pos = length(s.deserialized)
                continue
            else
                break
            end
        end

        if b == 0
            b = Int32(read(s.io, UInt8)::UInt8)
            x = desertag(b)
        elseif b >= VALUE_TAGS
            x = desertag(b)
        elseif b == DATATYPE_TAG
            x = deserialize_datatype(s)
        elseif b == SYMBOL_TAG
            x = Symbol(read(s.io, UInt8, Int(read(s.io, UInt8)::UInt8)))
        elseif b == LONGSYMBOL_TAG
            x = Symbol(read(s.io, UInt8, Int(read(s.io, Int32)::Int32)))
        elseif b == UNION_TAG
            x = deserialize_union(s)
        elseif b == MODULE_TAG
            x = deserialize_module(s)
        elseif b == ARRAY_TAG
            x = deserialize_array(s)
        elseif b == SIMPLEVECTOR_TAG
            l = read(s.io, Int)
            x = ccall(:jl_alloc_svec, Any, (Csize_t,), l)
        else
            if b == VALUE_TAG
                t = s.deserialized[read(s.io, Int)]::DataType
            else
                t = desertag(b)::DataType
            end
            nf = nfields(t)
            if nf == 0 && t.size > 0 # bitstype
                x = read(s.io, t)
            else
                x = ccall(:jl_new_struct_uninit, Any, (Any,), t)
                xptr = pointer_from_objref(x)
                for i = 1:nf
                    if !jl_field_isptr(t, i)
                        fldty = t.types[i]::DataType
                        @readfield(s, x, xptr, t, i, fldty)
                    end
                end
            end
        end
        #Core.println(x)
        push!(s.deserialized, x)
    end
    #Core.println(s.deserialized)
    for i = 1:read(s.io, Int)
        counter = read(s.io, Int) + pos
        field = read(s.io, Int)
        fid = read(s.io, Int)
        obj = s.deserialized[counter]
        fld = s.deserialized[fid > 0 ? fid : pos - fid]
        if isa(obj, SimpleVector)
            ptr = pointer_from_objref(obj) + field * sizeof(Ptr{Void})
            unsafe_store!(ptr, fld)
        else
            ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), obj, field-1, fld)
        end
    end
    for i = 1:read(s.io, Int)
        counter = read(s.io, Int) + pos
        a = s.deserialized[counter]::Array
        aptr = convert(Ptr{Any}, pointer(a))::Ptr{Any}
        nf = length(a)
        d = read!(s.io, Vector{Int}(nf))
        for j = 1:nf
            fid = d[j]
            if fid != 0
                unsafe_store!(aptr, s.deserialized[fid >= 0 ? fid : pos - fid], j)
            end
        end
    end
    #Core.println(s.deserialized)
    retid = read(s.io, Int)
    return s.deserialized[retid >= 0 ? retid : pos - retid]
end

function deserialize_module(s::FastDeSerializationState)
    m = Main
    while true
        id_mname = read(s.io, Int)
        id_mname == 0 && return m
        mname = s.deserialized[id_mname]
        if !isdefined(m, mname)
            warn("Module $mname not defined on process $(myid())")  # an error seemingly fails
        end
        m = getfield(m, mname)::Module
    end
end

function deserialize_datatype(s::FastDeSerializationState)
    id_tname = read(s.io, Int)
    id_mod = read(s.io, Int)
    name = s.deserialized[id_tname]::Symbol
    mod = s.deserialized[id_mod]::Module
    ty = getfield(mod, name)
    if isempty(ty.parameters)
        return ty
    end
    count = read(s.io, Int)
    params = Vector{Any}(count)
    for i = 1:count
        params[i] = s.deserialized[read(s.io, Int)]
    end
    return ty{params...}
end

function deserialize_union(s::FastDeSerializationState)
    count = read(s.io, Int)
    types = Vector{Any}(count)
    for i = 1:count
        types[i] = s.deserialized[i]
    end
    return Union{types...}
end

function deserialize_array(s::FastDeSerializationState)
    elty = s.deserialized[read(s.io, Int)]
    nd = read(s.io, Int)
    if nd == 1
        d1 = read(s.io, Int)
        if elty !== Bool
            a = Array{elty, 1}(d1)
            if isbits(elty)
                read!(s.io, a)
            end
            return a
        end
        dims = (Int(d1),)
    elseif nd == 2
        d1 = read(s.io, Int)
        d2 = read(s.io, Int)
        if elty !== Bool
            a = Array{elty, 2}(d1, d2)
            if isbits(elty)
                read!(s.io, a)
            end
            return a
        end
        dims = (Int(d1), Int(d2))
    else
        dims = ntuple(i -> read(s.io, Int), nd)
    end
    if isbits(elty)
        # deserialize_array_bits
        n = prod(dims)::Int
        if elty === Bool && n > 0
            A = Array{Bool, nd}(dims)
            i = 1
            while i <= n
                b = read(s.io, UInt8)::UInt8
                v = (b >> 7) != 0
                count = b & 0x7f
                nxt = i + count
                while i < nxt
                    A[i] = v
                    i += 1
                end
            end
        else
            A = read(s.io, elty, dims)
        end
    else
        A = Array{elty, nd}(dims)
    end
    return A
end

#
##indent_level = 0
#function fastdeserialize_any(s::FastDeSerializationState)
#    #global indent_level
#    b = Int32(read(s.io, UInt8)::UInt8)
#    if b == UNDEFREF_TAG
#        return s.deserialized[read(s.io, Int)]
#    end
#    first = length(s.deserialized) + 1
#    #indent_level += 1
#    while b != UNDEFREF_TAG
#        push!(s.deserialized, nothing)
#        idx = length(s.deserialized)
#        #println(" " ^ indent_level, idx, ' ', b, ' ', b != 0 ? desertag(b) : 0)
#        if b == 0
#            b = Int32(read(s.io, UInt8)::UInt8)
#            x = desertag(b)
#        elseif b >= VALUE_TAGS
#            x = desertag(b)
#        #elseif b == TUPLE_TAG
#        #    x = deserialize_tuple(s, Int(read(s.io, UInt8)::UInt8))
#        #elseif b == LONGTUPLE_TAG
#        #    x = deserialize_tuple(s, Int(read(s.io, Int32)::Int32))
#        elseif b == ARRAY_TAG
#            x = deserialize_array(s)
#        elseif b == DATATYPE_TAG
#            x = deserialize_datatype(s)
#        elseif b == SYMBOL_TAG
#            x = symbol(read(s.io, UInt8, Int(read(s.io, UInt8)::UInt8)))
#        elseif b == LONGSYMBOL_TAG
#            x = symbol(read(s.io, UInt8, Int(read(s.io, Int32)::Int32)))
#        elseif b == EXPR_TAG
#            x = deserialize_expr(s, Int(read(s.io, UInt8)::UInt8))
#        elseif b == LONGEXPR_TAG
#            x = deserialize_expr(s, Int(read(s.io, Int32)::Int32))
#        elseif b == SIMPLEVECTOR_TAG
#            x = deserialize_sv(s)
#        elseif b == UNION_TAG
#            x = deserialize_union(s)
#        elseif b == MODULE_TAG
#            x = deserialize_module(s)
#        else
#            if b == VALUE_TAG
#                t = fastdeserialize_any(s)::DataType
#            else
#                t = desertag(b)::DataType
#            end
#            nf = nfields(t)
#            if nf == 0 && t.size > 0 # bitstype
#                x = read(s.io, t)
#            else
#                x = ccall(:jl_new_struct_uninit, Any, (Any,), t)
#                xptr = pointer_from_objref(x)
#                for i = 1:nf
#                    fldty = t.types[i]
#                    if typeof(fldty) === DataType
#                        fldty = fldty::DataType
#                        if isbits(fldty::DataType)
#                            fldsz = fldty.size
#                            if fldsz == 1
#                                writefield(UInt8, s, xptr + Base.field_offset(t, i))
#                            elseif fldsz == 2
#                                writefield(UInt16, s, xptr + Base.field_offset(t, i))
#                            elseif fldsz == 4
#                                writefield(UInt32, s, xptr + Base.field_offset(t, i))
#                            elseif fldsz == 8
#                                writefield(UInt64, s, xptr + Base.field_offset(t, i))
#                            else
#                                writefield(fldty, s, xptr + Base.field_offset(t, i))
#                            end
#                            #ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), x, i-1, read(s.io, fldty))
#                        end
#                    end
#                end
#            end
#        end
#        s.deserialized[idx] = x
#        #println(idx, ' ', x)
#        b = Int32(read(s.io, UInt8)::UInt8)
#    end
#    #println(s.deserialized)
#    for i = 1:read(s.io, Int)
#        counter = read(s.io, Int)
#        field = read(s.io, Int)
#        id = read(s.io, Int)
#        ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), s.deserialized[counter], field-1, s.deserialized[id])
#    end
#    for i = 1:read(s.io, Int)
#        counter = read(s.io, Int)
#        a = s.deserialized[counter]::Array
#        nf = length(a)
#        d = read!(s.io, Vector{Int}(nf))
#        for j = 1:nf
#            id = d[j]
#            if id != 0
#                arrayset(a, s.deserialized[id], j)
#            end
#        end
#    end
#    #indent_level -= 1
#    return s.deserialized[first]
#end
#
#deserialize_tuple(s::SerializationState, len) = ntuple(i->deserialize(s), len)

#function writefield{T}(::Type{T}, s::FastDeSerializationState, fptr)
#    unsafe_store!(convert(Ptr{T}, fptr), read(s.io, T))
#    nothing
#end
#
#function deserialize_sv(s::FastDeSerializationState)
#    return svec((fastdeserialize_any(s)::Vector{Any})...)
#end
#
#function deserialize_array(s::FastDeSerializationState)
#    d1 = fastdeserialize_any(s)
#    if isa(d1, Type)
#        elty = d1
#        d1 = fastdeserialize_any(s)
#    else
#        elty = UInt8
#    end
#    if isa(d1, Integer)
#        if elty !== Bool && isbits(elty)
#            return read!(s.io, Array(elty, d1))::Array
#        end
#        dims = (Int(d1),)
#    else
#        dims = convert(Dims, d1)::Dims
#    end
#    if isbits(elty)
#        # deserialize_array_bits
#        n = prod(dims)::Int
#        if elty === Bool && n>0
#            A = Array(Bool, dims)
#            i = 1
#            while i <= n
#                b = read(s.io, UInt8)::UInt8
#                v = (b>>7) != 0
#                count = b&0x7f
#                nxt = i+count
#                while i < nxt
#                    A[i] = v; i+=1
#                end
#            end
#        else
#            A = read(s.io, elty, dims)
#        end
#    else
#        A = Array(elty, dims)
#    end
#    return A
#end
#
#
#function deserialize_expr(s::FastDeSerializationState, len)
#    hd = fastdeserialize_any(s)::Symbol
#    ty = fastdeserialize_any(s)
#    e = Expr(hd)
#    e.args = fastdeserialize_any(s)
#    e.typ = ty
#    e
#end
#

## MUCH OLDER CRAP ##########################

#serialize(s::SerializationState, p::Ptr) = serialize_any(s, oftype(p, C_NULL))

#function serialize{T,N,A<:Array}(s::SerializationState, a::SubArray{T,N,A})
#    if !isbits(T) || stride(a,1)!=1
#        return serialize(s, copy(a))
#    end
#    writetag(s.io, ARRAY_TAG)
#    serialize(s, T)
#    serialize(s, size(a))
#    serialize_array_data(s.io, a)
#end

#function serialize{T<:AbstractString}(s::SerializationState, ss::SubString{T})
#    # avoid saving a copy of the parent string, keeping the type of ss
#    serialize_any(s, convert(SubString{T}, convert(T,ss)))
#end

## Don't serialize the pointers
#function serialize(s::SerializationState, r::Regex)
#    serialize_type(s, typeof(r))
#    serialize(s, r.pattern)
#    serialize(s, r.compile_options)
#    serialize(s, r.match_options)
#end
#
#function serialize(s::SerializationState, n::BigInt)
#    serialize_type(s, BigInt)
#    serialize(s, base(62,n))
#end
#
#function serialize(s::SerializationState, n::BigFloat)
#    serialize_type(s, BigFloat)
#    serialize(s, string(n))
#end
#
#function serialize(s::SerializationState, t::Dict)
#    serialize_cycle(s, t) && return
#    serialize_type(s, typeof(t))
#    write(s.io, Int32(length(t)))
#    for (k,v) in t
#        serialize(s, k)
#        serialize(s, v)
#    end
#end
##
#function serialize(s::SerializationState, f::Function)
#    name = false
#    if isgeneric(f)
#        name = f.env.name
#    elseif isa(f.env,Symbol)
#        name = f.env
#    end
#    if isa(name,Symbol)
#        if isdefined(Base,name) && is(f,getfield(Base,name))
#            writetag(s.io, FUNCTION_TAG)
#            write(s.io, UInt8(0))
#            serialize(s, name)
#            return
#        end
#        mod = ()
#        if isa(f.env,Symbol)
#            mod = Core
#        elseif isdefined(f.env, :module) && isa(f.env.module, Module)
#            mod = f.env.module
#        elseif !is(f.env.defs, ())
#            mod = f.env.defs.func.code.module
#        end
#        if mod !== ()
#            if isdefined(mod,name) && is(f,getfield(mod,name))
#                # toplevel named func
#                writetag(s.io, FUNCTION_TAG)
#                write(s.io, UInt8(2))
#                serialize(s, mod)
#                serialize(s, name)
#                return
#            end
#        end
#        serialize_cycle(s, f) && return
#        writetag(s.io, FUNCTION_TAG)
#        write(s.io, UInt8(3))
#        serialize(s, f.env)
#    else
#        serialize_cycle(s, f) && return
#        writetag(s.io, FUNCTION_TAG)
#        write(s.io, UInt8(1))
#        linfo = f.code
#        @assert isa(linfo,LambdaStaticData)
#        serialize(s, linfo)
#        serialize(s, f.env)
#    end
#end
#
#const lambda_numbers = WeakKeyDict()
#lnumber_salt = 0
#function lambda_number(l::LambdaStaticData)
#    global lnumber_salt, lambda_numbers
#    if haskey(lambda_numbers, l)
#        return lambda_numbers[l]
#    end
#    # a hash function that always gives the same number to the same
#    # object on the same machine, and is unique over all machines.
#    ln = lnumber_salt+(UInt64(myid())<<44)
#    lnumber_salt += 1
#    lambda_numbers[l] = ln
#    return ln
#end
#
#function serialize(s::SerializationState, linfo::LambdaStaticData)
#    serialize_cycle(s, linfo) && return
#    writetag(s.io, LAMBDASTATICDATA_TAG)
#    serialize(s, lambda_number(linfo))
#    serialize(s, uncompressed_ast(linfo))
#    if isdefined(linfo.def, :roots)
#        serialize(s, linfo.def.roots::Vector{Any})
#    else
#        serialize(s, Any[])
#    end
#    serialize(s, linfo.sparams)
#    serialize(s, linfo.inferred)
#    serialize(s, linfo.module)
#    if isdefined(linfo, :capt)
#        serialize(s, linfo.capt)
#    else
#        serialize(s, nothing)
#    end
#end
#
#function serialize(s::SerializationState, t::Task)
#    serialize_cycle(s, t) && return
#    if istaskstarted(t) && !istaskdone(t)
#        error("cannot serialize a running Task")
#    end
#    state = [t.code,
#        t.storage,
#        t.state == :queued || t.state == :runnable ? (:runnable) : t.state,
#        t.result,
#        t.exception]
#    writetag(s.io, TASK_TAG)
#    for fld in state
#        serialize(s, fld)
#    end
#end

## deserializing values ##

#const known_lambda_data = Dict()
#
#function deserialize(s::SerializationState, ::Type{Function})
#    b = read(s.io, UInt8)::UInt8
#    if b==0
#        name = deserialize(s)::Symbol
#        if !isdefined(Base,name)
#            f = (args...)->error("function $name not defined on process $(myid())")
#        else
#            f = getfield(Base,name)::Function
#        end
#    elseif b==2
#        mod = deserialize(s)::Module
#        name = deserialize(s)::Symbol
#        if !isdefined(mod,name)
#            f = (args...)->error("function $name not defined on process $(myid())")
#        else
#            f = getfield(mod,name)::Function
#        end
#    elseif b==3
#        f = ccall(:jl_new_gf_internal, Any, (Any,), nothing)::Function
#        deserialize_cycle(s, f)
#        f.env = deserialize(s)
#    else
#        f = ccall(:jl_new_closure, Any, (Ptr{Void}, Ptr{Void}, Any),
#                  cglobal(:jl_trampoline), C_NULL, nothing)::Function
#        deserialize_cycle(s, f)
#        f.code = li = deserialize(s)
#        f.fptr = ccall(:jl_linfo_fptr, Ptr{Void}, (Any,), li)
#        f.env = deserialize(s)
#    end
#
#    return f
#end
#
#function deserialize(s::SerializationState, ::Type{LambdaStaticData})
#    lnumber = deserialize(s)
#    if haskey(known_lambda_data, lnumber)
#        linfo = known_lambda_data[lnumber]::LambdaStaticData
#        makenew = false
#    else
#        linfo = ccall(:jl_new_lambda_info, Any, (Ptr{Void}, Ptr{Void}, Ptr{Void}), C_NULL, C_NULL, C_NULL)::LambdaStaticData
#        makenew = true
#    end
#    deserialize_cycle(s, linfo)
#    ast = deserialize(s)::Expr
#    roots = deserialize(s)::Vector{Any}
#    sparams = deserialize(s)::SimpleVector
#    infr = deserialize(s)::Bool
#    mod = deserialize(s)::Module
#    capt = deserialize(s)
#    if makenew
#        linfo.ast = ast
#        linfo.sparams = sparams
#        linfo.inferred = infr
#        linfo.module = mod
#        linfo.roots = roots
#        if !is(capt,nothing)
#            linfo.capt = capt::Vector{Any}
#        end
#        known_lambda_data[lnumber] = linfo
#    end
#    return linfo
#end

#function deserialize(s::SerializationState, ::Type{Task})
#    t = Task(()->nothing)
#    deserialize_cycle(s, t)
#    t.code = deserialize(s)
#    t.storage = deserialize(s)
#    t.state = deserialize(s)
#    t.result = deserialize(s)
#    t.exception = deserialize(s)
#    t
#end

#function deserialize{K,V}(s::SerializationState, T::Type{Dict{K,V}})
#    n = read(s.io, Int32)
#    t = T(); sizehint!(t, n)
#    deserialize_cycle(s, t)
#    for i = 1:n
#        k = deserialize(s)
#        v = deserialize(s)
#        t[k] = v
#    end
#    return t
#end

#deserialize(s::SerializationState, ::Type{BigFloat}) = parse(BigFloat, deserialize(s))
#
#deserialize(s::SerializationState, ::Type{BigInt}) = get(GMP.tryparse_internal(BigInt, deserialize(s), 62, true))
#
#deserialize(s::SerializationState, ::Type{BigInt}) = get(GMP.tryparse_internal(BigInt, deserialize(s), 62, true))
#
#function deserialize(s::SerializationState, t::Type{Regex})
#    pattern = deserialize(s)
#    compile_options = deserialize(s)
#    match_options = deserialize(s)
#    Regex(pattern, compile_options, match_options)
#end

end

max_stars_repo_name: vtjnash/OctoRelocator

max_issues_repo_name: vtjnash/OctoRelocator

max_forks_repo_name: vtjnash/OctoRelocator


text:
import Base: ==, hash

# Define struct not yet exported in POMDPs.jl framework
struct AlphaVec
    alpha::Vector{Float64} # alpha vector
    action::Any # action associated wtih alpha vector
end

AlphaVec() = AlphaVec([0.0], 0)

# define alpha vector equality
Base.hash(a::AlphaVec, h::UInt) = hash(a.alpha, hash(a.action, h))
==(a::AlphaVec, b::AlphaVec) = (a.alpha,a.action) == (b.alpha, b.action)

import BeliefUpdaters: pdf
pdf(b::DiscreteBelief, s) = b.b[typeof(HorizonLength(b.pomdp)) == InfiniteHorizon ?
                                stateindex(b.pomdp, s) : stage_stateindex(b.pomdp, s)]

import BeliefUpdaters: initialize_belief
function initialize_belief(bu::DiscreteUpdater, dist::FiniteHorizonPOMDPs.InStageDistribution)
    state_list = ordered_stage_states(bu.pomdp, stage(dist))
    ns = length(state_list)
    b = zeros(ns)
    belief = DiscreteBelief(bu.pomdp, state_list, b)
    for s in support(dist)
        sidx = stage_stateindex(bu.pomdp, s)
        belief.b[sidx] = pdf(dist, s)
    end
    return belief
end

import BeliefUpdaters: update
function update(bu::DiscreteUpdater, b::DiscreteBelief, a, o)
    pomdp = bu.pomdp
    state_space = b.state_list
    bp = zeros(length(state_space))

    for (si, s) in enumerate(state_space)
        si = stage_stateindex(pomdp, s)

        if pdf(b, s) > 0.0
            td = transition(pomdp, s, a)
            for (sp, tp) in weighted_iterator(td)
                spi = stage_stateindex(pomdp, sp)
                op = obs_weight(pomdp, s, a, sp, o) # shortcut for observation probability from POMDPModelTools
                op2 = pdf(observation(pomdp, a, sp), o)

                bp[spi] += op * tp * b.b[si]
            end
        end
    end

    bp_sum = sum(bp)

    if bp_sum == 0.0
        error("""
              Failed discrete belief update: new probabilities sum to zero.

              b = $b
              a = $a
              o = $o

              Failed discrete belief update: new probabilities sum to zero.
              """)
    end

    # Normalize
    bp ./= bp_sum

    return DiscreteBelief(pomdp, ordered_stage_states(pomdp, stage(pomdp, o)+1), bp)
end

methods(update)

#test
let
    hor = 5
    pomdp = fixhorizon(TigerPOMDP(), hor)

    bu = DiscreteUpdater(pomdp.m)
    b0 = initialize_belief(bu, initialstate_distribution(pomdp.m))

    Γs = []
    # I only know correct alpha vectors from horizon stage
    for i in 1:4
        push!(Γs, [AlphaVec([0., 0.], 0), AlphaVec([0., 0.], 0), AlphaVec([0., 0.], 0)])
    end
    push!(Γs, [AlphaVec([10., 0.], 1), AlphaVec([0., 10.], 2), AlphaVec([-0.5, -0.5], 0)])

    staged_policies = [AlphaVectorPolicy(pomdp.m, length(stage_states(pomdp, t)), [a.alpha for a in Γs[t]],
        convert(Vector{actiontype(pomdp)}, [a.action for a in Γs[t]])) for t in 1:horizon(pomdp)]

    policy = StagedPolicy(pomdp, staged_policies)

    # these values were gotten from FiVI.jl tests
    @test Set(alphapairs(policy, hor)) == Set([[10., 0.]=>1, [0., 10.]=>2, [-0.5, -0.5]=>0])
    @test Set(alphavectors(policy, hor)) == Set([[10., 0.], [0., 10.], [-0.5, -0.5]])

    # initial belief is 50% tiger position confidence
    @test isapprox(value(policy, b0, hor), hor)
    @test isapprox(value(policy, [0.5, 0.5], hor), 5)
    # I am not sure whether the result should be [-0.5, 5. 5.] or [-1., 5. 5.]
    # may check the correctness in the future
    @test_broken isapprox(actionvalues(policy, b0, hor), [-1, 5., 5.])
    @test length(actionvalues(policy, b0, hor)) == length(actions(pomdp))

    # The agent only has one decision, so he has to risk choosing one door
    @test action(policy, b0, hor) == 1 || action(policy, b0, hor) == 2

    # try pushing new vector
    push!(policy, [0.0,0.0], 0, hor)
    @test isapprox(length(policy.staged_policies[hor].alphas), 4)

    # test simulation
    up = updater(policy)
    @test isapprox(simulate(RolloutSimulator(), pomdp, policy, up), 4.435, rtol=.1)

    # test infinite horizon pomdp
    pomdp = BabyPOMDP()
    @test_throws ArgumentError StagedPolicy(pomdp, staged_policies)
end

ext: jl


text:
using Pkg

Pkg.add("NPZ")
Pkg.add("Profile")
Pkg.add("BenchmarkTools")
Pkg.add("Interpolations")
Pkg.add("LinearAlgebra")
Pkg.add("Distributed")
Pkg.add("Statistics")
Pkg.add("DifferentialEquations")
Pkg.add("OrdinaryDiffEq")
Pkg.add("Plots")

max_stars_repo_name: dtoconnor/Annealing-Samplers

max_issues_repo_name: dtoconnor/Annealing-Samplers

max_forks_repo_name: dtoconnor/Annealing-Samplers


text:
using NCDatasets
if VERSION >= v"0.7"
    using Test
else
    using Base.Test
end

filename = tempname()

A = [123. 2.; 45. 23.]
dt = [DateTime(2001,1,1), DateTime(2002,1,1)]

NCDatasets.ncsave(filename,Dict("A"  => A), "dt" => dt)

data = NCDatasets.ncload(filename)

@test data["A"] == A
@test data["dt"] == dt


text:
function merge_pools(pools, names...)
    (names[1] == [] || MLLabelUtils.islabelenc(collect(names), SVDD.learning_pool_enc)) || throw(ArgumentError("$(collect(names)) is not a valid label encoding."))
    return reduce((r, key) -> vcat(r, haskey(pools, key) ? pools[key] : Int64[]), unique(names); init=Int64[])
end

classify(x::Number; opt_precision = OPT_PRECISION) = x > opt_precision ? :outlier : :inlier

function classify(predictions::Vector{Vector{Float64}}, scope::Scope; opt_precision = OPT_PRECISION)
    if isa(scope, Val{:Subspace})
        return map(x -> classify.(x; opt_precision = opt_precision), predictions)
    else
        is_global_outlier = mapreduce(x -> x .> opt_precision, (a,b) -> a .| b, predictions)
        return ifelse.(is_global_outlier, :outlier, :inlier)
    end
end

function adjust_kernel_matrix(K::Array{T, 2}; tolerance = 1e-15, warn_threshold = 1e-8) where T <: Real
    info(LOGGER, "Adjusting Kernel Matrix.")
    F = eigen(K)
    eltype(F.values) <: Complex && throw(ArgumentError("Matrix K has complex eigenvalues."))
    F.values[F.values .< tolerance] .= 0.0
    K_adjusted::Array{Float64, 2} = F.vectors * diagm(0 => F.values) * inv(F.vectors)
    K_diff = abs.(K_adjusted - K)
    sum_adjustment = sum(K_diff)
    max_adjustment = maximum(K_diff)
    info(LOGGER, "[ADJUST KERNEL] Maximum adjustemt of kernel matrix entry is $max_adjustment. The sum of adjustments is $(sum_adjustment)")
    max_adjustment .> warn_threshold && warn(LOGGER, "[ADJUST KERNEL] Maximum adjustment of kernel matrix entry exceeded the threshold of $(warn_threshold)!.")
    return K_adjusted
end

function adjust_kernel_matrix(K::Vector{Array{T, 2}}; tolerance = 1e-15, warn_threshold = 1e-8) where T <: Real
    adjust_kernel_matrix.(K)
end

function min_max_normalize(x)
   min, max = extrema(x)
   (x .- min) ./ (max .- min)
end


text:
"""
    AR_gmm_obj(gi:Function)

Returns a function `ar(θ)` where:
``ar(θ) = n [1/n \\sum g_i(θ)]' \\widehat{Var}(g_i(θ))^{-1}[1/n \\sum g_i(θ)] ``
"""
function AR_gmm_obj(gi::Function)
  function(θ)
    g = gi(θ)
    m = mean(g,dims=1)
    W = inv(cov(g))
    (size(g)[1]*( m*W*m')[1])
  end
end


text:
abstract type AbstractFunJac{J2} end
mutable struct FunJac{F, F2, J, P, M, J2, uType, uType2, Prec, PS} <: AbstractFunJac{J2}
    fun::F
    fun2::F2
    jac::J
    p::P
    mass_matrix::M
    jac_prototype::J2
    prec::Prec
    psetup::PS
    u::uType
    du::uType
    resid::uType2
end
FunJac(fun, jac, p, m, jac_prototype, prec, psetup, u, du) =
    FunJac(fun, nothing, jac, p, m, jac_prototype, prec, psetup, u, du, nothing)
FunJac(fun, jac, p, m, jac_prototype, prec, psetup, u, du, resid) =
    FunJac(fun, nothing, jac, p, m, jac_prototype, prec, psetup, u, du, resid)

function cvodefunjac(t::Float64, u::N_Vector, du::N_Vector, funjac::FunJac)
    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    funjac.du =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(du), length(funjac.du))
    _du = funjac.du
    _u = funjac.u
    funjac.fun(_du, _u, funjac.p, t)
    return CV_SUCCESS
end

function cvodefunjac2(t::Float64, u::N_Vector, du::N_Vector, funjac::FunJac)
    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    funjac.du =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(du), length(funjac.du))
    _du = funjac.du
    _u = funjac.u
    funjac.fun2(_du, _u, funjac.p, t)
    return CV_SUCCESS
end

function cvodejac(
    t::realtype,
    u::N_Vector,
    du::N_Vector,
    J::SUNMatrix,
    funjac::AbstractFunJac{Nothing},
    tmp1::N_Vector,
    tmp2::N_Vector,
    tmp3::N_Vector,
)

    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    _u = funjac.u
    funjac.jac(convert(Matrix, J), _u, funjac.p, t)
    return CV_SUCCESS
end

function cvodejac(
    t::realtype,
    u::N_Vector,
    du::N_Vector,
    _J::SUNMatrix,
    funjac::AbstractFunJac{<:SparseArrays.SparseMatrixCSC},
    tmp1::N_Vector,
    tmp2::N_Vector,
    tmp3::N_Vector,
)
    jac_prototype = funjac.jac_prototype

    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    _u = funjac.u

    funjac.jac(jac_prototype, _u, funjac.p, t)

    copyto!(_J, jac_prototype)

    return CV_SUCCESS
end

function idasolfun(t::Float64, u::N_Vector, du::N_Vector, resid::N_Vector, funjac::FunJac)
    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    _u = funjac.u
    funjac.du =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(du), length(funjac.du))
    _du = funjac.du
    funjac.resid =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(resid), length(funjac.resid))
    _resid = funjac.resid
    funjac.fun(_resid, _du, _u, funjac.p, t)
    return IDA_SUCCESS
end

function idajac(
    t::realtype,
    cj::realtype,
    u::N_Vector,
    du::N_Vector,
    res::N_Vector,
    J::SUNMatrix,
    funjac::AbstractFunJac{Nothing},
    tmp1::N_Vector,
    tmp2::N_Vector,
    tmp3::N_Vector,
)

    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    _u = funjac.u
    funjac.du =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(du), length(funjac.du))
    _du = funjac.du

    funjac.jac(convert(Matrix, J), _du, _u, funjac.p, cj, t)
    return IDA_SUCCESS
end

function idajac(
    t::realtype,
    cj::realtype,
    u::N_Vector,
    du::N_Vector,
    res::N_Vector,
    _J::SUNMatrix,
    funjac::AbstractFunJac{<:SparseArrays.SparseMatrixCSC},
    tmp1::N_Vector,
    tmp2::N_Vector,
    tmp3::N_Vector,
)

    jac_prototype = funjac.jac_prototype

    funjac.u = unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(u), length(funjac.u))
    _u = funjac.u
    funjac.du =
        unsafe_wrap(Vector{Float64}, N_VGetArrayPointer_Serial(du), length(funjac.du))
    _du = funjac.du

    funjac.jac(jac_prototype, _du, _u, funjac.p, cj, t)

    copyto!(_J, jac_prototype)

    return IDA_SUCCESS
end

function massmat(
    t::Float64,
    _M::SUNMatrix,
    mmf::AbstractFunJac,
    tmp1::N_Vector,
    tmp2::N_Vector,
    tmp3::N_Vector,
)
    if typeof(mmf.mass_matrix) <: Array
        M = convert(Matrix, _M)
        M .= mmf.mass_matrix
    else
        copyto!(_M, mmf.mass_matrix)
    end

    return IDA_SUCCESS
end

function jactimes(
    v::N_Vector,
    Jv::N_Vector,
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    fj::AbstractFunJac,
    tmp::N_Vector,
)
    DiffEqBase.update_coefficients!(fj.jac_prototype, y, fj.p, t)
    LinearAlgebra.mul!(convert(Vector, Jv), fj.jac_prototype, convert(Vector, v))
    return CV_SUCCESS
end

function idajactimes(
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    r::N_Vector,
    v::N_Vector,
    Jv::N_Vector,
    cj::Float64,
    fj::AbstractFunJac,
    tmp1::N_Vector,
    tmp2::N_Vector,
)
    DiffEqBase.update_coefficients!(fj.jac_prototype, y, fj.p, t)
    LinearAlgebra.mul!(convert(Vector, Jv), fj.jac_prototype, convert(Vector, v))
    return IDA_SUCCESS
end

function precsolve(
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    r::N_Vector,
    z::N_Vector,
    gamma::Float64,
    delta::Float64,
    lr::Int,
    fj::AbstractFunJac,
)
    fj.prec(
        convert(Vector, z),
        convert(Vector, r),
        fj.p,
        t,
        convert(Vector, y),
        convert(Vector, fy),
        gamma,
        delta,
        lr,
    )
    return CV_SUCCESS
end

function precsetup(
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    jok::Int,
    jcurPtr::Ref{Int},
    gamma::Float64,
    fj::AbstractFunJac,
)
    fj.psetup(
        fj.p,
        t,
        convert(Vector, y),
        convert(Vector, fy),
        jok == 1,
        Base.unsafe_wrap(Vector{Int}, jcurPtr, 1),
        gamma,
    )
    return CV_SUCCESS
end

function idaprecsolve(
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    resid::N_Vector,
    r::N_Vector,
    z::N_Vector,
    gamma::Float64,
    delta::Float64,
    fj::AbstractFunJac,
)
    fj.prec(
        convert(Vector, z),
        convert(Vector, r),
        fj.p,
        t,
        convert(Vector, y),
        convert(Vector, fy),
        convert(Vector, resid),
        gamma,
        delta,
    )
    return IDA_SUCCESS
end

function idaprecsetup(
    t::Float64,
    y::N_Vector,
    fy::N_Vector,
    rr::N_Vector,
    gamma::Float64,
    fj::AbstractFunJac,
)
    fj.psetup(fj.p, t, convert(Vector, rr), convert(Vector, y), convert(Vector, fy), gamma)
    return IDA_SUCCESS
end


text:
module Day5

TESTINPUT = """0,9 -> 5,9
8,0 -> 0,8
9,4 -> 3,4
2,2 -> 2,1
7,0 -> 7,4
6,4 -> 2,0
0,9 -> 2,9
3,4 -> 1,4
0,0 -> 8,8
5,5 -> 8,2"""

struct Coord
    x::Int64
    y::Int64
end

Base.:+(a::Coord, b::Coord) = Coord(a.x + b.x, a.y + b.y)
Base.:-(a::Coord, b::Coord) = Coord(a.x - b.x, a.y - b.y)

Coord(s) = split(s, ",") |>
    x -> parse.(Int64, x) |>
    x -> Coord(x...)

struct Line
    from::Coord
    to::Coord
end

is_vertical_or_horizontal(l::Line) = l.from.x == l.to.x || l.from.y == l.to.y
gradient(l::Line) = (l.to.y - l.from.y) // (l.to.x - l.from.x)
x_direction(l::Line) = sign(l.to.x - l.from.x)
is_coord_on_line(c::Coord, l::Line) = (l.to.x - l.from.x) * (c.y - l.from.y) == (l.to.y - l.from.y) * (c.x - l.from.x)
function step(l::Line)
    m = gradient(l)
    if m.num == 0
        Coord(sign(l.to.x - l.from.x), 0)
    elseif m.den == 0
        Coord(0, sign(l.to.y - l.from.y))
    else
        Coord(x_direction(l) * m.den, x_direction(l) * m.num)
    end
end

function num_steps(l::Line)
    s = step(l)
    if s.x == 0
        Int64((l.to.y - l.from.y) / s.y) + 1
    else
        Int64((l.to.x - l.from.x) / s.x) + 1
    end
end

Base.iterate(l::Line) = (l.from, l.from + step(l))
Base.iterate(l::Line, c::Coord) = c == (l.to + step(l)) ? nothing : (c, c + step(l))
Base.IteratorSize(::Type{Line}) = Base.HasLength()
Base.IteratorEltype(::Type{Line}) = Base.HasEllype()
Base.eltype(l::Line) = Coord
Base.length(l::Line) = num_steps(l)

min_x(l::Line) = min(l.from.x, l.to.x)
max_x(l::Line) = max(l.from.x, l.to.x)
min_y(l::Line) = min(l.from.y, l.to.y)
max_y(l::Line) = max(l.from.y, l.to.y)

BBox = Tuple{Int64, Int64, Int64, Int64}
bbox(l::Line) = (min_x(l), min_y(l), max_x(l), max_y(l))
bigger_bbox(left::BBox, right::BBox) = (min(left[1], right[1]), min(left[2], right[2]), max(left[3], right[3]), max(left[4], right[4]))

Line(s) = split(s, " -> ") |>
    x -> Coord.(x) |>
    x -> Line(x...)

mutable struct Diagram
    x::Array{Int64, 2}
    top_left::Coord
    width::Int64
    height::Int64
end

diagram_size(ls::Vector{Line}) = mapreduce(bbox, bigger_bbox, ls)

function Diagram(ls::Vector{Line})
    box = diagram_size(ls)
    top_left = Coord(box[1], box[2])
    width = box[3] - box[1] + 1
    height = box[4] - box[2] + 1
    x = zeros(Int64, height, width)
    Diagram(x, top_left, width, height)
end

function mark!(d::Diagram, c::Coord)
    col = c.x - d.top_left.x + 1
    row = c.y - d.top_left.y + 1
    d.x[row, col] += 1
    d
end

function mark!(d::Diagram, l::Line)
    for c in l
        mark!(d, c)
    end
    d
end

function mark!(d::Diagram, lines::Vector{Line})
    for line in lines
        mark!(d, line)
    end
    d
end

marked_diagram(lines::Vector{Line}) = Diagram(lines) |> d -> mark!(d, lines)

count_at_least(d::Diagram, x::Int64) = sum(d.x .>= x)

run_part1(data_dir::String) = joinpath(data_dir, "day5.txt") |>
    readlines |>
    ls -> Line.(ls) |>
    ls -> filter(is_vertical_or_horizontal, ls) |>
    marked_diagram |>
    d -> count_at_least(d, 2)

run_part2(data_dir::String) = joinpath(data_dir, "day5.txt") |>
    readlines |>
    ls -> Line.(ls) |>
    marked_diagram |>
    d -> count_at_least(d, 2)

end


text:
module Network

include("../controller/requests.jl")
include("../types.jl")
include("../utils.jl")

using .Types, .Utils, Sockets

function init_lobby!(storage)
    arr = Requests.get_lobby(storage.sql_conn)

    if (length(arr) == 0)
        return
    end

    arr = map(x -> string(x), arr)
    storage.active_channels[arr[1]] = Types.Channel(arr[1], arr[2], arr[3],
        parse(Int64, arr[4]), arr[5], arr[6])
end

function is_logged(storage, ip_addr::String)
    client = storage.active_clients[ip_addr]
    client.is_auth
end

function is_connected(storage, ip_addr::String)
    ip_addr in [key for (key, _) in storage.active_clients]
end

function disconnect!(storage, conn::IO)
    for (key, value) in storage.active_clients
        if (isopen(value.conn) == false)
            delete!(storage.active_clients, key)
        end
    end
end

function broadcast_channel(storage, msg::String, conn::IO)
    ip_addr = string(first(getpeername(conn)))

    if (is_logged(storage, ip_addr) == false)
        return
    end

    client = storage.active_clients[ip_addr]
    channel_id = client.current_channel_id

    for (_, value) in storage.active_clients
        if (value.current_channel_id == channel_id && value.id != client.id)

            user = storage.active_clients[ip_addr]
            channel_name = storage.active_channels[channel_id].name
            PS1 = "[$(date(12:19))][$channel_name][$(user.name)] -> "

            if (isopen(value.conn))
                write(value.conn, "\n" * PS1 * msg * "\n")
                fancy_write(storage, value.conn, "")
            end
        end
    end
end

function channel_exist(storage, id::String)
    id in [key for (key, _) in storage.active_channels]
end

function fancy_write(storage, conn::IO, msg::String = "")
    ip_addr = string(first(getpeername(conn)))

    if (is_logged(storage, ip_addr) == false)
        return (write(conn, "> "))
    end
    client = storage.active_clients[ip_addr]
    username = client.name
    channel_id = client.current_channel_id
    channel_name = storage.active_channels[channel_id].name
    PS1 = "[$(date(12:19))][$channel_name][$username] "
    write(conn, PS1 * msg)
end

end # Network


text:
using FourierTools, Documenter

 # set seed fixed for documentation
DocMeta.setdocmeta!(FourierTools, :DocTestSetup, :(using FourierTools); recursive=true)
makedocs(modules = [FourierTools],
         sitename = "FourierTools.jl",
         pages = Any[
            "FourierTools.jl" => "index.md",
            "FFT Helpers" => "helpers.md",
            "FFT Based Convolutions" => "convolutions.md",
            "Resampling (sinc Interpolation)" => "resampling.md",
            "Shifting with FFTs" => "shifting.md",
            "Image Shearing with FFTs" => "shear.md",
            "Image Rotation with FFTs" => "rotate.md",
            "Utility Functions" => "utils.md",
         ]
        )

deploydocs(repo = "github.com/bionanoimaging/FourierTools.jl.git", devbranch="main")


text:
"""
This function creates the MarketClearingProblem struct for endogenous Economic Dispatch.
"""
function create_economic_dispatch_problem(simulation::AgentSimulation,
                                          market_names::Vector{Symbol},
                                          num_invperiods::Int64,
                                          load_growth::AxisArrays.AxisArray{Float64, 1},
                                          existing_projects::Vector{<: Project{<: BuildPhase}},
                                          iteration_year::Int64)

    simulation_dir = get_data_dir(get_case(simulation))
    zones = get_zones(simulation)
    lines = get_lines(simulation)

    reserve_up_market_bool = false
    reserve_down_market_bool = false
    capacity_market_bool = false
    rec_market_bool = false

    #Find if the following markets are being simulated:
    if in(:ReserveUp, market_names)
        reserve_up_market_bool = true
    end

    if in(:ReserveDown, market_names)
        reserve_down_market_bool = true
    end

    if in(:Capacity, market_names)
        capacity_market_bool = true
    end

    if in(:REC, market_names)
        rec_market_bool = true
    end

    # Gather markets data-------------------------------------------------------------------------------
    load_data = read_data(joinpath(simulation_dir, "timeseries_data_files", "Load", "load_$(iteration_year - 1).csv"))
    reserve_up_demand_data = read_data(joinpath(simulation_dir,  "timeseries_data_files", "Reserves", "reserve_up_$(iteration_year - 1).csv"))
    reserve_down_demand_data = read_data(joinpath(simulation_dir,  "timeseries_data_files", "Reserves", "reserve_down_$(iteration_year - 1).csv"))

    num_hours = DataFrames.nrow(load_data)

    energy_mkt_params = read_data(joinpath(simulation_dir, "markets_data", "energy_mkt_param.csv"))
    price_cap_energy = AxisArrays.AxisArray(energy_mkt_params.price_cap * 1.0, zones)

    reserve_up_mkt_params = read_data(joinpath(simulation_dir, "markets_data", "reserve_up_mkt_param.csv"))
    reserve_up_num_points = AxisArrays.AxisArray(zeros(length(zones)), zones)
    reserve_up_break_points = AxisArrays.AxisArray([Float64[] for z in zones], zones)
    reserve_up_price_points = AxisArrays.AxisArray([Float64[] for z in zones], zones)

    reserve_down_mkt_params = read_data(joinpath(simulation_dir, "markets_data", "reserve_down_mkt_param.csv"))
    price_cap_reservedown = AxisArrays.AxisArray(reserve_down_mkt_params.price_cap * 1.0, zones)

    zonal_load = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))
    zonal_reserve_up = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))
    zonal_reserve_down = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))

    for (zone_num, zone) in enumerate(zones)
        idx = findfirst(x -> x == "$(zone)", reserve_up_mkt_params.zones)
        if reserve_up_mkt_params.ORDC[idx]
            reserve_up_num_points[zone] = reserve_up_mkt_params.ORDC_points[idx]
            for point in 1:reserve_up_num_points[zone]
                push!(reserve_up_break_points[zone], reserve_up_mkt_params[idx, Symbol("ORDC_x$(1)")]) * reserve_up_market_bool
                push!(reserve_up_price_points[zone], reserve_up_mkt_params[idx, Symbol("ORDC_y$(1)")]) * reserve_up_market_bool
            end
        else
            reserve_up_break_points[zone] = [0.0, 1.0, 1.0, 10.0] * reserve_up_market_bool
            reserve_up_price_points[zone] = [reserve_up_mkt_params.price_cap[idx], reserve_up_mkt_params.price_cap[idx], 0.0, 0.0] * reserve_up_market_bool
        end

        price_cap_reservedown[zone] = reserve_down_mkt_params.price_cap[idx]

        for h in 1:num_hours
            zonal_load[zone, h] = load_data[:, Symbol(zone_num)][h]
            zonal_reserve_up[zone, h] = reserve_up_demand_data[:, Symbol(zone_num)][h] * reserve_up_market_bool
            zonal_reserve_down[zone, h] = reserve_down_demand_data[:, Symbol(zone_num)][h] * reserve_down_market_bool
        end
    end

    energy_annual_increment = load_growth
    reserveup_annual_increment = load_growth
    reservedown_annual_increment = load_growth

    capacity_mkt_param_file = joinpath(simulation_dir, "markets_data", "capacity_mkt_param.csv")
    peak_load = get_peak_load(simulation)
    capacity_annual_increment = load_growth

    REC_mkt_params = read_data(joinpath(simulation_dir, "markets_data", "REC_mkt_param.csv"))
    price_cap_rec = REC_mkt_params.price_cap[1]
    rec_req = REC_mkt_params.rec_req[1] * rec_market_bool
    rec_annual_increment = REC_mkt_params.annual_increment[1] * rec_market_bool

    capacity_markets = Vector{CapacityMarket}(undef, num_invperiods)
    energy_markets = Vector{EnergyMarket}(undef, num_invperiods)
    reserve_up_markets = Vector{ReserveUpMarket}(undef, num_invperiods)
    reserve_down_markets = Vector{ReserveDownMarket}(undef, num_invperiods)
    rec_markets = Vector{RECMarket}(undef, num_invperiods)

    average_capacity_growth = Statistics.mean(capacity_annual_increment)

    # Create market products data for the horizon
    for p in 1:num_invperiods
        system_peak_load = (1 + average_capacity_growth) ^ (p) * peak_load
        capacity_markets[p] = create_capacity_demand_curve(capacity_mkt_param_file, system_peak_load, capacity_market_bool)

        energy_markets[p] = EnergyMarket(AxisArrays.AxisArray(zonal_load .* ((1 .+ energy_annual_increment) .^ p), zones, (1:num_hours)),
                                     price_cap_energy)
        reserve_up_markets[p] = ReserveUpMarket(reserve_up_break_points,
                                                reserve_up_price_points,
                                                AxisArrays.AxisArray(zonal_reserve_up .* ((1 .+ reserveup_annual_increment) .^ p), zones, (1:num_hours)))
        reserve_down_markets[p] = ReserveDownMarket(AxisArrays.AxisArray(zonal_reserve_down .* ((1 .+ reservedown_annual_increment) .^ p), zones, (1:num_hours)),
                                                  price_cap_reservedown)
        rec_markets[p] = RECMarket(min(rec_req + rec_annual_increment * (p + iteration_year - 1), 1),
                             price_cap_rec)
    end

    max_peak_loads = AxisArrays.AxisArray([maximum([maximum(market.demand[z, :]) for market in energy_markets]) for z in zones], zones)

    hour_weight = get_hour_weight(simulation)

    markets = MarketCollection.(capacity_markets,
                                energy_markets,
                                reserve_up_markets,
                                reserve_down_markets,
                                rec_markets)
    #--------------------------------------------------------------------------------------------------------------

    availability_df = read_data(joinpath(simulation_dir, "timeseries_data_files", "Availability", "DAY_AHEAD_availability.csv"))

    ed_projects = MarketProject[]

   for project in existing_projects
        tech = get_tech(project)
        ed_project = create_market_project(project,
                                          price_cap_energy[get_zone(tech)],
                                          maximum(reserve_up_break_points[get_zone(tech)]),
                                          price_cap_reservedown[get_zone(tech)],
                                          max_peak_loads,
                                          iteration_year,
                                          num_hours,
                                          num_invperiods,
                                          availability_df)

        push!(ed_projects, ed_project)
   end

   system = MarketClearingProblem(zones, lines, 0.0, markets, ed_projects, hour_weight)

    return zonal_load, system
end


text:
convert(::Type{Polygon2},
        poly::Union{ClosedLine,Line,Polygon,SurfacePolygon,
                    Khepri.ClosedPolygonalPath}) =
    Polygon2(convert.(Point2, poly.vertices))
convert(::Type{Polygon2}, sp::SurfacePath) = convert(Polygon2, sp.path)
@cxxdereference convert(::Type{Path}, poly::Polygon2) =
    closed_polygonal_path(convert.(Loc, vertices(poly)))
@cxxdereference convert(::Type{ClosedLine}, poly::Polygon2) =
    closed_line(path_vertices(convert(Path, poly)))
@cxxdereference convert(::Type{Line}, poly::Polygon2) =
    line(path_vertices(convert(Path, poly)))
@cxxdereference convert(::Type{Polygon}, poly::Polygon2) =
    polygon(path_vertices(convert(Path, poly)))
@cxxdereference convert(::Type{Surface}, poly::Polygon2) =
    surface(convert(Polygon, poly))
@cxxdereference convert(::Type{SurfacePolygon}, poly::Polygon2) =
    surface_polygon(path_vertices(convert(Path, poly)))
@cxxdereference convert(::Type{SurfacePath}, poly::Polygon2) =
    surface_path(convert(Path, poly))

convert(::Type{PolygonWithHoles2},
        v::AbstractVector{<:Union{ClosedLine,Line,Polygon,
                                  Khepri.ClosedPolygonalPath}}) =
    length(v) > 1 ?
        PolygonWithHoles2(convert(Polygon2, first(v)),
                          convert.(Polygon2, v[2:end])) :
        PolygonWithHoles2(convert(Polygon2, first(v)))

@cxxdereference convert(P::Type{<:Union{ClosedLine,Line,Path,Polygon}},
                        poly::PolygonWithHoles2) =
    convert(P, outer_boundary(poly))
@cxxdereference convert(::Type{Surface}, poly::PolygonWithHoles2) =
    subtraction(convert(Surface, outer_boundary(poly)),
                convert.(Surface, holes(poly)))


text:
#TODO don't know what to do if it is uninitalized
#so for now defining a Singleton for the default
struct SingletonInferenceVariable <: InferenceVariable end

"""
$(TYPEDEF)
"""
mutable struct VariableNodeData{T<:InferenceVariable}
  val::Array{Float64,2}
  bw::Array{Float64,2}
  BayesNetOutVertIDs::Array{Symbol,1}
  dimIDs::Array{Int,1} # Likely deprecate
  dims::Int
  eliminated::Bool
  BayesNetVertID::Symbol #  Union{Nothing, }
  separator::Array{Symbol,1}
  softtype::T
  initialized::Bool
  inferdim::Float64
  ismargin::Bool
  dontmargin::Bool
  # Tonio surprise TODO
  # frontalonly::Bool
  # A valid, packable default constructor is needed.

end

VariableNodeData(params...) = VariableNodeData{InferenceVariable}(params...)

function VariableNodeData()
    st = stacktrace()
    @warn "VariableNodeData() is depreciated please use VariableNodeData{T}() or VariableNodeData(softtype::T) where T <: InferenceVariable. Enable DEBUG logging for stack trace."
    @debug st
    VariableNodeData{InferenceVariable}(zeros(1,1), zeros(1,1), Symbol[], Int[], 0, false, :NOTHING, Symbol[], SingletonInferenceVariable(), false, 0.0, false, false)
end

VariableNodeData{T}() where {T <:InferenceVariable} =
        VariableNodeData{T}(zeros(1,1), zeros(1,1), Symbol[], Int[], 0, false, :NOTHING, Symbol[], T(), false, 0.0, false, false)

VariableNodeData(softtype::T) where T <: InferenceVariable =
        VariableNodeData{T}(zeros(1,1), zeros(1,1), Symbol[], Int[], 0, false, :NOTHING, Symbol[], softtype, false, 0.0, false, false)

"""
$(TYPEDEF)
"""
mutable struct PackedVariableNodeData
  vecval::Array{Float64,1}
  dimval::Int
  vecbw::Array{Float64,1}
  dimbw::Int
  BayesNetOutVertIDs::Array{Symbol,1} # Int
  dimIDs::Array{Int,1}
  dims::Int
  eliminated::Bool
  BayesNetVertID::Symbol # Int
  separator::Array{Symbol,1} # Int
  softtype::String
  initialized::Bool
  inferdim::Float64
  ismargin::Bool
  dontmargin::Bool
  PackedVariableNodeData() = new()
  PackedVariableNodeData(x1::Vector{Float64},
                         x2::Int,
                         x3::Vector{Float64},
                         x4::Int,
                         x5::Vector{Symbol}, # Int
                         x6::Vector{Int},
                         x7::Int,
                         x8::Bool,
                         x9::Symbol, # Int
                         x10::Vector{Symbol}, # Int
                         x11::String,
                         x12::Bool,
                         x13::Float64,
                         x14::Bool,
                         x15::Bool ) = new(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15)
end

# AbstractPointParametricEst interface
abstract type AbstractPointParametricEst end
"""
    $TYPEDEF

Data container to store Parameteric Point Estimate (PPE) for mean and max.
"""
struct MeanMaxPPE <: AbstractPointParametricEst
    solverKey::Symbol #repeated because of Sam's request
	suggested::Vector{Float64}
    max::Vector{Float64}
    mean::Vector{Float64}
    lastUpdatedTimestamp::DateTime
end
MeanMaxPPE(solverKey::Symbol, suggested::Vector{Float64}, max::Vector{Float64},mean::Vector{Float64}) = MeanMaxPPE(solverKey, suggested, max, mean, now())

getMaxPPE(est::AbstractPointParametricEst) = est.max
getMeanPPE(est::AbstractPointParametricEst) = est.mean
getLastUpdatedTimestamp(est::AbstractPointParametricEst) = est.lastUpdatedTimestamp

VariableEstimate(params...) = errror("VariableEstimate is depreciated, please use MeanMaxPPE")

"""
    $(TYPEDEF)
Fundamental structure for a DFG variable with fields:
$(TYPEDFIELDS)
"""
mutable struct DFGVariable <: AbstractDFGVariable
    label::Symbol
    timestamp::DateTime
    tags::Vector{Symbol}
    estimateDict::Dict{Symbol, <: AbstractPointParametricEst}
    solverDataDict::Dict{Symbol, VariableNodeData}
    smallData::Dict{String, String}
    bigData::Dict{Symbol, AbstractBigDataEntry}
    ready::Int
    backendset::Int
    _internalId::Int64
end

"""
    $SIGNATURES
DFGVariable constructors.
"""
function DFGVariable(label::Symbol, _internalId::Int64 = 0) #where {T <:InferenceVariable}
	st = stacktrace()
    @warn "DFGVariable(label::Symbol, _internalId::Int64 = 0) is depreciated please use DFGVariable(label::Symbol, softtype::T, _internalId::Int64 = 0) where T <: InferenceVariable. Enable DEBUG logging for the stack trace."
	@debug st
    T = InferenceVariable
    DFGVariable(label, now(), Symbol[],
                  Dict{Symbol, MeanMaxPPE}(),
                  Dict{Symbol, VariableNodeData{T}}(:default => VariableNodeData()),
                  Dict{String, String}(),
                  Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId)
end
DFGVariable(label::Symbol, softtype::T, _internalId::Int64 = 0) where {T <: InferenceVariable}  =
    DFGVariable(label, now(), Symbol[],
              Dict{Symbol, MeanMaxPPE}(),
              Dict{Symbol, VariableNodeData{T}}(:default => VariableNodeData{T}()),
              Dict{String, String}(),
              Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId)

# DFGVariable(label::Symbol, _internalId::Int64) =
#         DFGVariable(label, now(), Symbol[], Dict{Symbol, Dict{Symbol, VariableEstimate}}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId)
#
# DFGVariable(label::Symbol) =
#         DFGVariable(label, now(), Symbol[], Dict{Symbol, VariableEstimate}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, 0)
#

# Accessors
label(v::DFGVariable) = v.label
timestamp(v::DFGVariable) = v.timestamp
tags(v::DFGVariable) = v.tags
estimates(v::DFGVariable) = v.estimateDict
estimate(v::DFGVariable, key::Symbol=:default) = haskey(v.estimateDict, key) ? v.estimateDict[key] : nothing

"""
    $SIGNATURES

Retrieve the soft type name symbol for a DFGVariable or DFGVariableSummary. ie :Point2, Pose2, etc.
"""
softtype(v::DFGVariable)::Symbol = Symbol(typeof(getSofttype(v)))

"""
    $SIGNATURES

Retrieve solver data structure stored in a variable.
"""
solverData(v::DFGVariable, key::Symbol=:default) = haskey(v.solverDataDict, key) ? v.solverDataDict[key] : nothing
"""
    $SIGNATURES

Retrieve data structure stored in a variable.
"""
function getData(v::DFGVariable; solveKey::Symbol=:default)::VariableNodeData
  @warn "getData is deprecated, please use solverData()"
  return v.solverDataDict[solveKey]
end
"""
    $SIGNATURES

Set solver data structure stored in a variable.
"""
setSolverData(v::DFGVariable, data::VariableNodeData, key::Symbol=:default) = v.solverDataDict[key] = data
solverDataDict(v::DFGVariable) = v.solverDataDict
internalId(v::DFGVariable) = v._internalId
# Todo: Complete this.
smallData(v::DFGVariable) = v.smallData
bigData(v::DFGVariable) = v.bigData

"""
    $(SIGNATURES)
Structure for first-class citizens of a DFGVariable.
"""
mutable struct DFGVariableSummary <: AbstractDFGVariable
    label::Symbol
    timestamp::DateTime
    tags::Vector{Symbol}
    estimateDict::Dict{Symbol, <:AbstractPointParametricEst}
    softtypename::Symbol
    _internalId::Int64
end
label(v::DFGVariableSummary) = v.label
timestamp(v::DFGVariableSummary) = v.timestamp
tags(v::DFGVariableSummary) = v.tags
estimates(v::DFGVariableSummary) = v.estimateDict
estimate(v::DFGVariableSummary, key::Symbol=:default) = haskey(v.estimateDict, key) ? v.estimateDict[key] : nothing
softtype(v::DFGVariableSummary)::Symbol = v.softtypename
internalId(v::DFGVariableSummary) = v._internalId

# SKELETON DFG
"""
	$(TYPEDEF)
Skeleton factor with essentials.
"""
struct SkeletonDFGFactor <: AbstractDFGFactor
    label::Symbol
	tags::Vector{Symbol}
	_variableOrderSymbols::Vector{Symbol}
end

#NOTE I feel like a want to force a variableOrderSymbols
SkeletonDFGFactor(label::Symbol, variableOrderSymbols::Vector{Symbol} = Symbol[]) = SkeletonDFGFactor(label, Symbol[], variableOrderSymbols)

label(f::SkeletonDFGFactor) = f.label
tags(f::SkeletonDFGFactor) = f.tags


text:
@testset "Row" begin
  R = FlintZZ

  # Construction

  A = @inferred SRow{fmpz}()
  @test A isa SRow{fmpz}

  B = @inferred SRow(R)
  @test B isa SRow{fmpz}

  C = @inferred sparse_row(R)
  @test C isa SRow{fmpz}

  D = @inferred sparse_row(R, [(1, fmpz(1)), (2, fmpz(2))])
  @test D isa SRow{fmpz}

  E = @inferred sparse_row(R, [(1, 1), (2, 2)])
  @test E isa SRow{fmpz}

  F = @inferred sparse_row(R, [1, 2], [fmpz(1), fmpz(2)])
  @test F isa SRow{fmpz}

  # Equality

  @test @inferred ==(D, E)
  @test @inferred ==(E, F)

  # Hashing

  h = @inferred hash(D)
  @test h == hash(E)

  # Accesing elements

  @test fmpz(1) == @inferred D[1]
  @test fmpz(2) == @inferred D[2]
  @test fmpz(0) == @inferred D[3]
  @test fmpz(0) == @inferred D[1000]

  G = @inferred copy(F)
  @test G == F

  # Zero test

  @test @inferred iszero(A)
  @test !iszero(D)

  # Modular reduction

  for T in [Int, fmpz]
    G = sparse_row(FlintZZ, collect(1:5), map(fmpz, collect(1:5)))
    mod!(G, T(2))
    @test G == sparse_row(FlintZZ, [1, 3, 5], fmpz[1, 1, 1])

    G = sparse_row(FlintZZ, collect(1:5), map(fmpz, collect(1:5)))
    mod_sym!(G, T(3))
    @test G == sparse_row(FlintZZ, [1, 2, 4, 5], fmpz[1, -1, 1, -1])
  end

  # Change ring

  G = sparse_row(FlintZZ, collect(1:5), map(fmpz, collect(1:5)))

  f = x -> (x^2 - 4)
  H = @inferred map_entries(f, G)
  @test H == sparse_row(FlintZZ, [1, 3, 4, 5], fmpz[-3, 5, 12, 21])

  Rx, x = PolynomialRing(R, "x", cached = false)
  H = @inferred change_base_ring(Rx, G)
  @test H == sparse_row(Rx, collect(1:5), map(Rx, collect(1:5)))

  # Iterator interface

  @test [(i, fmpz(i)) for i in 1:5] == @inferred collect(G)

  # Dot product

  A = sparse_row(FlintZZ, [1, 3, 5], fmpz[1, 2, 3])
  B = sparse_row(FlintZZ, [3, 4, 6], fmpz[10, 1, 1])

  @test fmpz(20) == @inferred dot(A, B)

  # Inplace scaling

  A = sparse_row(FlintZZ, [1, 2, 4], fmpz[1, 2, 3])
  scale_row!(A, fmpz(2))
  @test A == sparse_row(FlintZZ, [1, 2, 4], fmpz[2, 4, 6])

  # Addition
  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[1, 2, 3, 5])
  B = sparse_row(FlintZZ, [2, 3, 4, 6], fmpz[-2, 4, 3, 1])
  @test sparse_row(FlintZZ, [1, 3, 4, 5, 6], fmpz[1, 7, 3, 5, 1]) == @inferred A + B

  # Subtraction

  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[1, 2, 3, 5])
  B = sparse_row(FlintZZ, [2, 3, 4, 6], fmpz[2, -4, -3, -1])
  @test sparse_row(FlintZZ, [1, 3, 4, 5, 6], fmpz[1, 7, 3, 5, 1]) == @inferred A - B

  # Scalar multiplication
  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[2, 4, 8, 6])
  for T in [Int, BigInt, fmpz]
    b = T(2)
    B = @inferred b * A
    @test B == map_entries(x -> T(2) * x, A)

    b = T(2)
    B = @inferred div(A, b)
    @test B == map_entries(x -> div(x, fmpz(2)), A)

    b = T(2)
    B = @inferred divexact(A, b)
    @test B == map_entries(x -> divexact(x, fmpz(2)), A)
  end

  # Elementary row operation
  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[1, 2, 3, 5])
  B = sparse_row(FlintZZ, [2, 3, 4, 6], fmpz[-1, 4, 3, 1])
  C = add_scaled_row(B, A, fmpz(2))

  @test C == sparse_row(FlintZZ, [1, 3, 4, 5, 6], fmpz[1, 11, 6, 5, 2])

  # Maximum

  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[-5, 2, 4, 10])
  @test 10 == @inferred maximum(A)

  # Minimum

  @test -5 == @inferred minimum(A)

  # Lifting

  S = ResidueRing(FlintZZ, 5)
  A = sparse_row(S, [1, 2, 3, 5], [1, 1, 2, 3])
  B = @inferred lift(A)
  @test sparse_row(R, [1, 2, 3, 5], [1, 1, 2, 3]) == B

  # 2-norm

  A = sparse_row(FlintZZ, [1, 2, 3, 5], fmpz[-5, 2, 4, 10])
  b = @inferred norm2(A)
  @test b == fmpz(25 + 4 + 16 + 100)

  S = ResidueRing(FlintZZ, 5)
  A = sparse_row(S, [1, 2, 3, 5], [1, 1, 2, 3])
  b = @inferred norm2(A)
  @test b == R(0)

  # Maximum/minimum

  A = sparse_row(FlintZZ, [1, 3, 4, 5], fmpz[-5, 2, -10, 1])
  @test maximum(abs, A) == fmpz(10)
  B = sparse_row(FlintQQ, [1, 2, 4, 5], map(fmpq, [1, 2, 9//4, 1]))
  @test maximum(B) == fmpq(9, 4)
  C = sparse_row(FlintZZ, [1, 2, 4, 5], fmpz[-10, 100, 1, 1])
  @test minimum(C) == fmpz(-10)

end


text:
using Compat
using Compat.Test
using Opus
using Ogg
using DSP
using FFTW

testdir = dirname(@__FILE__)

# First, try roundtripping some signals and ensuring they're reasonably close
function avg_roundtrip_error(audio; preskip=312)
    fio = IOBuffer()
    Opus.save(fio, audio, 48000)
    seekstart(fio)
    audio_dec = Opus.load(fio)[1]

    N = length(audio) - preskip
    avg_error = sum(abs.(audio_dec[preskip+1:min(end,length(audio))] - audio[1:end-preskip]))/N
    return avg_error
end

# We'll construct "one second" of signal
t = range(0,stop=1,length=48000)
sin_signal = sin.(2*π*440*t)
harmonic_signal = sum([.1*k^-1.5*sin.(2*π*440*k*t) for k in 1:.1:4])
filtered_noise = filt([0.018, 0.054, 0.054, 0.018], [1.0, -1.760, 1.182, -0.278], randn(1000))

# Each signal is harder for Opus to model, so increase the error bounds for each one
@test avg_roundtrip_error(sin_signal) < .01
@test avg_roundtrip_error(harmonic_signal) < .02
@test avg_roundtrip_error(filtered_noise) < .1

# Now perform some tests to ensure that we can decode some Ogg Opus files
packets = Ogg.load(joinpath(testdir, "4410hz.opus"))

serial = 1035355421

# Ensure there's only a single stream decoded
@test length(packets) == 1
# Ensure that stream has the serial number we're expecting
@test first(keys(packets)) == serial

# Ensure that stream has the number of packets that we're expecting
@test length(packets[serial]) == 3

# Build a decoder for it
opus_dec = OpusDecoder(48000,1)
audio = Opus.decode_all_packets(opus_dec, packets[serial])

# Make sure we get as many samples as we expected (note that there is some padding
# done by libopus during encoding due to the extremely short input length)
@test length(audio) == 960

# Make sure the frequency we recover is within 10 Hz of what we expect
audio_freq = (argmax(abs.(fft(audio)[1:div(end,2)])) - 1)*48000/length(audio)
@test abs(audio_freq - 4410) <= 10