text:
export CosDoc,
       cosDocOpen,
       cosDocClose,
       cosDocGetRoot,
       cosDocGetInfo,
       cosDocGetObject,
       cosDocGetPageNumbers,
       merge_streams

using Base: notnothing
using ..Common

"""
```
    CosDoc
```
PDF document format is developed in two layers. A logical PDF document
information is represented over a physical file structure called COS. `CosDoc` is
an access object to the physical file structure of the PDF document. To be used
for accessing PDF internal objects from document structure when no direct API is
available.

One can access any aspect of PDF using the COS level APIs alone. However, they
may require you to know the PDF specification in details and they are not the
most intuititive.
"""
abstract type CosDoc end

mutable struct CosDocImpl <: CosDoc
    filepath::String
    size::Int
    io::IOStream
    ps::IOStream
    hoffset::Int
    header::String
    startxref::Int
    version::Tuple{Int,Int}
    xref::Dict{CosIndirectObjectRef, CosObjectLoc}
    trailer::Vector{CosDict}
    xrefstm::Vector{CosIndirectObject{CosStream}}
    tmpfiles::Vector{AbstractString}
    isPDF::Bool
    hasNativeXRefStm::Bool
    function CosDocImpl(fp::AbstractString)
        io = util_open(fp,"r")
        sz = filesize(fp)
        ps = io
        new(fp, sz, io, ps, 0, "", 0, (0, 0),
            Dict{CosIndirectObjectRef, CosObjectLoc}(),
            [], [], [], false, false)
    end
end

Base.seek(doc::CosDoc, loc::Int) = seek(doc.ps, loc + doc.hoffset)

"""
```
    show(io::IO, doc::CosDoc)
```
Prints the CosDoc. The intent is to print lesser information from the structure
as default can be overwhelming flooding the REPL.
"""
function show(io::IO, doc::CosDoc)
    print(io, "\nCosDoc ==>\n")
    print(io, "\tfilepath:\t\t$(doc.filepath)\n")
    print(io, "\tsize:\t\t\t$(doc.size)\n")
    print(io, "\thasNativeXRefStm:\t $(doc.hasNativeXRefStm)\n")
    print(io, "\tTrailer dictionaries: \n")
    for t in doc.trailer
        print(io, '\t')
        print(io, t)
        print(io, '\n')
    end
end

"""
```
    cosDocClose(doc::CosDoc)
```
Reclaims all system resources consumed by the `CosDoc`. The `CosDoc` should not
be used after this method is called. `cosDocClose` only needs to be explicitly
called if you have opened the document by 'cosDocOpen'. Documents opened with
`pdDocOpen` do not need to use this method.
"""
function cosDocClose(doc::CosDocImpl)
    util_close(doc.ps)
    for path in doc.tmpfiles
        rm(path)
    end
end

"""
```
    cosDocOpen(filepath::AbstractString) -> CosDoc
```
Provides the access to the physical file and file structure of the PDF document.
Returns a `CosDoc` which can be subsequently used for all query into the PDF
files. Remember to release the document with `cosDocClose`, once the object is
used.
"""
function cosDocOpen(fp::AbstractString)
    doc = CosDocImpl(abspath(fp));
    ps = doc.ps
    h = read_header(ps)
    doc.version = (h[1], h[2])
    doc.header = String(h[3])
    doc.hoffset = h[4]
    doc.isPDF = (doc.header == "PDF")
    doc_trailer_update(ps, doc)
    return doc
end

"""
```
    cosDocGetRoot(doc::CosDoc) -> CosDoc
```
The structural starting point of a PDF document. Also known as document root
dictionary. This provides details of object locations and document access
methodology. This should not be confused with the `catalog` object of the PDF
document.
"""
cosDocGetRoot(doc::CosDoc) = CosNull

"""
```
    cosDocGetObject(doc::CosDoc, obj::CosObject) -> CosObject
```
PDF objects are distributed in the file and can be cross referenced from one
location to another. This is called as indirect object referencing. However, to
extract actual information one needs access to the complete object (direct
object). This method provides access to the direct object after searching for the
object in the document structure. If an indirect object reference is passed as
`obj` parameter the complete `indirect object` (reference as well as all content
of the object) are returned. A `direct object` passed to the method is returned
as is without any translation. This ensures the user does not have to go through
checking the type of the objects before accessing the contents.
"""
cosDocGetObject(doc::CosDoc, obj::CosObject) = CosNull

"""
```
    cosDocGetObject(doc::CosDoc, dict::CosObject, key::CosName) -> CosObject
```
Returns the object referenced inside the `dict` dictionary. `dict` can be a PDF
dictionary object reference or an indirect object or a direct `CosDict` object.
"""
cosDocGetObject(doc::CosDoc,
                dict::CosIndirectObject{CosDict},
                key::Union{CosName, CosNullType}) =
    cosDocGetObject(doc, dict.obj, key)

function cosDocGetObject(doc::CosDoc,
                         dict::CosIndirectObjectRef,
                         key::Union{CosName, CosNullType})
    dictd = cosDocGetObject(doc, dict)
    dictd === CosNull && return CosNull
    return cosDocGetObject(doc, dictd, key)
end

cosDocGetObject(doc::CosDoc, dict::CosDict, key::CosName) =
    cosDocGetObject(doc, get(dict, key))

function cosDocGetObject(doc::CosDoc, dict::CosDict, key::CosNullType)
    dres = CosDict()
    for (k, v) in dict.val
        set!(dres, k, cosDocGetObject(doc, v))
    end
    return dres
end

function cosDocGetRoot(doc::CosDocImpl)
    root = doc.hasNativeXRefStm ? get(doc.xrefstm[1], CosName("Root")) :
                                  get(doc.trailer[1], CosName("Root"))
    return cosDocGetObject(doc, root)
end

function cosDocGetInfo(doc::CosDocImpl)::Union{CosNullType,
                                               CosDict,
                                               CosIndirectObject{CosDict}}
    info = doc.hasNativeXRefStm ? get(doc.xrefstm[1], cn"Info") :
        get(doc.trailer[1], cn"Info")
    return cosDocGetObject(doc, info)
end

cosDocGetObject(doc::CosDocImpl, obj::CosObject) = obj

function cosDocGetObject(doc::CosDocImpl, ref::CosIndirectObjectRef)
    locObj = get(doc.xref, ref, CosObjectLoc(-1))
    locObj.loc == -1 && return CosNull
    return cosDocGetObject(doc, locObj.stm, ref, locObj)
end

function cosDocGetObject(doc::CosDocImpl, stm::CosNullType,
                         ref::CosIndirectObjectRef, locObj::CosObjectLoc)
    if (locObj.obj == CosNull)
        seek(doc, locObj.loc)
        locObj.obj = parse_indirect_obj(doc.ps, doc.hoffset, doc.xref)
        attach_object(doc, locObj.obj)
    end
    return locObj.obj
end

function cosDocGetObject(doc::CosDocImpl, stmref::CosIndirectObjectRef,
                         ref::CosIndirectObjectRef, locObj::CosObjectLoc)
    objstm = cosDocGetObject(doc, stmref)
    if (objstm === CosNull)
        #= This is not really needed but PDF specification is kind of
        equivocal if object streams should be referenced in the XRef stream.
        An object stream itself, like any stream, shall be an indirect object,
        and therefore, there shall be an entry for it in a cross-reference table
        or cross-reference stream (see 7.5.8, "Cross-Reference Streams"),
        although there might not be any references to it (of the form 243 0 R).
        =#
        objstm = scan_object_stream(doc, stmref)
        attach_object(doc, objstm)
    end
    (objstm === CosNull) && return CosNull
    if (locObj.obj === CosNull)
        locObj.obj = cosObjectStreamGetObject(objstm, ref, locObj.loc)
        attach_object(doc, locObj.obj)
    end
    return locObj.obj
end

function scan_object_stream(doc::CosDocImpl, stmref::CosIndirectObjectRef)
    look_ahead = 2048
    loc = doc.startxref - look_ahead
    if loc < 0
        loc = 0
    end
    seek(doc, loc)
    look_ahead = doc.startxref - loc
    ref = get(stmref)
    keyword = "$(ref[1]) $(ref[2]) obj"
    loc1 = locate_keyword!(doc.ps, transcode(UInt8, keyword), look_ahead)
    loc1 < 0 && return CosNull
    seek(doc, loc + loc1)
    obj = parse_indirect_obj(doc.ps, doc.hoffset, doc.xref)
    obj === CosNull && return CosNull
    doc.xref[stmref] = CosObjectLoc(loc + loc1, CosNull, obj)
    return obj
end

function read_header(ps)
    major = minor = 0x0
    b = UInt8[]
    pos = 0
    while !eof(ps)
        c = _peekb(ps)
        if c != PERCENT
            readline(ps)
            continue
        end
        cnt = 1
        skipv(ps, PERCENT)
        pos = position(ps) - 1
        empty!(b)
        c = advance!(ps)
        while c != MINUS_SIGN && !is_crorlf(c) && !eof(ps)
            cnt += 1
            push!(b, c)
            c = advance!(ps)
        end
        if eof(ps)
            empty!(b)
            break
        end
        if cnt == 4 && c == MINUS_SIGN
            major = advance!(ps)
            if !ispdfdigit(major)
                readline(ps)
                continue
            end
            major -= DIGIT_ZERO
            skipv(ps, PERIOD)
            minor = advance!(ps)
            if !ispdfdigit(minor)
                readline(ps)
                continue
            end
            minor -= DIGIT_ZERO
            break
        end
        pos = 0
        readline(ps)
    end
    return [major, minor, b, pos]
end

function read_trailer(ps::IOStream, lookahead::Int)
    chomp_space!(ps)
    locate_keyword!(ps,TRAILER,lookahead) < 0 && error(E_UNEXPECTED_CHAR)
    chomp_eol!(ps)
    skipv(ps,LESS_THAN)
    skipv(ps,LESS_THAN)

    dict = parse_dict(ps)
    chomp_space!(ps)
    return dict
end

function doc_trailer_update(ps::IOStream, doc::CosDocImpl)
    TRAILER_REWIND = 50

    seek(ps, doc.size-TRAILER_REWIND)

    if (doc.isPDF)
        locate_keyword!(ps,STARTXREF,TRAILER_REWIND) < 0 &&
            error(E_UNEXPECTED_CHAR)
        chomp_space!(ps)
        doc.startxref = parse_number(ps).val
        chomp_space!(ps)
        #Check for EOF
        locate_keyword!(ps,EOF) != 0 && error(E_UNEXPECTED_CHAR)
    end

    if doc.isPDF
        seek(doc, doc.startxref)
        chomp_space!(ps)
        doc.hasNativeXRefStm = ps |> _peekb |> ispdfdigit
        (doc.hasNativeXRefStm) ? read_xref_streams(ps, doc) :
                                 read_xref_tables(ps, doc)
    end
end

attach_object(doc::CosDocImpl, obj::CosObject) = nothing

attach_object(doc::CosDocImpl, objstm::CosIndirectObject{CosObjectStream})=
  attach_object(doc,objstm.obj.stm)

attach_object(doc::CosDocImpl, indstm::CosIndirectObject{CosStream})=
  attach_object(doc,indstm.obj)

function attach_object(doc::CosDocImpl, stm::CosStream)
    tmpfile = get(get(stm,  CosName("F")))
    push!(doc.tmpfiles, String(tmpfile))
    return nothing
end

function attach_xref_stream(doc::CosDocImpl,
                            xrefstm::CosIndirectObject{CosStream})
    attach_object(doc, xrefstm)
    push!(doc.xrefstm, xrefstm)
end

function read_xref_streams(ps::IOStream, doc::CosDocImpl)
    found = false
    while(true)
        xrefstm = parse_indirect_obj(ps, doc.hoffset, doc.xref)

        if (!found)
            get(xrefstm,  CosName("Root")) == CosNull && error(E_BAD_TRAILER)
            attach_xref_stream(doc, xrefstm)
            found = true
        else
            attach_xref_stream(doc, xrefstm)
        end
        read_xref_stream(xrefstm, doc)

        prev = get(xrefstm,  CosName("Prev"))
        prev === CosNull && break
        seek(doc, get(prev))
    end
end

function read_xref_tables(ps::IOStream, doc::CosDocImpl)
    found = false
    while(true)
        read_xref_table(ps,doc)
        trailer = read_trailer(ps, length(TRAILER))

        if (!found)
            get(trailer,  CosName("Root")) == CosNull && error(E_BAD_TRAILER)
            push!(doc.trailer, trailer)
            found = true
        else
            push!(doc.trailer, trailer)
        end
        #Hybrid case
        loc = get(trailer,  CosName("XRefStm"))
        if (loc != CosNull)
            seek(doc, get(loc))
            xrefstm = parse_indirect_obj(ps, doc.hoffset, doc.xref)
            attach_object(doc, xrefstm)
            read_xref_stream(xrefstm,doc)
        end
        prev = get(trailer, CosName("Prev"))
        prev == CosNull && break
        seek(doc, get(prev))
    end
end

# The xref stream may be accessed later. There is no point encrypting this data
# Ideal will be to remove the filter.
read_xref_stream(xrefstm::CosObject, doc::CosDocImpl) =
    read_xref_stream(xrefstm, doc.xref)

function read_xref_table(ps::IOStream, doc::CosDocImpl)
    skipv(ps, XREF)
    chomp_eol!(ps)

    while (true)
        ps |> _peekb |> ispdfdigit || break

        oid = parse_unsignednumber(ps).val
        n_entry = parse_unsignednumber(ps).val

        for i=1:n_entry
            v = UInt8[]

            for j = 1:20
                push!(v, advance!(ps))
            end

            if (v[18] != LATIN_F)
                ref = CosIndirectObjectRef(oid, parse(Int,String(v[12:16])))

                if !haskey(doc.xref,ref)
                    doc.xref[ref] = CosObjectLoc(parse(Int,String(v[1:10])))
                end
            end

            oid +=1
        end
    end
    return doc.xref
end

function find_ntree(fn::Function, doc::CosDoc,
                    node::CosTreeNode{K}, key::K, refdata::R) where {K, R}
    inrange = 0
    if node.range !== nothing
        inrange = (key < node.range[1]) ? -1 :
                  (key > node.range[2]) ?  1 : 0
    end
    if inrange == 0
        # This is the leaf
        # TBD: look into the values.
        node.kids === nothing && return (0, fn(doc, node.values, key, refdata))
        for kid in kids
            kidobj = cosDocGetObject(doc, kid)
            kidnode = createTreeNode(K, kidobj)
            inrange, val = find_ntree(fn, doc, kidnode, key, refdata)
            inrange == -1 && break
            inrange == 0 && return (inrange,val)
        end
    else
        return (inrange, nothing)
    end
    return (inrange, nothing)
end

using LabelNumerals
using RomanNumerals

const PDF_PageNumerals = [AlphaNumeral, RomanNumeral, Int]

# This may look non-intuitive but PDF pages can have the same page labels for
# multiple pages
# Table 159 - PDF Spec
function find_page_label(doc::CosDoc, values::Vector{Tuple{Int,CosObject}},
                         key::Int, label::String)
    prev_pageno = 0
    found = false
    lno = nothing
    start = 1
    for (pageno, obj) in values
        if found
            if lno !== nothing
                ln = notnothing(lno)
                ln < start && throw(ErrorException(E_INVALID_PAGE_LABEL))
                found_page = prev_pageno + 1 + ln - start
                found_page <= pageno && return range(found_page, length=1)
            else
                return range(prev_pageno + 1, length=(pageno - prev_pageno))
            end
            found = false
            prev_pageno = pageno
        end
        plDict = cosDocGetObject(doc, obj)
        s  = get(plDict, cn"S")
        p  = get(plDict, cn"P")
        st = get(plDict, cn"St")

        start = (st === CosNull) ? 1 : get(st)
        pfx = ""
        if p !== CosNull
            pfx = String(p)
        end
        if s === CosNull
            if (p === CosNull && label == "") || (label == pfx)
                prev_pageno = pageno
                found = true
                continue
            end
        else
            try
                ln = (s == cn"D") ? LabelNumeral(Int, label; prefix=pfx) :
                     (s == cn"R") ? LabelNumeral(RomanNumeral, label;
                                                prefix=pfx) :
                     (s == cn"r") ? LabelNumeral(RomanNumeral, label;
                                                 prefix=pfx, caselower=true) :
                     (s == cn"A") ? LabelNumeral(AlphaNumeral, label;
                                                 prefix=pfx) :
                     (s == cn"a") ? LabelNumeral(AlphaNumeral, label;
                                                 prefix=pfx, caselower=true) :
                     throw(ErrorException(E_INVALID_PAGE_LABEL))
                lno = ln
                prev_pageno = pageno
                found = true
            catch
            end
        end
    end
    found && lno === nothing &&
        return range(prev_pageno + 1, length=(pageno - prev_pageno))
    if found && lno !== nothing
        ln = notnothing(lno)
        ln < start && throw(ErrorException(E_INVALID_PAGE_LABEL))
        found_page = prev_pageno + 1 + ln - start
        return range(found_page, length=1)
    end
    throw(ErrorException(E_INVALID_PAGE_LABEL))
end

function get_internal_pagecount(dict::CosObject)
    mytype = get(dict, cn"Type")
    isequal(mytype, cn"Pages") && return get(get(dict, cn"Count"))
    isequal(mytype, cn"Page" ) && return 1
    error(E_INVALID_OBJECT)
end

"""
```
cosDocGetPageNumbers(doc::CosDoc, catalog::CosObject, label::AbstractString) -> Range{Int}
```
PDF utilizes two pagination schemes. An internal global page number that is
maintained serially as an integer and `PageLabel` that is shown by the viewers.
Given a `label` this method returns a `range` of valid page numbers.
"""
function cosDocGetPageNumbers(doc::CosDoc,
                              catalog::CosObject, label::AbstractString)
    ref = get(catalog, cn"PageLabels")
    ref === CosNull && throw(ErrorException(E_INVALID_PAGE_LABEL))
    plroot = cosDocGetObject(doc, ref)
    troot = createTreeNode(Int, plroot)
    return find_ntree(find_page_label, doc, troot, -1, label)[2]
end

function merge_streams(cosdoc::CosDoc, stms::CosArray)
    (path,io) = get_tempfilepath()
    try
        dict = CosDict()
        set!(dict, cn"F", CosLiteralString(path))
        ret = CosStream(dict, false)
        v = get(stms)
        for stm in v
            bufstm = decode(stm)
            data = read(bufstm)
            util_close(bufstm)
            write(io, data)
        end
        attach_object(cosdoc, ret)
        return ret
    finally
        util_close(io)
    end
    return CosNull
end


text:
using DeepReshapes
using Base.Test

a = [1, 2, 3]

b = Real[1.0, 2, 6//2]

C = [1.0 2.0 3.0;
     4.0 5.0 6.0]

D = [1.0 2.0 3.0 4.0]

E = (1, 2.0, 1//2)

F = (a, b, C, (D, E))

g = [1, (2, 3)]

h = [1 (2, 3)]

I = Vector[a, b]

# Untyped describe

@test describe(1, typed = false) == ()
@test describe(a, typed = false) == (3,)
@test describe(b, typed = false) == (3,)
@test describe(C, typed = false) == (2,3)
@test describe(D, typed = false) == (1,4)
@test describe(E, typed = false) == ((), (), ())
@test describe(F, typed = false) == ((3,), (3,), (2, 3), ((1, 4), ((), (), ())))
@test describe(g, typed = false) == [(), ((), ())]
@test describe(h, typed = false) == [() ((), ())]
@test describe(I, typed = false) == [(3,), (3,)]

# Typed (default) describe

@test describe(1) == Int
@test describe(a) == (Int, 3)
@test describe(b) == (Real, 3)
@test describe(C) == (Float64, 2,3)
@test describe(D) == (Float64, 1,4)
@test describe(E) == (Int, Float64, Rational{Int})
@test describe(F) == ((Int, 3), (Real, 3), (Float64, 2,3), ((Float64, 1,4), (Int, Float64, Rational{Int})))
@test describe(g) == [Int, (Int, Int)]
@test describe(h) == [Int (Int, Int)]
@test describe(I) == [(Int, 3), (Real, 3)]

# Length determination

@test deep_length(1) == 1
@test deep_length(a) == 3
@test deep_length(b) == 3
@test deep_length(C) == 6
@test deep_length(D) == 4
@test deep_length(E) == 3
@test deep_length(F) == 19
@test deep_length(g) == 3
@test deep_length(h) == 3
@test deep_length(I) == 6

# Flattening reshapes

@test deep_reshape(1, (Int,)) == 1
@test deep_reshape(a, (Int, 3)) == [1, 2, 3]
@test deep_reshape(b, (Real, 3)) == [1.0, 2, 3//1]
@test deep_reshape(C, (Float64, 6)) == [1.0, 4.0, 2.0, 5.0, 3.0, 6.0]
@test deep_reshape(D, (Float64, 4)) == [1.0, 2.0, 3.0, 4.0]
@test deep_reshape(E, (Real, 3)) == [1, 2.0, 1//2]
@test deep_reshape(F, (Real, 19)) == [1, 2, 3, 1, 2.0, 3//1, 1.0, 4.0, 2.0, 5.0, 3.0, 6.0, 1.0, 2.0, 3.0, 4.0, 1, 2.0, 1//2]
@test deep_reshape(g, (Int, 3)) == [1, 2, 3]
@test deep_reshape(h, (Int, 3)) == [1, 2, 3]
@test deep_reshape(I, (Real, 6)) == [1, 2, 3, 1, 2.0, 3//1]

# Structuring reshapes

@test deep_reshape(1, ()) == 1
@test deep_reshape(1, (Int,)) == 1
@test deep_reshape(1, (Float64,)) == 1.0

@test deep_reshape([1, 2, 3], (Int, Int, Int)) == (1, 2, 3)
@test deep_reshape([1, 2, 3], (Int, Float64, Rational{Int})) == (1, 2.0, 3//1)
@test deep_reshape([1, 2, 3], [Int, Int, Int]) == [1, 2, 3]
@test deep_reshape([1, 2, 3], (Int, (Int, Int))) == (1, (2, 3))
@test deep_reshape([1, 2, 3], ((Int, Int), Int)) == ((1, 2), 3)
@test deep_reshape([1, 2, 3], (Int, [Int, Int])) == (1, [2, 3])
@test deep_reshape([1, 2, 3], [Int, (Int, Int)]) == [1, (2, 3)]

@test deep_reshape(collect(1:10), (2, 5)) == [1 3 5 7 9; 2 4 6 8 10]
@test deep_reshape(collect(1:10), (Int, 2, 5)) == [1 3 5 7 9; 2 4 6 8 10]
@test eltype(deep_reshape(collect(1:10), (Int, 2, 5))) == Int
@test deep_reshape(collect(1:10), (Float64, 2, 5)) == [1.0 3.0 5.0 7.0 9.0; 2.0 4.0 6.0 8.0 10.0]
@test eltype(deep_reshape(collect(1:10), (Float64, 2, 5))) == Float64

@test deep_reshape(collect(1:25), ((3, 3), (4, 4))) == ([1 4 7; 2 5 8; 3 6 9], [10  14  18  22; 11  15  19  23; 12  16  20  24; 13  17  21  25])
@test deep_reshape([1, 2, 3, 1.0, 2, 3//1], [(3,), (3,)]) == I

# Arbitrary reshapes

@test deep_reshape((C, D), (Float64, 2, 5)) == [1.0 2.0 3.0 1.0 3.0; 4.0 5.0 6.0 2.0 4.0]
@test deep_reshape((collect(1:10), [11 13 15; 12 14 16]), (2, 2, 2, 2)) == reshape(collect(1:16), 2, 2, 2, 2)

# Reshapes with non-default recursion behavior (experimental)

@test deep_reshape(1:6, (2, 3); Deep = Range) == [1 3 5; 2 4 6]
@test deep_reshape((1:3, 4:6, 7, 8:10), (10,); Deep = Union{Tuple, Range}) == collect(1:10)

# Convenience wrappers

@test deep_reshape(collect(1:100), 10, 10) == reshape(1:100, 10, 10)
@test deep_reshape([1, 2, 3], Float64, Int, Rational{Int}) == (1.0, 2, 3//1)

@test flatten(I) == [1, 2, 3, 1, 2.0, 3//1]
@test flatten(Float64, C, D) == [1.0, 4.0, 2.0, 5.0, 3.0, 6.0, 1.0, 2.0, 3.0, 4.0]
@test eltype(flatten(Float64, C, D)) == Float64

@test deep_reshape(pack(C, D)...) == (C, D)
@test deep_reshape(pack(Float64, C, D)...) == (C, D)

ext: jl


text:
"""
    HBlkDiag

A homogeneous block diagonal matrix

These are "homogeneous" in the sense that all the diagonal blocks are of
the same size.  The `k` diagonal blocks of size `r × s` are stored as an
`r × s × k` array.
"""
immutable HBlkDiag{T} <: AbstractMatrix{T}
    arr::Array{T,3}
end

function Base.cholfact!(A::HBlkDiag, uplo::Symbol=:U)
    Aa = A.arr
    r, s, k = size(Aa)
    if r != s
        throw(ArgumentError("A must be square"))
    end
    for j in 1:k
        cholfact!(Compat.view(Aa, :, :, j), uplo)
    end
    A
end

Base.copy!{T}(d::HBlkDiag{T}, s::HBlkDiag{T}) = (copy!(d.arr, s.arr); d)

Base.copy{T}(s::HBlkDiag{T}) = HBlkDiag(copy(s.arr))

Base.eltype{T}(A::HBlkDiag{T}) = T

function Base.full(A::HBlkDiag)
    aa = A.arr
    res = zeros(eltype(aa), size(A))
    p, q, l = size(aa)
    for b in 1:l
        bm1 = b - 1
        for j in 1:q, i in 1:p
            res[bm1 * p + i, bm1 * q + j] = aa[i, j, b]
        end
    end
    res
end

function Base.getindex{T}(A::HBlkDiag{T}, i::Integer, j::Integer)
    Aa = A.arr
    r, s, k = size(Aa)
    bi, ri = divrem(i - 1, r)
    bj, rj = divrem(j - 1, s)
    if bi ≠ bj  # i and j are not in a diagonal block
        return zero(T)
    end
    Aa[ri + 1, rj + 1, bi + 1]
end

Base.size(A::HBlkDiag) = ((r, s, k) = size(A.arr); (r * k, s * k))

function Base.size(A::HBlkDiag,i::Integer)
    i < 1 && throw(BoundsError())
    i > 2 && return 1
    r, s, k = size(A.arr)
    (i == 1 ? r : s) * k
end

function Base.LinAlg.A_ldiv_B!{T}(A::UpperTriangular{T,HBlkDiag{T}}, B::DenseVecOrMat{T})
    Aa = A.data.arr
    r, s, t = size(Aa)
    if r ≠ s
        throw(ArgumentError("A must be square"))
    end
    m, n = size(B, 1), size(B, 2)  # need to call size twice in case B is a vector
    if m ≠ r * t
        throw(DimensionMismatch("size(A, 2) ≠ size(B, 1)"))
    end
    scm = Array(T, (r, r))
    scv = Array(T, (r, ))
    for k in 1 : t
        offset = (k - 1) * r
        for j in 1 : r, i in 1 : j
            scm[i, j] = Aa[i, j, k]
        end
        for j in 1 : n
            for i in 1 : r
                scv[i] = B[offset + i, j]
            end
            BLAS.trsv!('U', 'N', 'N', scm, scv)
            for i in 1 : r
                B[offset + i, j] = scv[i]
            end
        end
    end
    A
end

function LinAlg.Ac_ldiv_B!{T}(A::UpperTriangular{T,HBlkDiag{T}}, B::DenseMatrix{T})
    m, n = size(B)
    aa = A.data.arr
    r, s, t = size(aa)
    if m ≠ Compat.LinAlg.checksquare(A)
        throw(DimensionMismatch("size(A,2) ≠ size(B,1)"))
    end
    scv = Array(T, (r,))
    scm = Array(T, (r, r))
    for k in 1:t
        for j in 1 : r, i in 1 : j
            scm[i, j] = aa[i, j, k]
        end
        rowoff = (k - 1) * r
        for j in 1:n
            for i in 1 : r
                scv[i] = B[rowoff + i, j]
            end
            BLAS.trsv!('U', 'T', 'N', scm, scv)
            for i in 1 : r
                B[rowoff + i, j] = scv[i]
            end
        end
    end
    B
end

function LinAlg.Ac_ldiv_B!{T}(A::UpperTriangular{T,HBlkDiag{T}}, B::SparseMatrixCSC{T})
    m, n = size(B)
    aa = A.data.arr
    r, s, t = size(aa)
    if r ≠ s || r * t ≠ m
        throw(DimensionMismatch("size(A,2) ≠ size(B,1)"))
    end
    rows = rowvals(B)
    vals = nonzeros(B)
    for j in 1 : n
        nzrj = nzrange(B, j)
        q, r = divrem(length(nzrj), s)
        if r ≠ 0
            error("length(nzrange(B, $j)) is not divisible by $s")
        end
        for b in 1 : q
            subrng = nzrj[(b - 1) * s + (1 : s)]
            rr = Compat.view(rows, subrng)
            if any(d -> d ≠ 1, diff(rr))
                error("rows of block $b in column $j are not contiguous")
            end
            q1, r1 = divrem(rr[end], s)
            if r1 ≠ 0
                error("rows of block $b in column $j do not end in a multiple of $s")
            end
            BLAS.trsv!('U', 'T', 'N', Compat.view(aa, :, :, q1), Compat.view(vals, subrng))
        end
    end
    B
end


text:
using Statistics
using NCDatasets

using ArgParse
using JSON
using Formatting

include("LinearRegression.jl")
include("nanop.jl")
include("CESMReader2.jl")

using .CESMReader2

correlation = (x1, x2) -> x1' * x2 / (sum(x1.^2)*sum(x2.^2)).^0.5

function parse_commandline()

    s = ArgParseSettings()
    @add_arg_table s begin

        "--data-file-prefix"
            help = "Data filename prefix including folder and path until the timestamp. File extension `nc` is assumed."
            arg_type = String
            required = true

        "--output-file"
            help = "Output file."
            arg_type = String
            required = true

        "--beg-year"
            help = "Year of begin."
            arg_type = Int64
            required = true

        "--end-year"
            help = "Year of end."
            arg_type = Int64
            required = true

        "--varname"
            help = "Variable name. If it is 2D then it should be (lon, lat, time), if it is 3D then it should be (lon, lat, lev/depth, time)"
            arg_type = String
            required = true

    end

    return parse_args(ARGS, s)
end

parsed = parse_commandline()
print(json(parsed, 4))

output_file = parsed["output-file"]

filename_format = format("{:s}{{:04d}}-{{:02d}}.nc", joinpath(parsed["data-file-prefix"]))

# Determine the shape of the variable

CESMReader2.iterFilenames(filename_format, (parsed["beg-year"], parsed["end-year"])) do filename, y, m

    Dataset(filename, "r") do ds
        v = ds[parsed["varname"]]
        global var_dins = dimnames(v)
        global var_dims = [ ds.dim[var_dins[i]] for i=1:length(var_dins) ]

        if var_dins[end] != "time"
            throw(ErrorException("The last dimension is not time"))
        end

        if var_dims[end] != 1
            throw(ErrorException("The last dimension time has more than 1 record"))
        end

        global spatial_dins = var_dins[1:end-1]
        global spatial_dims = var_dims[1:end-1]
        global spatial_cols = [Colon() for i=1:length(spatial_dims)]

        println("var_dins: ", string(var_dins))
        println("var_dims: ", string(var_dims))

    end

    return false
end

# M = monthly
# S = seasonal
# A = annual
data_MM    = zeros(Float64, spatial_dims..., 12)
data_MMSQR = zeros(Float64, spatial_dims..., 12)

tmp_AM     = zeros(Float64, spatial_dims...)
data_AM    = zeros(Float64, spatial_dims...)
data_AMSQR = zeros(Float64, spatial_dims...)

years   = parsed["end-year"] - parsed["beg-year"] + 1
months  = years * 12

CESMReader2.iterFilenames(filename_format, (parsed["beg-year"], parsed["end-year"])) do filename, y, m
    println("Loading $filename")
    Dataset(filename, "r") do ds

        global data_MM, data_MMSQR, tmp_AM, data_AM, data_AMSQR

        v = ds[parsed["varname"]][spatial_cols..., 1]

        data_MM[spatial_cols..., m] .+= v
        data_MMSQR[spatial_cols..., m] .+= v.^2

        tmp_AM .+= v
        if m == 12
            @. tmp_AM /= 12.0
            @. data_AM += tmp_AM
            @. data_AMSQR += tmp_AM^2
            @. tmp_AM = 0.0
        end
    end
end

data_MM ./= years
data_MMSQR ./= years
data_MMVAR = abs.(data_MMSQR - data_MM.^2)
data_MMSTD = data_MMVAR.^0.5

data_AM ./= years
data_AMSQR ./= years
data_AMVAR = abs.(data_AMSQR - data_AM.^2)
data_AMSTD = data_AMVAR.^0.5

Dataset(output_file, "c") do ds

    defDim(ds, "months", 12)

    for i=1:length(var_dins)
        defDim(ds, var_dins[i], var_dims[i])
    end

    datas =  convert(Array{Any}, [

        (format("{:s}_MM",    parsed["varname"]),       data_MM,    (spatial_dins..., "months"), Dict()),
        (format("{:s}_MMVAR", parsed["varname"]),       data_MMVAR, (spatial_dins..., "months"), Dict()),
        (format("{:s}_MMSTD", parsed["varname"]),       data_MMSTD, (spatial_dins..., "months"), Dict()),

        (format("{:s}_AM",    parsed["varname"]),       data_AM,    (spatial_dins...,), Dict()),
        (format("{:s}_AMVAR", parsed["varname"]),       data_AMVAR, (spatial_dins...,), Dict()),
        (format("{:s}_AMSTD", parsed["varname"]),       data_AMSTD, (spatial_dins...,), Dict()),

    ])

    for (varname, vardata, vardim, attrib) in datas
        if ! haskey(ds, varname)
            var = defVar(ds, varname, Float64, vardim)
            var.attrib["_FillValue"] = 1e20
        end

        println("Writing variable:  ", varname)
        var = ds[varname]

        for (k, v) in attrib
            var.attrib[k] = v
        end
        var[:] = vardata
    end

end

max_stars_repo_name: meteorologytoday/CESM-diagnostic

max_issues_repo_name: meteorologytoday/CESM-diagnostic

max_forks_repo_name: meteorologytoday/CESM-diagnostic


text:
"""
    toString(array)

Returns a `String` from a `Array{UInt8, 1}`. Stops at the first `\0` found if the byte string is null-terminated.
"""
function toString(array::Array{UInt8,1})
    chars = Char.(array);
    stop = findfirst(isequal('\0'), chars);
    return isnothing(stop) ? String(chars) : String(chars[1:stop-1]);
end

"""
    toggleNoAssertions([value])

Toggles the assertions in the libIIO_jl module.

# Arguments
- `value::Bool` : an optional value to set the `NO_ASSERT` toggle to.

# Returns
- `NO_ASSERT::Bool` : the actual value of the `NO_ASSERT` toggle.
"""
function toggleNoAssertions(value = nothing)
    if typeof(value) <: Bool
        global NO_ASSERT = value;
    else
        global NO_ASSERT = !NO_ASSERT;
    end
end

"""
    iio_decode_blocks(buf, size)

Decodes an array of blocks containing attributes values.
Those come from `iio_[target]_attr_read` functions when C_NULL ("" for the julia wrapper) is passed as the attribute to read.

# Arguments
- `buf::Array{UInt8}` : an array containing the blocks returned by an IIO function.
- `size::Cssize_t`    : the total size of the blocks in the array

# Returns
An array containing :
- `(errno::Int, "")` for attributes that cannot be read as a string, where errno is a negative error number.
- `(size::Int, value::String)` for attributes that can be read. Size is the size of the block and not the actual size of the value.
"""
function iio_decode_blocks(buf::Array{UInt8}, size::Cssize_t)
    bytes_read = 0;
    attrs = Tuple{Int, String}[];
    while bytes_read < size
        block_size = reinterpret(Int32, reverse(buf[(1:4) .+ bytes_read]))[];
        if block_size % 4 != 0
            block_size += 4 - (block_size % 4); # You need to round to the next multiple of 4. Source : read byte per byte the array.
        end
        if block_size < 0
            push!(attrs, (block_size, ""));
            bytes_read += 4;
        else
            push!(attrs, (block_size, toString(buf[(1:block_size-1) .+ (4 + bytes_read)])));
            bytes_read += 4 + block_size;
        end
    end
    return attrs;
end


text:
# *****************************************************************************
# Written by Ritchie Lee, ritchie.lee@sv.cmu.edu
# *****************************************************************************
# Copyright ã 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration. All
# rights reserved.  The Reinforcement Learning Encounter Simulator (RLES)
# platform is licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You
# may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0. Unless required by applicable
# law or agreed to in writing, software distributed under the License is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.
# _____________________________________________________________________________
# Reinforcement Learning Encounter Simulator (RLES) includes the following
# third party software. The SISLES.jl package is licensed under the MIT Expat
# License: Copyright (c) 2014: Youngjun Kim.
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED
# "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# *****************************************************************************

"""
Deprecated.
Manipulation tools for DataFrame object.  Consider using built-ins and DataFramesMeta instead.
"""
module DataFrameFeatures

export add_features!, transform, transform!

import Compat.ASCIIString
using DataFrames
using Iterators
using RLESUtils, FileUtils, LookupCallbacks

function add_features!(df_files::Vector{ASCIIString}, feature_map::Vector{LookupCallback},
                        feature_names::Vector{ASCIIString}; overwrite::Bool=false)
  map(df_files) do f
    add_features!(f, feature_map, feature_names; overwrite=overwrite)
  end
end

function add_features!(df_file::AbstractString, feature_map::Vector{LookupCallback},
                        feature_names::Vector{ASCIIString}; overwrite::Bool=false)
  D = readtable(df_file)
  add_features!(D, feature_map, feature_names)
  fileroot, ext = splitext(df_file)
  f = overwrite ? df_file : fileroot * "_addfeats" * ext
  writetable(f, D)
  return f
end

function add_features!(Ds::Vector{DataFrame}, feature_map::Vector{LookupCallback},
                        feature_names::Vector{ASCIIString})
  map(Ds) do D
    add_features!(D, feature_map, feature_names)
  end
end

function add_features!(D::DataFrame, feature_map::Vector{LookupCallback},
                        feature_names::Vector{ASCIIString})
  for i = 1:length(feature_map)
    lookups = map(Symbol, feature_map[i].lookups) #map lookups to symbols
    f = feature_map[i].callback #function
    feat_name = Symbol(feature_names[i]) #as a symbol
    D[feat_name] = Array(Any, size(D, 1)) #add a new column
    for r = 1:size(D, 1)
      x = map(l -> D[r, l], lookups) #extract to a vector
      D[r, feat_name] = f(x...)
    end
  end
  return D
end

function transform(df_files::Vector{ASCIIString}, args...;
                    overwrite::Bool=false)
  map(df_files) do f
    transform(f, args..., overwrite=overwrite)
  end
end

function transform(df_file::AbstractString, args...;
                   overwrite::Bool=false)
  D = readtable(df_file)
  transform!(D, args...)
  fileroot, ext = splitext(df_file)
  f = overwrite ? df_file : fileroot * "_transform" * ext
  writetable(f, D)
  return f
end

function transform!(D::DataFrame, feat_transforms::Tuple{Symbol,Function}...)
  for (key, f) in feat_transforms
    D[key] = map(f, D[key])
  end
end

function transform!(D::DataFrame, lookup_callbacks::Vector{LookupCallback})
  for lcb in lookup_callbacks
    lookups = map(Symbol, lcb.lookups)
    f = lcb.callback
    for row = 1:size(D, 1)
      inputs = [D[row, l] for l in lookups]
      outputs = f(inputs...)
      @assert length(inputs) == length(outputs) == length(lookups)
      for i = 1:length(lookups)
        D[row, lookups[i]] = outputs[i]
      end
    end
  end
end

end #module


text:
# AUTO GENERATED FILE - DO NOT EDIT

export vtk_pointcloudrepresentation

"""
    vtk_pointcloudrepresentation(;kwargs...)

A PointCloudRepresentation component.
PointCloudRepresentation expect the following set of properties
  - xyz: [x0, y0, z0, x1, ..., zn]
  - rgb: [...]
  - rgba: [...]
  - scalars: [...]
Keyword arguments:
- `xyz` (Array of Reals; optional): Points coordinates
- `rgb` (Array of Reals; optional): Use RGB values to attach to the points/vertex
- `rgba` (Array of Reals; optional): Use RGBA values to attach to the points/vertex
- `scalars` (Array of Reals; optional): Field values to attach to the points
- `colorMapPreset` (String; optional): Preset name for the lookup table color map
- `colorDataRange` (Array of Reals; optional): Data range use for the colorMap
- `property` (Dict; optional): Properties to set to the actor.property
"""
function vtk_pointcloudrepresentation(; kwargs...)
        available_props = Symbol[:xyz, :rgb, :rgba, :scalars, :colorMapPreset, :colorDataRange, :property]
        wild_props = Symbol[]
        return Component("vtk_pointcloudrepresentation", "PointCloudRepresentation", "dash_vtk", available_props, wild_props; kwargs...)
end

max_stars_repo_name: ericbdaniels/dash-vtk

max_issues_repo_name: ericbdaniels/dash-vtk

max_forks_repo_name: ericbdaniels/dash-vtk


text:
# The union, intersection and difference of domains are represented with lazy domains.

issubset(d1::Domain, d2::Domain) = promotable_domains(d1, d2) && issubset1(promote_domains(d1, d2)...)
issubset(d1, d2::Domain) = issubset1(promote_domains(d1, d2)...)
issubset1(d1, d2) = issubset2(d1, d2)
issubset2(d1, d2) = d1 == d2
# this last fallback is only an approximation of the truth: if d1 equals d2, then
# d1 is a subset of d2, but the reverse is not true. So we might be returning false
# even when the correct mathematical answer is true.
# What `issubset` means for the optimizations below is:
# - if true: we are sure that d1 is a subset of d2
# - if false: either it really is false, or we don't really know
# On top of that, we only invoke issubset(d1,d2) when the arguments are both
# Domains, or when d2 is a Domain, anticipating that issubset(d1::Domain,d2)
# is often harder to implement.

issubset(d1::AbstractArray, d2::Domain) = all(in.(d1, Ref(d2)))
issubset(d1::AbstractSet, d2::Domain) = all(in.(d1, Ref(d2)))

############################
# The union of two domains
############################

"""
```using DomainSets```

A `UnionDomain` represents the union of a set of domains.
"""
struct UnionDomain{T,DD} <: CompositeDomain{T}
	domains	::	DD
end

"""
The `UnionDomain` and `UnionDomain{T}` constructors can be invoked in three ways:
- with a list of arguments: UnionDomain(d1, d2, ...)
- with a single domain: UnionDomain(d::Domain)
- or with any iterable list of domains: UnionDomain(domains)
"""
UnionDomain(domains...) = UnionDomain(domains)
UnionDomain(d::Domain) = UnionDomain((d,))
UnionDomain(domains) = _UnionDomain(promote_domains(domains))
_UnionDomain(domains) = _UnionDomain(eltype(first(domains)), domains)

UnionDomain{T}(domains...) where {T} = UnionDomain{T}(domains)
UnionDomain{T}(d::Domain) where {T} = UnionDomain{T}((d,))
UnionDomain{T}(domains) where {T} = _UnionDomain(T, convert_eltype.(T, domains))
_UnionDomain(::Type{T}, domains) where {T} = UnionDomain{T,typeof(domains)}(domains)

# The union of domains corresponds to a logical OR of their characteristic functions
composition(d::UnionDomain) = Combination()
combine(d::UnionDomain, results) = reduce(|, results)

# Make d1 ∪ d2 invoke `uniondomain` if one of the first two arguments is a Domain
union(d1::Domain, d2::Domain, domains...) = uniondomain(d1, d2, domains...)
union(d1::Domain, d2, domains...) = uniondomain(d1, d2, domains...)
union(d1, d2::Domain, domains...) = uniondomain(d1, d2, domains...)

uniondomain() = EmptySpace{Any}()
uniondomain(d1) = d1
uniondomain(d1, d2) = uniondomain1(promote_domains(d1, d2)...)

# simplification: in case the domains are equal, we don't create a union
uniondomain1(d1, d2) = d1 == d2 ? d1 : uniondomain2(d1, d2)
# if d1 is a Domain, we go one step further and invoke `issubset`
uniondomain1(d1::Domain, d2) = issubset(d2, d1) ? d1 : uniondomain2(d1, d2)
uniondomain2(d1, d2) = UnionDomain(d1, d2)
uniondomain2(d1, d2::Domain) = issubset(d1, d2) ? d2 : UnionDomain(d1, d2)

uniondomain(d1, d2, d3) = _ud3(promote_domains(d1, d2, d3)...)
_ud3(d1, d2, d3) =
	_ud3(d1, d2, d3, uniondomain(d1, d2), uniondomain(d2, d3), uniondomain(d1, d3))
_ud3(d1, d2, d3, d12::UnionDomain, d23::UnionDomain, d13::UnionDomain) =
	uniondomain(d12, d3)
_ud3(d1, d2, d3, d12, d23::UnionDomain, d13::UnionDomain) =
	uniondomain(d12, d3)
_ud3(d1, d2, d3, d12::UnionDomain, d23, d13::UnionDomain) =
	uniondomain(d23, d1)
_ud3(d1, d2, d3, d12::UnionDomain, d23::UnionDomain, d13) =
	uniondomain(d13, d2)
_ud3(d1, d2, d3, d12::UnionDomain, d23, d13) =
	uniondomain(d23, d1)
_ud3(d1, d2, d3, d12, d23::UnionDomain, d13) =
	uniondomain(d12, d3)
_ud3(d1, d2, d3, d12, d23, d13::UnionDomain) =
	uniondomain(d12, d3)
_ud3(d1, d2, d3, d12, d23, d13) =
	uniondomain(d12, d3)

uniondomain(d1, d2, d3, domains...) = _ud(promote_domains(d1, d2, d3, domains...)...)
_ud(d1, d2, d3, domains...) = uniondomain(uniondomain(d1, d2, d3), domains...)

# avoid nested union domains
uniondomain(d1::UnionDomain, d2::UnionDomain) =
	d1 == d2 ? d1 : UnionDomain(collect(Set(components(d1)) ∪ Set(components(d2))))
uniondomain1(d1::UnionDomain, d2) = UnionDomain(components(d1)..., d2)
uniondomain2(d1, d2::UnionDomain) = UnionDomain(d1, components(d2)...)

==(a::UnionDomain, b::UnionDomain) = Set(components(a)) == Set(components(b))
hash(d::UnionDomain, h::UInt) = hashrec("UnionDomain", Set(d.domains), h)

convert(::Type{Domain}, v::AbstractVector{<:Domain}) = UnionDomain(v)
convert(::Type{Domain}, v::AbstractSet{<:Domain}) = UnionDomain(v)
convert(::Type{Domain}, s::AbstractSet) = UnionDomain(map(Point,collect(s)))
convert(::Type{Domain{T}}, v::AbstractVector{<:Domain}) where {T} = UnionDomain{T}(v)
convert(::Type{Domain{T}}, v::AbstractSet{<:Domain}) where {T} = UnionDomain{T}(v)
convert(::Type{Domain{T}}, s::AbstractSet) where {T} = UnionDomain{T}(map(Point,collect(s)))

similardomain(d::UnionDomain, ::Type{T}) where {T} =
    UnionDomain(convert.(Domain{T}, components(d)))

point_in_domain(d::UnionDomain) = convert(eltype(d), point_in_domain(component(d,1)))

isempty(d::UnionDomain) = all(isempty, d.domains)

interior(d::UnionDomain) = UnionDomain(map(interior, components(d)))
closure(d::UnionDomain) = UnionDomain(map(closure, components(d)))

boundingbox(d::UnionDomain) = unionbox(map(boundingbox, components(d))...)

boundary(d::UnionDomain) = uniondomain(map(boundary, components(d))...)

Display.combinationsymbol(d::UnionDomain) = Display.Symbol('∪')
Display.displaystencil(d::UnionDomain) = composite_displaystencil(d)
show(io::IO, mime::MIME"text/plain", d::UnionDomain) = Display.composite_show(io, mime, d)
show(io::IO, d::UnionDomain) = Display.composite_show_compact(io, d)

## ualgebra

# preserve the uniondomain when mapping
map_domain(map, domain::UnionDomain) = UnionDomain(map_domain.(Ref(map), components(domain)))
mapped_domain(map, domain::UnionDomain) = UnionDomain(mapped_domain.(Ref(map), components(domain)))

# TODO: what is the correct semantics for these functions? Should we have them?
for (op, mop) in ((:minimum, :min), (:maximum, :max), (:infimum, :min), (:supremum, :max))
    @eval $op(d::UnionDomain) = mapreduce($op, $mop, components(d))
end

setdiffdomain(d1::UnionDomain, d2::UnionDomain) =
	UnionDomain(setdiffdomain.(components(d1), Ref(d2)))

function setdiffdomain1(d1::UnionDomain, d2)
    s = Set(components(d1))
    # check if any element is in d1 and just remove
    s2 = Set(setdiff(s, tuple(d2)))
    s2 ≠ s && return UnionDomain(s2)
    UnionDomain(setdiffdomain.(components(d1), Ref(d2)))
end

function setdiffdomain2(d1, d2::UnionDomain)
    result = d1
    for d in components(d2)
        result = setdiffdomain(result, d)
    end
    result
end

##############################
# The intersection of domains
##############################

"""
An `IntersectDomain` represents the intersection of a set of domains.
"""
struct IntersectDomain{T,DD} <: CompositeDomain{T}
    domains ::  DD
end

"""
The `IntersectDomain` constructor can be invoked in one of three ways:
- with a list of arguments: IntersectDomain(d1, d2, ...)
- with a single domain: IntersectDomain(d::Domain)
- or with any iterable list of domains: IntersectDomain(domains)
"""
IntersectDomain(domains...) = IntersectDomain(domains)
IntersectDomain(d::Domain) = IntersectDomain((d,))
IntersectDomain(domains) = _IntersectDomain(promote_domains(domains))
_IntersectDomain(domains) = IntersectDomain{eltype(first(domains))}(domains)

IntersectDomain{T}(domains...) where {T} = IntersectDomain{T}(domains)
IntersectDomain{T}(d::Domain) where {T} = IntersectDomain{T}((d,))
IntersectDomain{T}(domains) where {T} = _IntersectDomain(T, convert_eltype.(T, domains))
_IntersectDomain(::Type{T}, domains) where {T} = IntersectDomain{T,typeof(domains)}(domains)

# The intersection of domains corresponds to a logical AND of their characteristic functions
composition(d::IntersectDomain) = Combination()
combine(d::IntersectDomain, results) = reduce(&, results)

# Make d1 ∩ d2 invoke `intersectdomain` if one of the first two arguments is a Domain
intersect(d1::Domain, d2::Domain, domains...) = intersectdomain(d1, d2, domains...)
intersect(d1::Domain, d2, domains...) = intersectdomain(d1, d2, domains...)
intersect(d1, d2::Domain, domains...) = intersectdomain(d1, d2, domains...)

intersectdomain() = EmptySpace{Any}()
intersectdomain(d1) = d1
intersectdomain(d1, d2) = intersectdomain1(promote_domains(d1, d2)...)

intersectdomain1(d1, d2) = d1 == d2 ? d1 : intersectdomain2(d1, d2)
intersectdomain1(d1::Domain, d2) = issubset(d2, d1) ? d2 : intersectdomain2(d1, d2)
intersectdomain2(d1, d2) = IntersectDomain(d1, d2)
intersectdomain2(d1, d2::Domain) = issubset(d1, d2) ? d1 : IntersectDomain(d1, d2)

intersectdomain(d1, d2, d3) = _id3(promote_domains(d1, d2, d3)...)
_id3(d1, d2, d3) =
	_id3(d1, d2, d3, intersectdomain(d1, d2), intersectdomain(d2, d3), intersectdomain(d1, d3))
_id3(d1, d2, d3, d12::IntersectDomain, d23::IntersectDomain, d13::IntersectDomain) =
	intersectdomain(d12, d3)
_id3(d1, d2, d3, d12, d23::IntersectDomain, d13::IntersectDomain) =
	intersectdomain(d12, d3)
_id3(d1, d2, d3, d12::IntersectDomain, d23, d13::IntersectDomain) =
	intersectdomain(d23, d1)
_id3(d1, d2, d3, d12::IntersectDomain, d23::IntersectDomain, d13) =
	intersectdomain(d13, d2)
_id3(d1, d2, d3, d12::IntersectDomain, d23, d13) =
	intersectdomain(d23, d1)
_id3(d1, d2, d3, d12, d23::IntersectDomain, d13) =
	intersectdomain(d12, d3)
_id3(d1, d2, d3, d12, d23, d13::IntersectDomain) =
	intersectdomain(d12, d3)
_id3(d1, d2, d3, d12, d23, d13) =
	intersectdomain(d12, d3)

intersectdomain(d1, d2, d3, domains...) = _id(promote_domains(d1, d2, d3, domains...)...)
_id(d1, d2, d3, domains...) = intersectdomain(intersectdomain(d1, d2, d3), domains...)

# avoid nested intersect domains
intersectdomain(d1::IntersectDomain, d2::IntersectDomain) =
	d1 == d2 ? d1 : IntersectDomain(components(d1)..., components(d2)...)
intersectdomain1(d1::IntersectDomain, d2) = IntersectDomain(components(d1)..., d2)
intersectdomain2(d1, d2::IntersectDomain) = IntersectDomain(d1, components(d2)...)

function intersectdomain(d1::UnionDomain, d2::UnionDomain)
    d1 == d2 && return d1
    uniondomain(intersectdomain.(Ref(d1), components(d2))...)
end
intersectdomain(d1::UnionDomain, d2::Domain) = uniondomain(intersectdomain.(d1.domains, Ref(d2))...)
intersectdomain(d1::Domain, d2::UnionDomain) = uniondomain(intersectdomain.(Ref(d1), d2.domains)...)

(&)(d1::Domain, d2::Domain) = intersectdomain(d1,d2)

function intersectdomain(d1::ProductDomain, d2::ProductDomain)
	if compatibleproductdims(d1, d2)
        ProductDomain(map(intersectdomain, components(d1), components(d2)))
    else
        IntersectDomain(d1, d2)
    end
end

similardomain(d::IntersectDomain, ::Type{T}) where {T} =
    IntersectDomain(convert.(Domain{T}, components(d)))

==(a::IntersectDomain, b::IntersectDomain) = Set(components(a)) == Set(components(b))
hash(d::IntersectDomain, h::UInt) = hashrec("IntersectDomain", Set(components(d)), h)

boundingbox(d::IntersectDomain) = intersectbox(map(boundingbox, components(d))...)

Display.combinationsymbol(d::IntersectDomain) = Display.Symbol('∩')
Display.displaystencil(d::IntersectDomain) = composite_displaystencil(d)
show(io::IO, mime::MIME"text/plain", d::IntersectDomain) = Display.composite_show(io, mime, d)
show(io::IO, d::IntersectDomain) = Display.composite_show_compact(io, d)

#########################################
### The difference between two domains
#########################################

"A `SetdiffDomain` represents the difference between two domains."
struct SetdiffDomain{T,DD} <: CompositeDomain{T}
    domains	::	DD
	function SetdiffDomain{T,DD}(domains::DD) where {T,DD}
		@assert length(domains) == 2
		new(domains)
	end
end

SetdiffDomain(d1, d2) = _SetdiffDomain(promote_domains((d1, d2))...)
_SetdiffDomain(d1, d2) = SetdiffDomain{eltype(d1)}((d1,d2))
SetdiffDomain{T}(domains) where {T} = SetdiffDomain{T,typeof(domains)}(domains)

# The difference between two domains corresponds to a logical AND NOT of their characteristic functions
composition(d::SetdiffDomain) = Combination()
combine(d::SetdiffDomain, results) = results[1] & !results[2]

# It is difficult to calculate approximate membership exactly, but we can at
# least not enlarge the subtracted domain by invoking in rather than approx_in on it.
_approx_indomain(x, d::SetdiffDomain, comp::Combination, domains, tolerance) =
    approx_in(x, domains[1], tolerance) & !in(x, domains[2])

similardomain(d::SetdiffDomain, ::Type{T}) where {T} =
    SetdiffDomain(convert(Domain{T}, d.domains[1]), convert(Domain{T}, d.domains[2]))

# use \ as a synomym for setdiff, in the context of domains (though, generically,
# \ means left division in Julia)
\(d1::Domain, d2::Domain) = setdiffdomain(d1, d2)
\(d1::Domain, d2) = setdiffdomain(d1, d2)
\(d1, d2::Domain) = setdiffdomain(d1, d2)

# Make setdiff invoke `setdiffdomain` if one of the arguments is a Domain
setdiff(d1::Domain, d2::Domain) = setdiffdomain(d1, d2)
setdiff(d1::Domain, d2) = setdiffdomain(d1, d2)
setdiff(d1, d2::Domain) = setdiffdomain(d1, d2)

setdiffdomain(d1, d2) = setdiffdomain1(promote_domains(d1, d2)...)
setdiffdomain1(d1, d2) = setdiffdomain2(d1, d2)

function setdiffdomain2(d1, d2)
	if isempty(d2)
		d1
	elseif d1 == d2
		EmptySpace{eltype(d1)}()
	else
		SetdiffDomain(d1, d2)
	end
end
# similar to uniondomain, if d2 is a Domain we use `issubset` as an optimization
function setdiffdomain2(d1, d2::Domain)
	if isempty(d2)
		d1
	elseif issubset(d1,d2)
		EmptySpace{eltype(d1)}()
	else
		SetdiffDomain(d1, d2)
	end
end

# avoid nested difference domains
setdiffdomain1(d1::SetdiffDomain, d2) = setdiffdomain(d1.domains[1], uniondomain(d2, d1.domains[2]))

==(a::SetdiffDomain, b::SetdiffDomain) = a.domains == b.domains

boundingbox(d::SetdiffDomain) =  boundingbox(d.domains[1])

Display.combinationsymbol(d::SetdiffDomain) = Display.Symbol('\\')
Display.displaystencil(d::SetdiffDomain) = composite_displaystencil(d)
show(io::IO, mime::MIME"text/plain", d::SetdiffDomain) = Display.composite_show(io, mime, d)
show(io::IO, d::SetdiffDomain) = Display.composite_show_compact(io, d)


text:
###########################
### Auxiliary Functions ###
###########################

"""
Check if Way is One - Way
"""
function oneway(w::OpenStreetMapX.Way)
    v = get(w.tags,"oneway", "")
    if v == "false" || v == "no" || v == "0"
        return false
    elseif v == "-1" || v == "true" || v == "yes" || v == "1"
        return true
    end
    highway = get(w.tags,"highway", "")
    junction = get(w.tags,"junction", "")
    return (highway == "motorway" || highway == "motorway_link" || junction == "roundabout")
end

"""
Check if Way is Reverse
"""
reverseway(w::OpenStreetMapX.Way) = (get(w.tags,"oneway", "") == "-1")

"""
Compute the distance of a route for some `nodes` data
"""
function distance(nodes::Dict{Int,T}, route::Vector{Int}) where T<:(Union{OpenStreetMapX.ENU,OpenStreetMapX.ECEF})
    if length(route) == 0
        return Inf
    end
    dist = sum(distance(nodes[route[i-1]],nodes[route[i]]) for i = 2:length(route))
end

"""
Find Intersections of Highways ###
"""
function find_intersections(highways::Vector{OpenStreetMapX.Way})
    seen = Set{Int}()
    intersections = Dict{Int,Set{Int}}()
    for highway in highways
        for i = 1:length(highway.nodes)
            if i == 1 || i == length(highway.nodes) || (highway.nodes[i] in seen)
                get!(Set{Int}, intersections, highway.nodes[i])
            else
                push!(seen, highway.nodes[i])
            end
        end
    end
    for highway in highways
        for i = 1:length(highway.nodes)
            if i == 1 || i == length(highway.nodes) || haskey(intersections, highway.nodes[i])
                push!(intersections[highway.nodes[i]], highway.id)
            end
        end
    end
    return intersections
end

"""
Find Segments of Highways ###
"""
function find_segments(nodes::Dict{Int,T}, highways::Vector{OpenStreetMapX.Way}, intersections::Dict{Int,Set{Int}}) where T<:Union{OpenStreetMapX.ENU,OpenStreetMapX.ECEF}
    segments = OpenStreetMapX.Segment[]
    for highway in highways
        firstNode = 1
        for j = 2:length(highway.nodes)
            if highway.nodes[firstNode] != highway.nodes[j] && (haskey(intersections, highway.nodes[j])|| j == length(highway.nodes))
                if !reverseway(highway)
                    seg = OpenStreetMapX.Segment(highway.nodes[firstNode],highway.nodes[j],highway.nodes[firstNode:j], OpenStreetMapX.distance(nodes, highway.nodes[firstNode:j]), highway.id)
                    push!(segments,seg)
                else
                    seg = OpenStreetMapX.Segment(highway.nodes[j],highway.nodes[firstNode],reverse(highway.nodes[firstNode:j]), OpenStreetMapX.distance(nodes, highway.nodes[firstNode:j]), highway.id)
                    push!(segments,seg)
                end
                if !oneway(highway)
                    seg = OpenStreetMapX.Segment(highway.nodes[j],highway.nodes[firstNode],reverse(highway.nodes[firstNode:j]), OpenStreetMapX.distance(nodes, highway.nodes[firstNode:j]), highway.id)
                    push!(segments,seg)
                end
				firstNode = j
            end
        end
    end
    return segments
end


text:
module MsgHandler

import JSON

struct UnKnownMethod <: Exception end

event_id = 0

# recieve message from server
function msgRecv(sock)
  # read comming message length
  len = read(sock, 8)
  len_str = ""
  for num in len
    len_str = len_str * string(Char(num))
  end
  len = parse(Int, len_str)
  # read message
  msg_array = read(sock, len)
  msg = ""
  for ch in msg_array
    msg = msg * Char(ch)
  end
  return msg
end

# handle comming msg
# return msg length, id, method, params
function msgParse(msg)

  json_obj = JSON.parse(msg)
  # get msg id
  id = json_obj["id"]
  method = json_obj["method"]
  params = json_obj["params"]

  # println("recv id: $(id), method: $(method)")

  return (id, method, params)

end

function msgCreate(id, result)
  json_obj = Dict("jsonrpc" => "2.0",
                  "id" => id,
                  "result" => result)
  # to string
  msg = JSON.json(json_obj)
  len = "$(length(msg))"
  msg = ("0" ^ (8 - length(len))) * len * msg
  return msg
end

# create event to notify adapter
function eventCreate(method, params)
  global event_id
  event_id += 1
  json_obj = Dict("jsonrpc" => "2.0",
                  "method" => method,
                  "params" => params)
  msg = JSON.json(json_obj)
  len = "$(length(msg))"
  msg = ("0" ^ (8 - length(len))) * len * msg
  return msg
end

end # MsgHandler Module

max_stars_repo_name: judy-vscode/judy

max_issues_repo_name: judy-vscode/judy

max_forks_repo_name: judy-vscode/judy


text:
__precompile__()
module NaNMath

using Compat
# using DecFP, Quadmath

const libm = Base.libm_name

for f in (:sin, :cos, :tan, :asin, :acos, :acosh, :atanh, :log, :log2, :log10,
          :lgamma, :log1p)
    @eval begin
        ($f)(x::Float16)  = ccall(($(string(f,"f")),libm), Float16, (Float16,), x)
        ($f)(x::Float32)  = ccall(($(string(f,"f")),libm), Float32, (Float32,), x)
        ($f)(x::Float64)  = ccall(($(string(f)),libm), Float64, (Float64,), x)
        ($f)(x::BigFloat) = ccall(($(string(f,"f")),libm), BigFloat, (BigFloat,), x)
        # ($f)(x::Dec32) = ccall(($(string(f,"f")),libm), Dec32, (Dec32,), x)
        # ($f)(x::Dec64) = ccall(($(string(f,"f")),libm), Dec64, (Dec64,), x)
        # ($f)(x::Dec128) = ccall(($(string(f,"f")),libm), Dec128, (Dec128,), x)
        # ($f)(x::Float128) = ccall(($(string(f,"f")),libm), Float128, (Float128,), x)
        ($f)(x::Real) = ($f)(float(x))
        function ($f)(x::AbstractArray{T}) where T<:Number
            Base.depwarn("$f{T<:Number}(x::AbstractArray{T}) is deprecated, use $f.(x) instead.", $f)
            return ($f).(x)
        end
    end
end

# Would be more efficient to remove the domain check in Base.sqrt(),
# but this doesn't seem easy to do.
sqrt(x::Real) = x < 0.0 ? NaN : Base.sqrt(x)

# Don't override built-in ^ operator
pow(x::Float64, y::Float64) = ccall((:pow,libm),  Float64, (Float64,Float64), x, y)
pow(x::Float32, y::Float32) = ccall((:powf,libm), Float32, (Float32,Float32), x, y)
pow(x::Number,y::Number) = pow(float(x),float(y))

"""
NaNMath.sum(A)

##### Args:
* `A`: An array of floating point numbers

##### Returns:
*    Returns the sum of all elements in the array, ignoring NaN's.

##### Examples:
```julia
using NaNMath as nm
nm.sum([1., 2., NaN]) # result: 3.0
```
"""
function sum(x::AbstractArray{T}) where T<:AbstractFloat
    if length(x) == 0
        result = zero(eltype(x))
    else
        result = convert(eltype(x), NaN)
        for i in x
            if !isnan(i)
                if isnan(result)
                    result = i
                else
                    result += i
                end
            end
        end
    end

    if isnan(result)
        Compat.@warn "All elements of the array, passed to \"sum\" are NaN!"
    end
    return result
end

"""
NaNMath.maximum(A)

##### Args:
* `A`: An array of floating point numbers

##### Returns:
*    Returns the maximum of all elements in the array, ignoring NaN's.

##### Examples:
```julia
using NaNMath as nm
nm.maximum([1., 2., NaN]) # result: 2.0
```
"""
function maximum(x::AbstractArray{T}) where T<:AbstractFloat
    result = convert(eltype(x), NaN)
    for i in x
        if !isnan(i)
            if (isnan(result) || i > result)
                result = i
            end
        end
    end
    return result
end

"""
NaNMath.minimum(A)

##### Args:
* `A`: An array of floating point numbers

##### Returns:
*    Returns the minimum of all elements in the array, ignoring NaN's.

##### Examples:
```julia
using NaNMath as nm
nm.minimum([1., 2., NaN]) # result: 1.0
```
"""
function minimum(x::AbstractArray{T}) where T<:AbstractFloat
    result = convert(eltype(x), NaN)
    for i in x
        if !isnan(i)
            if (isnan(result) || i < result)
                result = i
            end
        end
    end
    return result
end

"""
NaNMath.extrema(A)

##### Args:
* `A`: An array of floating point numbers

##### Returns:
*    Returns the minimum and maximum of all elements in the array, ignoring NaN's.

##### Examples:
```julia
using NaNMath as nm
nm.extrema([1., 2., NaN]) # result: 1.0, 2.0
```
"""
function extrema(x::AbstractArray{T}) where T<:AbstractFloat
    resultmin, resultmax = convert(eltype(x), NaN), convert(eltype(x), NaN)
    for i in x
        if !isnan(i)
            if (isnan(resultmin) || i < resultmin)
                resultmin = i
            end
            if (isnan(resultmax) || i > resultmax)
                resultmax = i
            end
        end
    end
    return resultmin, resultmax
end

"""
NaNMath.mean(A)

##### Args:
* `A`: An array of floating point numbers

##### Returns:
*    Returns the arithmetic mean of all elements in the array, ignoring NaN's.

##### Examples:
```julia
using NaNMath as nm
nm.mean([1., 2., NaN]) # result: 1.5
```
"""
function mean(x::AbstractArray{T}) where T<:AbstractFloat
    return mean_count(x)[1]
end

"""
Returns a tuple of the arithmetic mean of all elements in the array, ignoring NaN's,
and the number of non-NaN values in the array.
"""
function mean_count(x::AbstractArray{T}) where T<:AbstractFloat
    sum = convert(eltype(x), NaN)
    count = 0
    for i in x
        if !isnan(i)
            if isnan(sum)
                sum = i
                count = 1
            else
                sum += i
                count += 1
            end
        end
    end
    result = sum / count
    return (result, count)
end

"""
NaNMath.var(A)

##### Args:
* `A`: A one dimensional array of floating point numbers

##### Returns:
* Returns the sample variance of a vector A. The algorithm will return
  an estimator of the  generative distribution's variance under the
  assumption that each entry of v is an IID drawn from that generative
  distribution. This computation is  equivalent to calculating \\
  sum((v - mean(v)).^2) / (length(v) - 1). NaN values are ignored.

##### Examples:
```julia
using NaNMath as nm
nm.var([1., 2., NaN]) # result: 0.5
```
"""
function var(x::Vector{T}) where T<:AbstractFloat
    mean_val, n = mean_count(x)
    if !isnan(mean_val)
        sum_square = zero(eltype(x))
        for i in x
            if !isnan(i)
                sum_square += (i - mean_val)^2
            end
        end
        return sum_square / (n - one(eltype(x)))
    else
        return mean_val # NaN or NaN32
    end
end

"""
NaNMath.std(A)

##### Args:
* `A`: A one dimensional array of floating point numbers

##### Returns:
* Returns the standard deviation of a vector A. The algorithm will return
  an estimator of the  generative distribution's standard deviation under the
  assumption that each entry of v is an IID drawn from that generative
  distribution. This computation is  equivalent to calculating \\
  sqrt(sum((v - mean(v)).^2) / (length(v) - 1)). NaN values are ignored.

##### Examples:
```julia
using NaNMath as nm
nm.std([1., 2., NaN]) # result: 0.7071067811865476
```
"""
function std(x::Vector{T}) where T<:AbstractFloat
    return sqrt(var(x))
end

"""
    NaNMath.min(x, y)

Compute the IEEE 754-2008 compliant minimum of `x` and `y`. As of version 0.6 of Julia,
`Base.min(x, y)` will return `NaN` if `x` or `y` is `NaN`. `NanMath.min` favors values over
`NaN`, and will return whichever `x` or `y` is not `NaN` in that case.

## Examples

```julia
julia> NanMath.min(NaN, 0.0)
0.0

julia> NaNMath.min(1, 2)
1
```
"""
min(x::T, y::T) where {T<:AbstractFloat} = ifelse((y < x) | (signbit(y) > signbit(x)),
                                           ifelse(isnan(y), x, y),
                                           ifelse(isnan(x), y, x))

"""
    NaNMath.max(x, y)

Compute the IEEE 754-2008 compliant maximum of `x` and `y`. As of version 0.6 of Julia,
`Base.max(x, y)` will return `NaN` if `x` or `y` is `NaN`. `NaNMath.max` favors values over
`NaN`, and will return whichever `x` or `y` is not `NaN` in that case.

## Examples

```julia
julia> NaNMath.max(NaN, 0.0)
0.0

julia> NaNMath.max(1, 2)
2
```
"""
max(x::T, y::T) where {T<:AbstractFloat} = ifelse((y > x) | (signbit(y) < signbit(x)),
                                           ifelse(isnan(y), x, y),
                                           ifelse(isnan(x), y, x))

min(x::Real, y::Real) = min(promote(x, y)...)
max(x::Real, y::Real) = max(promote(x, y)...)

function min(x::BigFloat, y::BigFloat)
    isnan(x) && return y
    isnan(y) && return x
    return Base.min(x, y)
end

function max(x::BigFloat, y::BigFloat)
    isnan(x) && return y
    isnan(y) && return x
    return Base.max(x, y)
end

# Integers can't represent NaN
min(x::Integer, y::Integer) = Base.min(x, y)
max(x::Integer, y::Integer) = Base.max(x, y)

min(x::Real) = x
max(x::Real) = x

# Multi-arg versions
for f in (:min, :max)
    @eval ($f)(a, b, c, xs...) = Base.afoldl($f, ($f)(($f)(a, b), c), xs...)
end

end


text:
# syntax: proto2
using Compat
using ProtoBuf
import ProtoBuf.meta

mutable struct CreateEncryptionZoneRequestProto <: ProtoType
    src::AbstractString
    keyName::AbstractString
    CreateEncryptionZoneRequestProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct CreateEncryptionZoneRequestProto
const __req_CreateEncryptionZoneRequestProto = Symbol[:src]
meta(t::Type{CreateEncryptionZoneRequestProto}) = meta(t, __req_CreateEncryptionZoneRequestProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES)

mutable struct CreateEncryptionZoneResponseProto <: ProtoType
    CreateEncryptionZoneResponseProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct CreateEncryptionZoneResponseProto

mutable struct ListEncryptionZonesRequestProto <: ProtoType
    id::Int64
    ListEncryptionZonesRequestProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct ListEncryptionZonesRequestProto
const __req_ListEncryptionZonesRequestProto = Symbol[:id]
meta(t::Type{ListEncryptionZonesRequestProto}) = meta(t, __req_ListEncryptionZonesRequestProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES)

mutable struct EncryptionZoneProto <: ProtoType
    id::Int64
    path::AbstractString
    suite::Int32
    cryptoProtocolVersion::Int32
    keyName::AbstractString
    EncryptionZoneProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct EncryptionZoneProto
const __req_EncryptionZoneProto = Symbol[:id,:path,:suite,:cryptoProtocolVersion,:keyName]
meta(t::Type{EncryptionZoneProto}) = meta(t, __req_EncryptionZoneProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES)

mutable struct ListEncryptionZonesResponseProto <: ProtoType
    zones::Base.Vector{EncryptionZoneProto}
    hasMore::Bool
    ListEncryptionZonesResponseProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct ListEncryptionZonesResponseProto
const __req_ListEncryptionZonesResponseProto = Symbol[:hasMore]
meta(t::Type{ListEncryptionZonesResponseProto}) = meta(t, __req_ListEncryptionZonesResponseProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES)

mutable struct GetEZForPathRequestProto <: ProtoType
    src::AbstractString
    GetEZForPathRequestProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct GetEZForPathRequestProto
const __req_GetEZForPathRequestProto = Symbol[:src]
meta(t::Type{GetEZForPathRequestProto}) = meta(t, __req_GetEZForPathRequestProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES, ProtoBuf.DEF_ONEOFS, ProtoBuf.DEF_ONEOF_NAMES, ProtoBuf.DEF_FIELD_TYPES)

mutable struct GetEZForPathResponseProto <: ProtoType
    zone::EncryptionZoneProto
    GetEZForPathResponseProto(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #mutable struct GetEZForPathResponseProto

export CreateEncryptionZoneRequestProto, CreateEncryptionZoneResponseProto, ListEncryptionZonesRequestProto, EncryptionZoneProto, ListEncryptionZonesResponseProto, GetEZForPathRequestProto, GetEZForPathResponseProto


text:
"""
The `FinancialDerivatives` module provides convenient models for pricing financial derivatives in Julia.
"""
module FinancialDerivatives

import Distributions: cdf, Normal
import Statistics: mean

export InterestRateDerivative,
       AmericanOption,
       AsianOption,
       EuropeanOption,
       FXOption

export BlackKarasinski,
       BlackScholes,
       BrennanSchwartz,
       CoxIngersollRoss,
       CoxRossRubinstein,
       GarmanKohlhagen,
       JarrowRudd,
       LeisenReimer,
       LongstaffSchwartz,
       RendlemanBartter,
       Tian,
       Vasicek

export evaluate

abstract type Model end

include("derivatives.jl")

include("models/black_karasinski.jl")
include("models/black_scholes.jl")
include("models/brennan_schwartz.jl")
include("models/cox_ingersoll_ross.jl")
include("models/cox_ross_rubinstein.jl")
include("models/garman_kohlhagen.jl")
include("models/jarrow_rudd.jl")
include("models/leisen_reimer.jl")
include("models/longstaff_schwartz.jl")
include("models/rendleman_bartter.jl")
include("models/tian.jl")
include("models/vasicek.jl")

end # module


text:
module VulkanGen

using StructArrays: StructVector
using LightGraphs
using MLStyle
using DocStringExtensions
using Reexport

@template (FUNCTIONS, METHODS, MACROS) = """
                                         $(DOCSTRING)
                                         $(TYPEDSIGNATURES)
                                         """

@template TYPES = """
                  $(DOCSTRING)
                  $(TYPEDEF)
                  $(TYPEDSIGNATURES)
                  $(TYPEDFIELDS)
                  $(SIGNATURES)
                  """

include("types.jl")

include("spec/VulkanSpec.jl")
@reexport using .VulkanSpec

include("wrapper.jl")

end # module VulkanGen


text:
##
function outrs(cmd::Cmd)
    @> cmd read(String) chomp
end
export outrs
##
function runi(cmd ; kwargs...)
    run(ignorestatus(cmd) ; kwargs...)
end
export runi

function runbool(args... ; kwargs...)
    runi(args... ; kwargs...).exitcode == 0
end
runb = runbool
export runbool
export runb
##


text:
# Example: Maximum-likelihood estimate of the

using SDE
using Lyap
using LinProc

using Winston
using Optim

srand(7)
d = 2 # dimension
M = 500 #number of observations
TT = .1*M #total time span

# normalize values, returns values in interval [0,1]
function norma!(x)
 minx = min(x)
 x = (x - minx) / (max(x)-minx)
 x
end

grid = TT*norma!(sort!(rand(M))) # a grid of random design points in the interval [0, TT]

# diffusion coefficient (assumed to be known in this example)
sigma = [ 0.9   .2; -.2  .7]
A = sigma*sigma'

# starting point
u = [0.,0.]

# true drift function, b(x) = B0*x + beta0 (unknown, used to obtain observations)

B0 = [  -0.2  -1.0;  # true mean reversion matrix
	 0.5  -0.4]
beta0 = [0.,0.]

# many functions in LinProc expect the solution to the
# Lyapunov equation given B and A as argument instead of A

lambda0 = lyap(B0', -A) # solves B*lambda + lambda*B' = -A

# and many functions need the distances between the design points

dt = diff(grid)

# simulate exact Ornstein--Uhlenbeck process with parameter B0, beta0, A (lambda0)
X =  linexact(u, B0, beta0, lambda0, dt)

# as comparison: loglikelihood of true B
llB0 =  linll(X, B0, beta0, lambda0, dt)
println("True mean reversion matrix B0")
print(B0)
println("Likelihood of B0 ", round(llB0,3))

# looks like this
println("Plotting observations")
plot(X[1,:], X[2,:])

# objective function for maximum likelihood

function objective(Y)

	assert(length(Y) == d*d) # Y is a vector of length d*d parametrizing all stable matrices B
	B = LinProc.stable(Y, d, 0.02) # obtain stable matrix with eigenvalues with real part < 0.02 corresponding to numbers in Y
	lambda = lyap(B', -A)	# lambda depends on B

	ob =  #minimize likelihood
	try
		-linll(X, B, beta0, lambda, dt) # negative discrete observations loglikelihood
	catch y #catch numerical singularies
		if isa(y, Base.PosDefException)
			println("Skip numerical indefinite matrix")
			1.0E16 # move away!
		elseif isa(y, Base.Singular)
			println("Skip numerical singular matrix")
			1.0E16	# move away!
		else
		 throw(y)
		end

	end
	ob
end

# find maximum likelihood estimate for B
println("Maximize likelihood...")
O = optimize(objective, ones(4))
println(O)
B = round(LinProc.stable(O.minimum,2,0.02),3)
llmax = -O.f_minimum
print("\nEstimated mean reversion matrix B (log-likelihood ",round(llmax,3),")\n",round(B,3), "\nTrue B0 (log-likelihood ", round(llB0,3),")\n",B0 )


text:
function ant(puzzle::Dict{Complex,Int}, at::Complex, bursts::Int, update::Function)::Int
	d::Complex = -1
	count::Int = 0
	for _ in 1:bursts
		status = get(puzzle, at, 0)
		if status == 0
			d *= im
		elseif status == 2
			d *= -im
		elseif status == 3
			d *= im * im
		end
		puzzle[at] = status = update(status)
		count += status == 2
		at += d
	end
	count
end

part1(status::Int)::Int = status == 0 ? 2 : 0
part2(status::Int)::Int = (status + 1) % 4

function main()
	lines = readlines()
	w = length(lines)
	h = length(lines[1])
	grid = Dict{Complex,Int}()
	for (i, line) in enumerate(lines)
		for (j, c) in enumerate(line)
			c == '#' && setindex!(grid, 2, i + j*im)
		end
	end

	mid = complex(1+div(w, 2), 1+div(h, 2))
	println(ant(copy(grid), mid, 10000, part1))
	println(ant(grid, mid, 10000000, part2))
end

main()


text:
export TaylorParam
export taylorcircuit, taylorsolve, circuit_ends, circuit_intermediate
export v,lc
#export get_param_type, circuit_final

const C(m, x, opn, t, c) = norm(x)*(opn*t*c)^(m)/factorial(m)
const D(m, x, opn, t, c) = norm(x)*(opn*t*c)^(m-1)*t/factorial(m)
"""

    TaylorParam(k::Int, t::L, H::Matrix, x::Array{CPType,1})

    TaylorParam(k::Int,t::L,prob::QuLDEProblem{uType, tType, isinplace, F, P, T})

Assigns values to parameters required for Taylor series based Hamiltonian simulation.

    * k : sets order of Taylor series expansion
    * t : time of evolution
    * H : Hamiltonian
    * x : intial vector
    * prob : wrapper for Linear differential equation problems (contains H, x, b)

"""
struct TaylorParam{CPType, UType, L, HM}
    k::Int # Taylor expansion upto k
    t::L
    H::HM
    l::Int
    rs::Int
    C_tilda::CPType
    D_tilda::CPType
    N::CPType
    function TaylorParam(k::Int, t::L, H::Matrix, x::Array{CPType,1}) where {L,CPType}
        opn = opnorm(H)
        u = isunitary(H/opn)
        C_tilda = 0
        if u
            c = 1
            rs = log2i(k+1)
            l = 0
        else
            c = 2
            rs = k
            l = 2
        end

        for i in 0:k
            C_tilda = C_tilda + C(i, x, opn,t,c)
        end
        N = C_tilda
        C_tilda = sqrt(C_tilda)
        new{CPType, u, L, Array{CPType,2}}(k, t, H, l, rs, C_tilda, zero(C_tilda), N)
    end

    function TaylorParam(k::Int,t::L,prob::QuLDEProblem{uType, tType, isinplace, F, P, T}) where {L,uType, tType, isinplace, F, P, T}
        CPType = eltype(prob.A)
        opn = opnorm(prob.A)
        u = isunitary(prob.A/opn)
        D_tilda = 0
        C_tilda = 0
        if u
            c = 1
            rs = log2i(k+1)
            l = 0
        else
            c = 2
            rs = k
            l = 2
        end
        for i in 1:k
            D_tilda = D_tilda + D(i, prob.b, opn,t,c)
        end
        N = D_tilda
        D_tilda = sqrt(D_tilda)
        if !(T)
            for i in 0:k
                C_tilda = C_tilda + C(i, prob.u0, opn, t,c)
            end
            C_tilda = sqrt(C_tilda)
            N = sqrt(C_tilda^2 + D_tilda^2)
        end
        new{CPType, u, L, Array{CPType,2}}(k, t, prob.A, l, rs, C_tilda, D_tilda, N)
    end
end

"""
    calc_vs1(blk::TaylorParam, x::Vector{CPType}, opn::Real) ->  Matrix{CPType}

Calculates VS1 block for Taylor circuit.
"""
function calc_vs1(blk::TaylorParam, x::Vector{CPType}, opn::Real) where CPType
    k = blk.k
    rs = blk.rs
    t = blk.t
    C_tilda = blk.C_tilda
    VS1 = rand(CPType,2^rs,2^rs)
    @inbounds VS1[:,1] = zero(VS1[:,1])
    if rs == k
        @inbounds for j in 0:k
            VS1[(2^k - 2^(k-j) + 1),1] = sqrt(C(j, x, opn, t,2))/C_tilda
        end
    else
        @inbounds for j in 0:k
            VS1[j+1,1] = sqrt(C(j, x, opn, t,1))/C_tilda
        end
    end
    VS1 = -1*qr(VS1).Q
    return VS1
end

"""
    calc_vs2(blk::TaylorParam, x::Vector{CPType}, opn::Real) ->  Matrix{CPType}

    opn: operator norm of the input matrix.

Calculates VS2 block for Taylor circuit.
"""
function calc_vs2(blk::TaylorParam, x::Vector{CPType}, opn::Real) where CPType
    k = blk.k
    rs = blk.rs
    t = blk.t
    D_tilda = blk.D_tilda
    VS2 = rand(CPType,2^rs,2^rs)
    @inbounds VS2[:,1] = zero(VS2[:,1])
    if rs == k
        @inbounds for j in 0:k-1
            VS2[(2^k - 2^(k-j) + 1),1] = sqrt(D(j+1, x, opn, t, 2))/D_tilda
        end
    else
        @inbounds for j in 0:k - 1
            VS2[j+1,1] = sqrt(D(j+1, x, opn, t, 1))/D_tilda
        end
    end
    VS2 = -1*qr(VS2).Q
    return VS2
end

"""
    unitary_decompose(H::Array{T,2}) -> Array{Array{T,2},1}

Generates a linear compostion of unitary matrices for argument H.
"""
function unitary_decompose(H::Array{T,2}) where T
    if isunitary(H)
        return H
    end
    Mu = H/opnorm(H)
    nbit, = size(H)
    nbit = log2i(nbit)
    B1 = convert(Array{T,2},1/2*(Mu + Mu'))
    B2 = convert(Array{T,2},-im/2*(Mu - Mu'))
    iden = Matrix{T}(I,size(B1))
    F = Array{Array{T,2},1}(undef,4)
    F[1] = B1 + im*sqrt(iden - B1*B1)
    F[2] = B1 - im*sqrt(iden - B1*B1)
    F[3] = im*B2 - sqrt(iden - B2*B2)
    F[4] = im*B2 + sqrt(iden - B2*B2)
    return F

end

"""
    calc_vt(::Type{CPType}) -> Matrix{CPType}

Generates VT block for non-unitary Taylor circuit.
"""
function calc_vt(::Type{CPType} = ComplexF32) where CPType
    VT = rand(CPType,4,4)
    VT[:,1] = 0.5*ones(4)
    VT = -1*qr(VT).Q
    return VT
end

"""
    v(n::Int,c::Int, T::Int, V::AbstractMatrix) -> ChainBlock{n}

Builds T input block.
n : total number of qubits
V : T input matrix
c : starting qubit
"""
v(n::Int,c::Int, T::Int, V::AbstractMatrix) = concentrate(n, matblock(V), (c + 1:T + c...,))

"""
    v(n::Int,c::Int, j::Tuple, T::Int, V::AbstractMatrix) -> ControlBlock{n}

Builds a T input, j - control block.
n : total number of qubits
V : T input matrix
c : starting qubit
j : control bits tuple
"""
v(n::Int,c::Int, j::Tuple, T::Int, V::AbstractMatrix) = control(n, j,(1 + c:T + c...,)=>matblock(V))
lc(n::Int,c::Int, i::Int, k::Int,l::Int, V::AbstractMatrix) = concentrate(n, matblock(V), (k+c+1+(i-1)*l:k+1+c+i*l-1...,))

"""
    circuit_ends(n::Int, blk::TaylorParam{CPType, true}, VS1::AbstractMatrix, VS2::AbstractMatrix)

Generates the part of circuit that computes and decomputes the superposition of the ancilla bits, in unitary H `quldecircuit`
"""
circuit_ends(n::Int, blk::TaylorParam{CPType, true}, VS1::AbstractMatrix, VS2::AbstractMatrix) where CPType = chain(n, v(n, 1,(-1,), blk.rs, VS1),v(n, 1, (1,), blk.rs, VS2))

"""
    circuit_ends(n::Int, blk::TaylorParam{CPType, true}, VS1::AbstractMatrix)

Generates the part of circuit that computes and decomputes the superposition of the ancilla bits, in unitary H `taylorcircuit`
"""
circuit_ends(n::Int, blk::TaylorParam{CPType, true}, VS1::AbstractMatrix) where CPType = chain(n, v(n, 0, blk.rs, VS1))

"""
    circuit_ends(n::Int, blk::TaylorParam{CPType, false}, VS1::AbstractMatrix, VS2::AbstractMatrix, VT::AbstractMatrix)

Generates the part of circuit that computes and decomputes the superposition of the ancilla bits, in non-unitary H `quldecircuit`
"""
function circuit_ends(n::Int, blk::TaylorParam{CPType, false}, VS1::AbstractMatrix, VS2::AbstractMatrix, VT::AbstractMatrix) where CPType
    cir = chain(n, v(n, 1, (-1,), blk.rs, VS1),v(n, 1, (1,),blk.rs, VS2))
    for i in 1:blk.k
        push!(cir, lc(n,1,i,blk.k,blk.l,VT))
    end
    return cir
end

"""
    circuit_ends(n::Int, blk::TaylorParam{CPType, false}, VS1::AbstractMatrix, VT::AbstractMatrix)

Generates the part of circuit that computes and decomputes the superposition of the ancilla bits, in non-unitary H `taylorcircuit`
"""
function circuit_ends(n::Int, blk::TaylorParam{CPType, false}, VS1::AbstractMatrix, VT::AbstractMatrix) where CPType
    cir = chain(n, v(n, 0, blk.rs,VS1))
    for i in 1:blk.k
        push!(cir, lc(n,0,i,blk.k,blk.l,VT))
    end
    return cir
end

"""
    circuit_intermediate(n::Int, c::Int, blk::TaylorParam{CPType, true})

Generates the intermediate part of the circuit for unitary H.
"""
function circuit_intermediate(n::Int, c::Int, blk::TaylorParam{CPType, true}) where CPType
    H = blk.H/opnorm(blk.H)
    k = blk.k
    l = blk.l
    rs = blk.rs
    cir = chain(n)
    nbit = n - rs - c
    a = Array{Int32,1}(undef, rs)
    U = Matrix{CPType}(I, 1<<nbit,1<<nbit)

    for i in 0:k
        digits!(a,i,base = 2)
        G = matblock(U)
        push!(cir,control(n, (-1*collect(1+c:rs+c).*((-1*ones(Int, rs)).^a)...,), (rs+1+c:n...,)=>G))
        U = H*U
    end
    return cir
end

"""
    circuit_intermediate(n::Int, c::Int, blk::TaylorParam{CPType, false})

Generates the intermediate part of the circuit for non-unitary H.
"""
function circuit_intermediate(n::Int, c::Int, blk::TaylorParam{CPType, false}) where CPType
    H = blk.H/opnorm(blk.H)
    k = blk.k
    l = blk.l
    rs = blk.rs
    cir = chain(n)
    nbit = n - k*(l+1) - c
    F = unitary_decompose(H)
    a = Array{Int64,1}(undef, l)
    for i in 1:k
        for j in 0:2^l-1
            digits!(a,j,base = 2)
            push!(cir, control(n, (i + c, -1*collect(k+1+c+(i-1)*l:k+1+c+i*l-1).*((-1*ones(Int, l)).^a)...,), (n - nbit + 1 : n...,)=>matblock(F[j+1])))
        end
    end
    return cir
end

"""
    taylorcircuit(n::Int, blk::TaylorParam, VS1::Matrix) ->  ChainBlock{n}

Generates circuit for a unitary H input.
"""
function taylorcircuit(n::Int, blk::TaylorParam, VS1::Matrix)
    circinit = circuit_ends(n,blk,VS1)
    circmid = circuit_intermediate(n,0,blk)
    circfin =circuit_ends(n,blk,VS1')
    return chain(circinit, circmid, circfin)
end

"""
    taylorcircuit(n::Int, blk::TaylorParam, VS1::Matrix, VT::Matrix) ->->  ChainBlock{n}

Generates circuit for a non-unitary H input.
"""
function taylorcircuit(n::Int, blk::TaylorParam, VS1::Matrix, VT::Matrix)
    circinit = circuit_ends(n,blk,VS1,VT)
    circmid = circuit_intermediate(n,0,blk)
    circfin =circuit_ends(n,blk,VS1',VT')
    return chain(circinit, circmid, circfin)
end

"""
    taylorsolve(H::Array{CPType,2}, x::Vector{CPType}, k::Int, t::Real) -> ArrayReg, CPType

Simulates a Hamiltonian using the Taylor truncation method. Returns the state register and inverse probability of finding it.
"""
function taylorsolve(H::Array{CPType,2}, x::Vector{CPType}, k::Int, t::Real) where CPType
    opn = opnorm(H)
    nbit = log2i(length(x))
    blk = TaylorParam(k,t,H,x)
    VS1 = calc_vs1(blk,x,opn)
    rs = blk.rs
    k = blk.k
    l = blk.l
    if rs != k
        n = rs + nbit
        inreg = join(ArrayReg(x/norm(x)), zero_state(CPType,rs))
        cir = taylorcircuit(n, blk, VS1)
    else
        n = k*(1 + l) + nbit
        VT = calc_vt(CPType)
        inreg = join(ArrayReg(x/norm(x)), zero_state(CPType, k*l), zero_state(CPType, k))
        cir = taylorcircuit(n, blk, VS1, VT)
    end
    r = apply!(inreg,cir) |> focus!(1:n - nbit...,) |> select!(0)
    return r, blk.N
end


text:
# Tables.jl interface
Tables.istable(::Type{<:JSONWorksheet}) = true
Tables.isrowtable(::Type{<:JSONWorksheet}) = true
Tables.rowaccess(::Type{<:JSONWorksheet}) = true
Tables.rows(jws::JSONWorksheet) = jws.data

Tables.columns(jws::JSONWorksheet) = map(p -> jws[:, p], Tables.columnnames(jws))
Tables.columnnames(jws::JSONWorksheet) = jws.pointer
Tables.getcolumn(jws::JSONWorksheet, i::Int) = jws[:, i]
function Tables.getcolumn(jws::JSONWorksheet, p::Symbol)
    Tables.getcolumn(jws, JSONPointer.Pointer(string(p)))
end
Tables.getcolumn(jws::JSONWorksheet, p::Pointer) = jws[:, p]

Tables.materializer(jws::JSONWorksheet) = jws.data

function Tables.matrix(jws::JSONWorksheet, transpose::Bool=false)
    matrix = jws[:, :]
    if transpose
        return permutedims(matrix)
    elseif matrix isa AbstractVector
        # always return a matrix, for type stability
        return reshape(matrix, :, 1)
    else
        return matrix
    end
end


text:
# Block-based deque
#######################################
#
#  DequeBlock
#
#######################################

mutable struct DequeBlock{T}
    data::Vector{T}  # only data[front:back] is valid
    capa::Int
    front::Int
    back::Int
    prev::DequeBlock{T}  # ref to previous block
    next::DequeBlock{T}  # ref to next block

    function DequeBlock{T}(capa::Int, front::Int) where T
        data = Vector{T}(undef, capa)
        blk = new{T}(data, capa, front, front-1)
        blk.prev = blk
        blk.next = blk
        return blk
    end
end

# block at the rear of the chain, elements towards the front
rear_deque_block(ty::Type{T}, n::Int) where {T} = DequeBlock{T}(n, 1)

# block at the head of the train, elements towards the back
head_deque_block(ty::Type{T}, n::Int) where {T} = DequeBlock{T}(n, n+1)

capacity(blk::DequeBlock) = blk.capa
Base.length(blk::DequeBlock) = blk.back - blk.front + 1
Base.isempty(blk::DequeBlock) = blk.back < blk.front
ishead(blk::DequeBlock) = blk.prev === blk
isrear(blk::DequeBlock) =  blk.next === blk

# reset the block to empty, and position

function reset!(blk::DequeBlock{T}, front::Int) where T
    blk.front = front
    blk.back = front - 1
    blk.prev = blk
    blk.next = blk
end

function Base.show(io::IO, blk::DequeBlock)  # avoids recursion into prev and next
    x = blk.data[blk.front:blk.back]
    print(io, "$(typeof(blk))(capa = $(blk.capa), front = $(blk.front), back = $(blk.back)): $x")
end

#######################################
#
#  Deque
#
#######################################

const DEFAULT_DEQUEUE_BLOCKSIZE = 1024

"""
    Deque{T}
    Deque{T}(blksize::Int) where T

Constructs `Deque` object for elements of type `T`.

Parameters
----------

`T::Type` Deque element data type.

`blksize::Int` Deque block size (in bytes). Default = 1024.
"""
mutable struct Deque{T}
    nblocks::Int
    blksize::Int
    len::Int
    head::DequeBlock{T}
    rear::DequeBlock{T}

    function Deque{T}(blksize::Int) where T
        head = rear = rear_deque_block(T, blksize)
        new{T}(1, blksize, 0, head, rear)
    end

    Deque{T}() where {T} = Deque{T}(DEFAULT_DEQUEUE_BLOCKSIZE)
end

"""
    isempty(d::Deque)

Verifies if deque `d` is empty.
"""
Base.isempty(d::Deque) = d.len == 0

"""
    length(d::Deque)

Returns the number of elements in deque `d`.
"""
Base.length(d::Deque) = d.len
num_blocks(d::Deque) = d.nblocks
Base.eltype(::Type{Deque{T}}) where T = T

"""
    first(d::Deque)

Returns the first element of the deque `d`.
"""
function Base.first(d::Deque)
    isempty(d) && throw(ArgumentError("Deque must be non-empty"))
    blk = d.head
    return blk.data[blk.front]
end

"""
    last(d::Deque)

Returns the last element of the deque `d`.
"""
function Base.last(d::Deque)
    isempty(d) && throw(ArgumentError("Deque must be non-empty"))
    blk = d.rear
    return blk.data[blk.back]
end

# Iteration

struct DequeIterator{T}
    d::Deque{T}
end

Base.last(di::DequeIterator) = last(di.d)

function Base.iterate(di::DequeIterator{T}, (cb, i) = (di.d.head, di.d.head.front)) where T
    i > cb.back && return nothing
    x = cb.data[i]

    i += 1
    if i > cb.back && !isrear(cb)
        cb = cb.next
        i = 1
    end

    return (x, (cb, i))
end

# Backwards deque iteration

function Base.iterate(di::Iterators.Reverse{<:Deque}, (cb, i) = (di.itr.rear, di.itr.rear.back))
    i < cb.front && return nothing
    x = cb.data[i]

    i -= 1
    # If we're past the beginning of a block, go to the previous one
    if i < cb.front && !ishead(cb)
        cb = cb.prev
        i = cb.back
    end

    return (x, (cb, i))
end

Base.iterate(d::Deque{T}, s...) where {T} = iterate(DequeIterator{T}(d), s...)

Base.length(di::DequeIterator{T}) where {T} = di.d.len

Base.collect(d::Deque{T}) where {T} = T[x for x in d]

# Showing

function Base.show(io::IO, d::Deque)
    print(io, "Deque [$(collect(d))]")
end

function Base.dump(io::IO, d::Deque)
    println(io, "Deque (length = $(d.len), nblocks = $(d.nblocks))")
    cb::DequeBlock = d.head
    i = 1
    while true
        print(io, "block $i [$(cb.front):$(cb.back)] ==> ")
        for j = cb.front : cb.back
            print(io, cb.data[j])
            print(io, ' ')
        end
        println(io)

        cb_next::DequeBlock = cb.next
        if cb !== cb_next
            cb = cb_next
            i += 1
        else
            break
        end
    end
end

# Manipulation

"""
    empty!(d::Deque{T}) where T

Reset the deque `d`.
"""
function Base.empty!(d::Deque{T}) where T
    # release all blocks except the head
    if d.nblocks > 1
        cb::DequeBlock{T} = d.rear
        while cb != d.head
            empty!(cb.data)
            cb = cb.prev
        end
    end

    # clean the head block (but retain the block itself)
    reset!(d.head, 1)

    # reset queue fields
    d.nblocks = 1
    d.len = 0
    d.rear = d.head
    return d
end

"""
    push!(d::Deque{T}, x) where T

Add an element to the back of deque `d`.
"""
function Base.push!(d::Deque{T}, x) where T
    rear = d.rear

    if isempty(rear)
        rear.front = 1
        rear.back = 0
    end

    if rear.back < rear.capa
        @inbounds rear.data[rear.back += 1] = convert(T, x)
    else
        new_rear = rear_deque_block(T, d.blksize)
        new_rear.back = 1
        new_rear.data[1] = convert(T, x)
        new_rear.prev = rear
        d.rear = rear.next = new_rear
        d.nblocks += 1
    end
    d.len += 1
    return d
end

"""
    pushfirst!(d::Deque{T}, x) where T

Add an element to the front of deque `d`.
"""
function Base.pushfirst!(d::Deque{T}, x) where T
    head = d.head

    if isempty(head)
        n = head.capa
        head.front = n + 1
        head.back = n
    end

    if head.front > 1
        @inbounds head.data[head.front -= 1] = convert(T, x)
    else
        n::Int = d.blksize
        new_head = head_deque_block(T, n)
        new_head.front = n
        new_head.data[n] = convert(T, x)
        new_head.next = head
        d.head = head.prev = new_head
        d.nblocks += 1
    end
    d.len += 1
    return d
end

"""
    pop!(d::Deque{T}) where T

Remove the element at the back of deque `d`.
"""
function Base.pop!(d::Deque{T}) where T
    isempty(d) && throw(ArgumentError("Deque must be non-empty"))
    rear = d.rear
    @assert rear.back >= rear.front

    @inbounds x = rear.data[rear.back]
    rear.back -= 1
    if rear.back < rear.front
        if d.nblocks > 1
            # release and detach the rear block
            empty!(rear.data)
            d.rear = rear.prev::DequeBlock{T}
            d.rear.next = d.rear
            d.nblocks -= 1
        end
    end
    d.len -= 1
    return x
end

"""
    popfirst!(d::Deque{T}) where T

Remove the element at the front of deque `d`.
"""
function Base.popfirst!(d::Deque{T}) where T
    isempty(d) && throw(ArgumentError("Deque must be non-empty"))
    head = d.head
    @assert head.back >= head.front

    @inbounds x = head.data[head.front]
    head.front += 1
    if head.back < head.front
        if d.nblocks > 1
            # release and detach the head block
            empty!(head.data)
            d.head = head.next::DequeBlock{T}
            d.head.prev = d.head
            d.nblocks -= 1
        end
    end
    d.len -= 1
    return x
end

const _deque_hashseed = UInt === UInt64 ? 0x950aa17a3246be82 : 0x4f26f881
function Base.hash(x::Deque, h::UInt)
    h += _deque_hashseed
    for (i, x) in enumerate(x)
        h += i * hash(x)
    end
    return h
end

"""
    ==(x::Deque, y::Deque)

Verify if the deques `x` and `y` are equal in terms of their contents.
"""
function Base.:(==)(x::Deque, y::Deque)
    length(x) != length(y) && return false
    for (i, j) in zip(x, y)
        i == j || return false
    end
    return true
end


text:
function Φ_galerk_driven_cond(Φ::Array{ComplexF64}, V::Array{Float64},
    t::Float64, Δt::Float64, λ::Float64, r::ComplexF64, g::Float64,
    k::Array{Float64,1}, Δx::Float64, T::Float64,K::ComplexF64)
   ξ = sqrt(T * Δt / (2 * Δx)) * (sampler(size(Φ))+1im*sampler(size(Φ)))
   d = length(size(Φ))
   R = CartesianIndices(Φ)
   out = Array{Complex{Float64}}(undef, size(Φ))
   @. out = (Φ -  Δt * (g * abs(Φ)^2 + r + λ * V) * Φ + ξ)
   out = fft(out)
   for I in R
       out[I] = 1.0 / (1.0 + Δt * sum([K*k[I[i]]^2 for i = 1:d])) * out[I]
   end
   out = ifft(out)
   @. Φ = out
end

max_stars_repo_name: phyjonas/ColloidFluctuatingField


text:
module RouteHelpers

using Genie
using Genie.Renderers.Json, Genie.Renderers.Html

using Revise
using Dates
using RemoteREPL

import Stipple, Stipple.Pages

const logfile = "log/$(Genie.config.app_env)-$(Dates.today()).log";

function errors(defaultroute)
  route("$defaultroute/errors") do
    (:errors => Revise.queue_errors) |> json
  end

  nothing
end

function dir(defaultroute)
  route("$defaultroute/dir") do
    pth = params(:path, pwd())

    isdir(pth) || return (:error => "$pth is not a dir") |> json

    result = []
    for p in readdir(pth)
      push!(result, (isdir(joinpath(pth, p)) ? :dir => p : :file => p))
    end

    (:dir => result) |> json
  end

  nothing
end

function edit(defaultroute)
  route("$defaultroute/edit") do
    pth = params(:path, pwd())

    isfile(pth) || return (:error => "$pth is not a file") |> json

    io = open(pth, "r")
    content = read(io, String)
    close(io)

    (:content => content) |> json
  end

  nothing
end

function save(defaultroute)
  route("$defaultroute/save", method=POST) do
    pth = params(:path, pwd())

    isfile(pth) || return (:error => "$pth is not a file") |> json

    isempty(params(:payload, "")) && return (:error => "empty payload") |> json

    (:save => open(pth, "w") do f
      write(f, params(:payload))
    end)

    (:status => :OK) |> json
  end

  nothing
end

function exec(defaultroute)
  routes("$defaultroute/exec", method=[GET, POST]) do
    Core.eval(Main, Meta.parse(params(:cmd))) |> html
  end

  nothing
end

function id(defaultroute)
  route("$defaultroute/id") do
    (:id => Main.UserApp) |> json
  end

  nothing
end

function log(defaultroute)
  route("$defaultroute/log") do
    isfile(logfile) || return (:error => "no log file found") |> json

    io = open(logfile, "r")
    content = read(io, String)
    close(io)

    (:content => content) |> json
  end

  nothing
end

function exit(defaultroute)
  route("$defaultroute/exit") do
    Base.exit()

    (:status => :OK) |> json
  end

  nothing
end

function up(defaultroute)
  route("$defaultroute/up") do
    up()

    (:status => :OK) |> json
  end

  nothing
end

function down(defaultroute)
  route("$defaultroute/down") do
    Genie.Server.down!()

    (:status => :OK) |> json
  end

  nothing
end

function pages(defaultroute)
  route("$defaultroute/pages") do
    (:pages => [Dict(
      :route => Dict(:method => p.route.method, :path => p.route.path),
      :view => p.view |> string,
      :model => Dict(:name => p.model,
        :fields => [fn for fn in fieldnames(p.model)],
        :types => [ft for ft in fieldtypes(p.model)]),
      :layout => p.layout |> string,
      :deps => modeldeps(p.model |> Base.invokelatest)
    ) for p in Stipple.Pages.pages()]) |> json
  end

  nothing
end

function modeldeps(m::M) where {M<:Stipple.ReactiveModel}
  Stipple.deps(m)

  scripts = String[]
  styles = String[]

  channelname = params(:CHANNEL__, "")

  if ! isempty(channelname)
    push!(scripts, "/$channelname.js")
    routename = "get_$(channelname)js" |> Symbol

    if ! Genie.Router.isroute(routename)
      Genie.Router.route("/$channelname.js", named = routename) do
        "window.CHANNEL = '$channelname';" |> Genie.Renderers.Js.js
      end
    end
  end

  for r in routes(reversed = false)
    ! isempty(channelname) && endswith(r.path, "$channelname.js") && continue # don't add the channel script again

    if endswith(r.path, ".js")
      push!(scripts, r.path)
    elseif endswith(r.path, ".css")
      push!(styles, r.path)
    end
  end

  Dict(:scripts => scripts, :styles => styles)
end

function startrepl(defaultroute)
  route("$defaultroute/startrepl") do
    port = params(:port, )
    port = isa(port, String) ? tryparse(Int, port) : port
    port = isnothing(port) ? rand(50_000:60_000) : port

    @async serve_repl(port)

    Dict(
      :status => :OK,
      :port => port
    ) |> json
  end

  nothing
end

end # module


text:
#Test if tf listner works correctly
using RobotOS

include("tf/tf_test.jl")
# include("tf2_ros/tf2_ros_test.jl")