text:
# NOTE: (0,0) is the top-left !!!

to_pixels(m::AbsoluteLength) = m.value / 0.254

const _cbar_width = 5mm
const defaultbox = BoundingBox(0mm, 0mm, 0mm, 0mm)

left(bbox::BoundingBox) = bbox.x0[1]
top(bbox::BoundingBox) = bbox.x0[2]
right(bbox::BoundingBox) = left(bbox) + width(bbox)
bottom(bbox::BoundingBox) = top(bbox) + height(bbox)
Base.size(bbox::BoundingBox) = (width(bbox), height(bbox))

# Base.:*{T,N}(m1::Length{T,N}, m2::Length{T,N}) = Length{T,N}(m1.value * m2.value)
ispositive(m::Measure) = m.value > 0

# union together bounding boxes
function Base.:+(bb1::BoundingBox, bb2::BoundingBox)
    # empty boxes don't change the union
    ispositive(width(bb1))  || return bb2
    ispositive(height(bb1)) || return bb2
    ispositive(width(bb2))  || return bb1
    ispositive(height(bb2)) || return bb1

    l = min(left(bb1), left(bb2))
    t = min(top(bb1), top(bb2))
    r = max(right(bb1), right(bb2))
    b = max(bottom(bb1), bottom(bb2))
    BoundingBox(l, t, r-l, b-t)
end

# this creates a bounding box in the parent's scope, where the child bounding box
# is relative to the parent
function crop(parent::BoundingBox, child::BoundingBox)
    l = left(parent) + left(child)
    t = top(parent) + top(child)
    w = width(child)
    h = height(child)
    BoundingBox(l, t, w, h)
end

# convert x,y coordinates from absolute coords to percentages...
# returns x_pct, y_pct
function xy_mm_to_pcts(x::AbsoluteLength, y::AbsoluteLength, figw, figh, flipy = true)
    xmm, ymm = x.value, y.value
    if flipy
        ymm = figh.value - ymm  # flip y when origin in bottom-left
    end
    xmm / figw.value, ymm / figh.value
end

# convert a bounding box from absolute coords to percentages...
# returns an array of percentages of figure size: [left, bottom, width, height]
function bbox_to_pcts(bb::BoundingBox, figw, figh, flipy = true)
    mms = Float64[f(bb).value for f in (left,bottom,width,height)]
    if flipy
        mms[2] = figh.value - mms[2]  # flip y when origin in bottom-left
    end
    mms ./ Float64[figw.value, figh.value, figw.value, figh.value]
end

function Base.show(io::IO, bbox::BoundingBox)
    print(io, "BBox{l,t,r,b,w,h = $(left(bbox)),$(top(bbox)), $(right(bbox)),$(bottom(bbox)), $(width(bbox)),$(height(bbox))}")
end

# -----------------------------------------------------------

# points combined by x/y, pct, and length
mutable struct MixedMeasures
    xy::Float64
    pct::Float64
    len::AbsoluteLength
end

function resolve_mixed(mix::MixedMeasures, sp::Subplot, letter::Symbol)
    xy = mix.xy
    pct = mix.pct
    if mix.len != 0mm
        f = (letter == :x ? width : height)
        totlen = f(plotarea(sp))
        pct += mix.len / totlen
    end
    if pct != 0
        amin, amax = axis_limits(sp, letter)
        xy += pct * (amax-amin)
    end
    xy
end

# -----------------------------------------------------------
# AbstractLayout

Base.show(io::IO, layout::AbstractLayout) = print(io, "$(typeof(layout))$(size(layout))")

make_measure_hor(n::Number) = n * w
make_measure_hor(m::Measure) = m

make_measure_vert(n::Number) = n * h
make_measure_vert(m::Measure) = m

"""
    bbox(x, y, w, h [,originargs...])
    bbox(layout)

Create a bounding box for plotting
"""
function bbox(x, y, w, h, oarg1::Symbol, originargs::Symbol...)
    oargs = vcat(oarg1, originargs...)
    orighor = :left
    origver = :top
    for oarg in oargs
        if oarg == :center
            orighor = origver = oarg
        elseif oarg in (:left, :right, :hcenter)
            orighor = oarg
        elseif oarg in (:top, :bottom, :vcenter)
            origver = oarg
        else
            @warn("Unused origin arg in bbox construction: $oarg")
        end
    end
    bbox(x, y, w, h; h_anchor = orighor, v_anchor = origver)
end

# create a new bbox
function bbox(x, y, width, height; h_anchor = :left, v_anchor = :top)
    x = make_measure_hor(x)
    y = make_measure_vert(y)
    width = make_measure_hor(width)
    height = make_measure_vert(height)
    left = if h_anchor == :left
        x
    elseif h_anchor in (:center, :hcenter)
        0.5w - 0.5width + x
    else
        1w - x - width
    end
    top = if v_anchor == :top
        y
    elseif v_anchor in (:center, :vcenter)
        0.5h - 0.5height + y
    else
        1h - y - height
    end
    BoundingBox(left, top, width, height)
end

# this is the available area for drawing everything in this layout... as percentages of total canvas
bbox(layout::AbstractLayout) = layout.bbox
bbox!(layout::AbstractLayout, bb::BoundingBox) = (layout.bbox = bb)

# layouts are recursive, tree-like structures, and most will have a parent field
Base.parent(layout::AbstractLayout) = layout.parent
parent_bbox(layout::AbstractLayout) = bbox(parent(layout))

# padding_w(layout::AbstractLayout) = left_padding(layout) + right_padding(layout)
# padding_h(layout::AbstractLayout) = bottom_padding(layout) + top_padding(layout)
# padding(layout::AbstractLayout) = (padding_w(layout), padding_h(layout))

update_position!(layout::AbstractLayout) = nothing
update_child_bboxes!(layout::AbstractLayout, minimum_perimeter = [0mm,0mm,0mm,0mm]) = nothing

left(layout::AbstractLayout) = left(bbox(layout))
top(layout::AbstractLayout) = top(bbox(layout))
right(layout::AbstractLayout) = right(bbox(layout))
bottom(layout::AbstractLayout) = bottom(bbox(layout))
width(layout::AbstractLayout) = width(bbox(layout))
height(layout::AbstractLayout) = height(bbox(layout))

# pass these through to the bbox methods if there's no plotarea
plotarea(layout::AbstractLayout) = bbox(layout)
plotarea!(layout::AbstractLayout, bb::BoundingBox) = bbox!(layout, bb)

attr(layout::AbstractLayout, k::Symbol) = layout.attr[k]
attr(layout::AbstractLayout, k::Symbol, v) = get(layout.attr, k, v)
attr!(layout::AbstractLayout, v, k::Symbol) = (layout.attr[k] = v)
hasattr(layout::AbstractLayout, k::Symbol) = haskey(layout.attr, k)

leftpad(layout::AbstractLayout)   = 0mm
toppad(layout::AbstractLayout)    = 0mm
rightpad(layout::AbstractLayout)  = 0mm
bottompad(layout::AbstractLayout) = 0mm

# -----------------------------------------------------------
# RootLayout

# this is the parent of the top-level layout
struct RootLayout <: AbstractLayout end

Base.show(io::IO, layout::RootLayout) = Base.show_default(io, layout)
Base.parent(::RootLayout) = nothing
parent_bbox(::RootLayout) = defaultbox
bbox(::RootLayout) = defaultbox

# -----------------------------------------------------------
# EmptyLayout

# contains blank space
mutable struct EmptyLayout <: AbstractLayout
    parent::AbstractLayout
    bbox::BoundingBox
    attr::KW  # store label, width, and height for initialization
    # label  # this is the label that the subplot will take (since we create a layout before initialization)
end
EmptyLayout(parent = RootLayout(); kw...) = EmptyLayout(parent, defaultbox, KW(kw))

Base.size(layout::EmptyLayout) = (0,0)
Base.length(layout::EmptyLayout) = 0
Base.getindex(layout::EmptyLayout, r::Int, c::Int) = nothing

_update_min_padding!(layout::EmptyLayout) = nothing

# -----------------------------------------------------------
# GridLayout

# nested, gridded layout with optional size percentages
mutable struct GridLayout <: AbstractLayout
    parent::AbstractLayout
    minpad::Tuple # leftpad, toppad, rightpad, bottompad
    bbox::BoundingBox
    grid::Matrix{AbstractLayout} # Nested layouts. Each position is a AbstractLayout, which allows for arbitrary recursion
    widths::Vector{Measure}
    heights::Vector{Measure}
    attr::KW
end

"""
    grid(args...; kw...)

Create a grid layout for subplots. `args` specify the dimensions, e.g.
`grid(3,2, widths = (0.6,0.4))` creates a grid with three rows and two
columns of different width.
"""
grid(args...; kw...) = GridLayout(args...; kw...)

function GridLayout(dims...;
                    parent = RootLayout(),
                    widths = zeros(dims[2]),
                    heights = zeros(dims[1]),
                    kw...)
    grid = Matrix{AbstractLayout}(undef, dims...)
    layout = GridLayout(
        parent,
        (20mm, 5mm, 2mm, 10mm),
        defaultbox,
        grid,
        Measure[w*pct for w in widths],
        Measure[h*pct for h in heights],
        # convert(Vector{Float64}, widths),
        # convert(Vector{Float64}, heights),
        KW(kw))
    for i in eachindex(grid)
        grid[i] = EmptyLayout(layout)
    end
    layout
end

Base.size(layout::GridLayout) = size(layout.grid)
Base.length(layout::GridLayout) = length(layout.grid)
Base.getindex(layout::GridLayout, r::Int, c::Int) = layout.grid[r,c]
function Base.setindex!(layout::GridLayout, v, r::Int, c::Int)
    layout.grid[r,c] = v
end

leftpad(layout::GridLayout)   = layout.minpad[1]
toppad(layout::GridLayout)    = layout.minpad[2]
rightpad(layout::GridLayout)  = layout.minpad[3]
bottompad(layout::GridLayout) = layout.minpad[4]

# here's how this works... first we recursively "update the minimum padding" (which
# means to calculate the minimum size needed from the edge of the subplot to plot area)
# for the whole layout tree.  then we can compute the "padding borders" of this
# layout as the biggest padding of the children on the perimeter.  then we need to
# recursively pass those borders back down the tree, one side at a time, but ONLY
# to those perimeter children.

# leftpad, toppad, rightpad, bottompad
function _update_min_padding!(layout::GridLayout)
    map(_update_min_padding!, layout.grid)
    layout.minpad = (
        maximum(map(leftpad,   layout.grid[:,1])),
        maximum(map(toppad,    layout.grid[1,:])),
        maximum(map(rightpad,  layout.grid[:,end])),
        maximum(map(bottompad, layout.grid[end,:]))
    )
end

function update_position!(layout::GridLayout)
    map(update_position!, layout.grid)
end

# some lengths are fixed... we have to split up the free space among the list v
function recompute_lengths(v)
    # dump(v)
    tot = 0pct
    cnt = 0
    for vi in v
        if vi == 0pct
            cnt += 1
        else
            tot += vi
        end
    end
    leftover = 1.0pct - tot
    if cnt > 1 && leftover.value <= 0
        error("Not enough length left over in layout!  v = $v, cnt = $cnt, leftover = $leftover")
    end

    # now fill in the blanks
    Measure[(vi == 0pct ? leftover / cnt : vi) for vi in v]
end

# recursively compute the bounding boxes for the layout and plotarea (relative to canvas!)
function update_child_bboxes!(layout::GridLayout, minimum_perimeter = [0mm,0mm,0mm,0mm])
    nr, nc = size(layout)
    # # create a matrix for each minimum padding direction
    # _update_min_padding!(layout)
    minpad_left   = map(leftpad,   layout.grid)
    minpad_top    = map(toppad,    layout.grid)
    minpad_right  = map(rightpad,  layout.grid)
    minpad_bottom = map(bottompad, layout.grid)

    # get the max horizontal (left and right) padding over columns,
    # and max vertical (bottom and top) padding over rows
    # TODO: add extra padding here
    pad_left   = maximum(minpad_left,   dims = 1)
    pad_top    = maximum(minpad_top,    dims = 2)
    pad_right  = maximum(minpad_right,  dims = 1)
    pad_bottom = maximum(minpad_bottom, dims = 2)

    # make sure the perimeter match the parent
    pad_left[1]     = max(pad_left[1], minimum_perimeter[1])
    pad_top[1]      = max(pad_top[1], minimum_perimeter[2])
    pad_right[end]  = max(pad_right[end], minimum_perimeter[3])
    pad_bottom[end] = max(pad_bottom[end], minimum_perimeter[4])

    # scale this up to the total padding in each direction
    total_pad_horizontal = sum(pad_left + pad_right)
    total_pad_vertical   = sum(pad_top + pad_bottom)

    # now we can compute the total plot area in each direction
    total_plotarea_horizontal = width(layout)  - total_pad_horizontal
    total_plotarea_vertical   = height(layout) - total_pad_vertical

    # recompute widths/heights
    layout.widths = recompute_lengths(layout.widths)
    layout.heights = recompute_lengths(layout.heights)

    # normalize widths/heights so they sum to 1
    # denom_w = sum(layout.widths)
    # denom_h = sum(layout.heights)

    # we have all the data we need... lets compute the plot areas and set the bounding boxes
    for r=1:nr, c=1:nc
        child = layout[r,c]

        # get the top-left corner of this child... the first one is top-left of the parent (i.e. layout)
        child_left = (c == 1 ? left(layout.bbox) : right(layout[r, c-1].bbox))
        child_top  = (r == 1 ? top(layout.bbox) : bottom(layout[r-1, c].bbox))

        # compute plot area
        plotarea_left   = child_left + pad_left[c]
        plotarea_top    = child_top + pad_top[r]
        plotarea_width  = total_plotarea_horizontal * layout.widths[c]
        plotarea_height = total_plotarea_vertical * layout.heights[r]
        plotarea!(child, BoundingBox(plotarea_left, plotarea_top, plotarea_width, plotarea_height))

        # compute child bbox
        child_width  = pad_left[c] + plotarea_width + pad_right[c]
        child_height = pad_top[r] + plotarea_height + pad_bottom[r]
        bbox!(child, BoundingBox(child_left, child_top, child_width, child_height))

        # this is the minimum perimeter as decided by this child's parent, so that
        # all children on this border have the same value
        min_child_perimeter = [
            c == 1  ? layout.minpad[1] : pad_left[c],
            r == 1  ? layout.minpad[2] : pad_top[r],
            c == nc ? layout.minpad[3] : pad_right[c],
            r == nr ? layout.minpad[4] : pad_bottom[r]
        ]

        # recursively update the child's children
        update_child_bboxes!(child, min_child_perimeter)
    end
end

# for each inset (floating) subplot, resolve the relative position
# to absolute canvas coordinates, relative to the parent's plotarea
function update_inset_bboxes!(plt::Plot)
    for sp in plt.inset_subplots
        p_area = Measures.resolve(plotarea(sp.parent), sp[:relative_bbox])
        plotarea!(sp, p_area)

        bbox!(sp, bbox(
            left(p_area) - leftpad(sp),
            top(p_area) - toppad(sp),
            width(p_area) + leftpad(sp) + rightpad(sp),
            height(p_area) + toppad(sp) + bottompad(sp)
        ))
    end
end

# ----------------------------------------------------------------------

calc_num_subplots(layout::AbstractLayout) = get(layout.attr, :blank, false) ? 0 : 1
function calc_num_subplots(layout::GridLayout)
    tot = 0
    for l in layout.grid
        tot += calc_num_subplots(l)
    end
    tot
end

function compute_gridsize(numplts::Int, nr::Int, nc::Int)
    # figure out how many rows/columns we need
    if nr < 1
        if nc < 1
            nr = round(Int, sqrt(numplts))
            nc = ceil(Int, numplts / nr)
        else
            nr = ceil(Int, numplts / nc)
        end
    else
        nc = ceil(Int, numplts / nr)
    end
    nr, nc
end

# ----------------------------------------------------------------------
# constructors

# pass the layout arg through
function layout_args(plotattributes::AKW)
    layout_args(plotattributes[:layout])
end

function layout_args(plotattributes::AKW, n_override::Integer)
    layout, n = layout_args(n_override, get(plotattributes, :layout, n_override))
    if n != n_override
        error("When doing layout, n ($n) != n_override ($(n_override)).  You're probably trying to force existing plots into a layout that doesn't fit them.")
    end
    layout, n
end

function layout_args(n::Integer)
    nr, nc = compute_gridsize(n, -1, -1)
    GridLayout(nr, nc), n
end

function layout_args(sztup::NTuple{2, Integer})
    nr, nc = sztup
    GridLayout(nr, nc), nr*nc
end

layout_args(n_override::Integer, n::Integer) = layout_args(n)
layout_args(n, sztup::NTuple{2, Integer}) = layout_args(sztup)

function layout_args(n, sztup::Tuple{Colon, Integer})
    nc = sztup[2]
    nr = ceil(Int, n / nc)
    GridLayout(nr, nc), n
end

function layout_args(n, sztup::Tuple{Integer, Colon})
    nr = sztup[1]
    nc = ceil(Int, n / nr)
    GridLayout(nr, nc), n
end

function layout_args(sztup::NTuple{3, Integer})
    n, nr, nc = sztup
    nr, nc = compute_gridsize(n, nr, nc)
    GridLayout(nr, nc), n
end

# compute number of subplots
function layout_args(layout::GridLayout)
    # recursively get the size of the grid
    n = calc_num_subplots(layout)
    layout, n
end

layout_args(n_override::Integer, layout::GridLayout) = layout_args(layout)

layout_args(huh) = error("unhandled layout type $(typeof(huh)): $huh")

# ----------------------------------------------------------------------

function build_layout(args...)
    layout, n = layout_args(args...)
    build_layout(layout, n)
end

# # just a single subplot
# function build_layout(sp::Subplot, n::Integer)
#     sp, Subplot[sp], SubplotMap(gensym() => sp)
# end

# n is the number of subplots... build a grid and initialize the inner subplots recursively
function build_layout(layout::GridLayout, n::Integer)
    nr, nc = size(layout)
    subplots = Subplot[]
    spmap = SubplotMap()
    i = 0
    for r=1:nr, c=1:nc
        l = layout[r,c]
        if isa(l, EmptyLayout) && !get(l.attr, :blank, false)
            sp = Subplot(backend(), parent=layout)
            layout[r,c] = sp
            push!(subplots, sp)
            spmap[attr(l,:label,gensym())] = sp
            if get(l.attr, :width, :auto) != :auto
                layout.widths[c] = attr(l,:width)
            end
            if get(l.attr, :height, :auto) != :auto
                layout.heights[r] = attr(l,:height)
            end
            i += 1
        elseif isa(l, GridLayout)
            # sub-grid
            if get(l.attr, :width, :auto) != :auto
                layout.widths[c] = attr(l,:width)
            end
            if get(l.attr, :height, :auto) != :auto
                layout.heights[r] = attr(l,:height)
            end
            l, sps, m = build_layout(l, n-i)
            append!(subplots, sps)
            merge!(spmap, m)
            i += length(sps)
        elseif isa(l, Subplot)
            error("Subplot exists. Cannot re-use existing layout.  Please make a new one.")
        end
        i >= n && break  # only add n subplots
    end

    layout, subplots, spmap
end

# build a layout from a list of existing Plot objects
# TODO... much of the logic overlaps with the method above... can we merge?
function build_layout(layout::GridLayout, numsp::Integer, plts::AVec{Plot})
    nr, nc = size(layout)
    subplots = Subplot[]
    spmap = SubplotMap()
    i = 0
    for r=1:nr, c=1:nc
        l = layout[r,c]
        if isa(l, EmptyLayout) && !get(l.attr, :blank, false)
            plt = popfirst!(plts)  # grab the first plot out of the list
            layout[r,c] = plt.layout
            append!(subplots, plt.subplots)
            merge!(spmap, plt.spmap)
            if get(l.attr, :width, :auto) != :auto
                layout.widths[c] = attr(l,:width)
            end
            if get(l.attr, :height, :auto) != :auto
                layout.heights[r] = attr(l,:height)
            end
            i += length(plt.subplots)
        elseif isa(l, GridLayout)
            # sub-grid
            if get(l.attr, :width, :auto) != :auto
                layout.widths[c] = attr(l,:width)
            end
            if get(l.attr, :height, :auto) != :auto
                layout.heights[r] = attr(l,:height)
            end
            l, sps, m = build_layout(l, numsp-i, plts)
            append!(subplots, sps)
            merge!(spmap, m)
            i += length(sps)
        end
        i >= numsp && break  # only add n subplots
    end
    layout, subplots, spmap
end

# ----------------------------------------------------------------------
# @layout macro

function add_layout_pct!(kw::AKW, v::Expr, idx::Integer, nidx::Integer)
    # dump(v)
    # something like {0.2w}?
    if v.head == :call && v.args[1] == :*
        num = v.args[2]
        if length(v.args) == 3 && isa(num, Number)
            units = v.args[3]
            if units == :h
                return kw[:h] = num*pct
            elseif units == :w
                return kw[:w] = num*pct
            elseif units in (:pct, :px, :mm, :cm, :inch)
                idx == 1 && (kw[:w] = v)
                (idx == 2 || nidx == 1) && (kw[:h] = v)
                # return kw[idx == 1 ? :w : :h] = v
            end
        end
    end
    error("Couldn't match layout curly (idx=$idx): $v")
end

function add_layout_pct!(kw::AKW, v::Number, idx::Integer)
    # kw[idx == 1 ? :w : :h] = v*pct
    idx == 1 && (kw[:w] = v*pct)
    (idx == 2 || nidx == 1) && (kw[:h] = v*pct)
end

isrow(v) = isa(v, Expr) && v.head in (:hcat,:row)
iscol(v) = isa(v, Expr) && v.head == :vcat
rowsize(v) = isrow(v) ? length(v.args) : 1

function create_grid(expr::Expr)
    if iscol(expr)
        create_grid_vcat(expr)
    elseif isrow(expr)
        :(let cell = GridLayout(1, $(length(expr.args)))
            $([:(cell[1,$i] = $(create_grid(v))) for (i,v) in enumerate(expr.args)]...)
            cell
        end)

    elseif expr.head == :curly
        create_grid_curly(expr)
    else
        # if it's something else, just return that (might be an existing layout?)
        esc(expr)
    end
end

function create_grid_vcat(expr::Expr)
    rowsizes = map(rowsize, expr.args)
    rmin, rmax = extrema(rowsizes)
    if rmin > 0 && rmin == rmax
        # we have a grid... build the whole thing
        # note: rmin is the number of columns
        nr = length(expr.args)
        nc = rmin
        body = Expr(:block)
        for r=1:nr
            arg = expr.args[r]
            if isrow(arg)
                for (c,item) in enumerate(arg.args)
                    push!(body.args, :(cell[$r,$c] = $(create_grid(item))))
                end
            else
                push!(body.args, :(cell[$r,1] = $(create_grid(arg))))
            end
        end
        :(let cell = GridLayout($nr, $nc)
            $body
            cell
        end)
    else
        # otherwise just build one row at a time
        :(let cell = GridLayout($(length(expr.args)), 1)
            $([:(cell[$i,1] = $(create_grid(v))) for (i,v) in enumerate(expr.args)]...)
            cell
        end)
    end
end

function create_grid_curly(expr::Expr)
    kw = KW()
    for (i,arg) in enumerate(expr.args[2:end])
        add_layout_pct!(kw, arg, i, length(expr.args)-1)
    end
    s = expr.args[1]
    if isa(s, Expr) && s.head == :call && s.args[1] == :grid
        create_grid(:(grid($(s.args[2:end]...), width = $(get(kw, :w, QuoteNode(:auto))), height = $(get(kw, :h, QuoteNode(:auto))))))
    elseif isa(s, Symbol)
        :(EmptyLayout(label = $(QuoteNode(s)), width = $(get(kw, :w, QuoteNode(:auto))), height = $(get(kw, :h, QuoteNode(:auto)))))
    else
        error("Unknown use of curly brackets: $expr")
    end
end

function create_grid(s::Symbol)
    :(EmptyLayout(label = $(QuoteNode(s)), blank = $(s == :_)))
end

macro layout(mat::Expr)
    create_grid(mat)
end

# -------------------------------------------------------------------------

# make all reference the same axis extrema/values.
# merge subplot lists.
function link_axes!(axes::Axis...)
    a1 = axes[1]
    for i=2:length(axes)
        a2 = axes[i]
        expand_extrema!(a1, ignorenan_extrema(a2))
        for k in (:extrema, :discrete_values, :continuous_values, :discrete_map)
            a2[k] = a1[k]
        end

        # make a2's subplot list refer to a1's and add any missing values
        sps2 = a2.sps
        for sp in sps2
            sp in a1.sps || push!(a1.sps, sp)
        end
        a2.sps = a1.sps
    end
end

# figure out which subplots to link
function link_subplots(a::AbstractArray{AbstractLayout}, axissym::Symbol)
    subplots = []
    for l in a
        if isa(l, Subplot)
            push!(subplots, l)
        elseif isa(l, GridLayout) && size(l) == (1,1)
            push!(subplots, l[1,1])
        end
    end
    subplots
end

# for some vector or matrix of layouts, filter only the Subplots and link those axes
function link_axes!(a::AbstractArray{AbstractLayout}, axissym::Symbol)
    subplots = link_subplots(a, axissym)
    axes = [sp.attr[axissym] for sp in subplots]
    if length(axes) > 0
        link_axes!(axes...)
    end
end

# don't do anything for most layout types
function link_axes!(l::AbstractLayout, link::Symbol)
end

# process a GridLayout, recursively linking axes according to the link symbol
function link_axes!(layout::GridLayout, link::Symbol)
    nr, nc = size(layout)
    if link in (:x, :both)
        for c=1:nc
            link_axes!(layout.grid[:,c], :xaxis)
        end
    end
    if link in (:y, :both)
        for r=1:nr
            link_axes!(layout.grid[r,:], :yaxis)
        end
    end
    if link == :square
        sps = filter(l -> isa(l, Subplot), layout.grid)
        if !isempty(sps)
            base_axis = sps[1][:xaxis]
            for sp in sps
                link_axes!(base_axis, sp[:xaxis])
                link_axes!(base_axis, sp[:yaxis])
            end
        end
    end
    if link == :all
        link_axes!(layout.grid, :xaxis)
        link_axes!(layout.grid, :yaxis)
    end
    for l in layout.grid
        link_axes!(l, link)
    end
end

# -------------------------------------------------------------------------

"Adds a new, empty subplot overlayed on top of `sp`, with a mirrored y-axis and linked x-axis."
function twinx(sp::Subplot)
    plot!(sp.plt,
        inset = (sp[:subplot_index], bbox(0,0,1,1)),
        right_margin = sp[:right_margin],
        left_margin = sp[:left_margin],
        top_margin = sp[:top_margin],
        bottom_margin = sp[:bottom_margin],
    )
    twinsp = sp.plt.subplots[end]
    twinsp[:xaxis][:grid] = false
    twinsp[:yaxis][:grid] = false
    twinsp[:xaxis][:showaxis] = false
    twinsp[:yaxis][:mirror] = true
    twinsp[:background_color_inside] = RGBA{Float64}(0,0,0,0)
    link_axes!(sp[:xaxis], twinsp[:xaxis])
    twinsp
end

twinx(plt::Plot = current()) = twinx(plt[1])


text:
call = FinancialDSL.Core.european_call(:PETR4, 20.0BRL, Date(2020, 5, 2))
put = FinancialDSL.Core.european_put(:PETR4, 20.0BRL, Date(2020, 5, 2))

@testset "BS Model internal API" begin

    not_an_option = FinancialDSL.Core.Unit(FinancialDSL.Core.Stock(:PETR4))

    @test FinancialDSL.Core.Compiler.is_european_call_option(call)
    @test !FinancialDSL.Core.Compiler.is_european_call_option(put)
    @test !FinancialDSL.Core.Compiler.is_european_put_option(call)
    @test FinancialDSL.Core.Compiler.is_european_put_option(put)
    @test !FinancialDSL.Core.Compiler.is_european_call_option(FinancialDSL.Core.Unit(FinancialDSL.Core.Stock(:PETR4)))
    @test !FinancialDSL.Core.Compiler.is_european_put_option(FinancialDSL.Core.Unit(FinancialDSL.Core.Stock(:PETR4)))

    @test FinancialDSL.Core.Compiler.is_european_option(call)
    @test FinancialDSL.Core.Compiler.is_european_option(put)
    @test !FinancialDSL.Core.Compiler.is_european_option(not_an_option)

    @test FinancialDSL.Core.Compiler.EuropeanOptionLegs(call) == FinancialDSL.Core.Compiler.EuropeanOptionLegs(:call, FinancialDSL.Core.Unit(FinancialDSL.Core.Stock(:PETR4)), FinancialDSL.Core.Scale(FinancialDSL.Core.Konst(20.0), FinancialDSL.Core.Unit(FinancialDSL.Core.SpotCurrency(FinancialDSL.Currencies.BRL))))
    @test FinancialDSL.Core.Compiler.EuropeanOptionLegs(put) == FinancialDSL.Core.Compiler.EuropeanOptionLegs(:put, FinancialDSL.Core.Unit(FinancialDSL.Core.Stock(:PETR4)), FinancialDSL.Core.Scale(FinancialDSL.Core.Konst(20.0), FinancialDSL.Core.Unit(FinancialDSL.Core.SpotCurrency(FinancialDSL.Currencies.BRL))))
end

@testset "Pricing European Option" begin

    pricing_date = Date(2018, 5, 29)
    currency_to_curves_map = Dict( "onshore" => Dict( :BRL => :PRE, :USD => :cpUSD, :PETR4 => Symbol("PETR4 DIVIDEND YIELD") ))
    static_model = FinancialDSL.Core.StaticHedgingModel(BRL, currency_to_curves_map)
    black_scholes_model = FinancialDSL.Core.BlackScholesModel(static_model)
    attr = FinancialDSL.Core.ContractAttributes("riskfree_curves" => "onshore", "carry_type" => "none")
    empty_provider = FinancialDSL.MarketData.EmptyMarketDataProvider()

    scenario_fixed = FinancialDSL.Core.FixedScenario()
    scenario_fixed[FinancialDSL.Core.DiscountFactor(Symbol("PETR4 DIVIDEND YIELD"), Date(2020, 5, 2))] = 0.95
    scenario_fixed[FinancialDSL.Core.Stock(:PETR4)] = 25.0BRL
    scenario_fixed[FinancialDSL.Core.DiscountFactor(:PRE, Date(2020, 5, 2))] = 0.8
    scenario_fixed[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:PETR4))] = 0.3

    call_pricer = FinancialDSL.Core.compile_pricer(empty_provider, pricing_date, black_scholes_model, call, attr)
    put_pricer = FinancialDSL.Core.compile_pricer(empty_provider, pricing_date, black_scholes_model, put, attr)

    # pricing
    call_p = FinancialDSL.Core.price(call_pricer, scenario_fixed)
    put_p = FinancialDSL.Core.price(put_pricer, scenario_fixed)

    t = BusinessDays.bdayscount(:Brazil, pricing_date, Date(2020, 5, 2)) / 252
    s = 25.0 * 0.95
    k = 20.0 * 0.8
    σ = 0.3

    @test call_p ≈ FinancialDSL.BS.bscall(s, k, t, σ)
    @test put_p ≈ FinancialDSL.BS.bsput(s, k, t, σ)

    native_call_pricer = FinancialDSL.Core.compile_pricer(empty_provider, pricing_date, black_scholes_model, call, attr, compiler=:native)
    native_put_pricer = FinancialDSL.Core.compile_pricer(empty_provider, pricing_date, black_scholes_model, put, attr, compiler=:native)
    @test call_p ≈ FinancialDSL.Core.price(native_call_pricer, scenario_fixed)
    @test put_p ≈ FinancialDSL.Core.price(native_put_pricer, scenario_fixed)

    # exposures
    call_exposures = FinancialDSL.Core.exposures(FinancialDSL.Core.DeltaGammaApproxExposuresMethod(), call_pricer, scenario_fixed)
    put_exposures = FinancialDSL.Core.exposures(FinancialDSL.Core.DeltaGammaApproxExposuresMethod(), put_pricer, scenario_fixed)

    native_call_exposures = FinancialDSL.Core.exposures(FinancialDSL.Core.DeltaGammaApproxExposuresMethod(), native_call_pricer, scenario_fixed)
    native_put_exposures = FinancialDSL.Core.exposures(FinancialDSL.Core.DeltaGammaApproxExposuresMethod(), native_put_pricer, scenario_fixed)

    # delta
    delta_fwd_diff = call_exposures[FinancialDSL.Core.Stock(:PETR4)]
    delta_bs = 25.0 * 0.95 * FinancialDSL.BS.delta(FinancialDSL.BS.EuropeanCall(), s, k, t, σ)
    @test delta_fwd_diff ≈ delta_bs
    @test delta_fwd_diff ≈ native_call_exposures[FinancialDSL.Core.Stock(:PETR4)]

    # vega
    vega_fwd_diff = call_exposures[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:PETR4))]
    vega_bs = σ * FinancialDSL.BS.vega(s, k, t, σ)
    @test vega_fwd_diff ≈ vega_bs
    @test vega_fwd_diff ≈ native_call_exposures[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:PETR4))]

    # gamma
    gamma_fwd_diff = call_exposures[FinancialDSL.Core.SecondOrderRiskFactor(FinancialDSL.Core.Stock(:PETR4))]
    gamma_bs = 0.5 * FinancialDSL.BS.gamma(s, k, t, σ) * s^2
    @test gamma_fwd_diff ≈ gamma_bs
end

@testset "BS European Call" begin
    k = 0.88
    maturity = Date(2025, 10, 28)
    ticker = :UNDERLYING_STOCK
    pricing_date = Date(2020, 3, 6)
    qty = 339_824

    currency_to_curves_map = Dict( "onshore" => Dict( :BRL => :PRE, :UNDERLYING_STOCK => Symbol("UNDERLYING_STOCK DIVIDEND YIELD") ))
    static_model = FinancialDSL.Core.StaticHedgingModel(BRL, currency_to_curves_map)
    black_scholes_model = FinancialDSL.Core.BlackScholesModel(static_model)
    attr = FinancialDSL.Core.ContractAttributes("riskfree_curves" => "onshore", "carry_type" => "none")
    empty_provider = FinancialDSL.MarketData.EmptyMarketDataProvider()

    contract = FinancialDSL.Core.european_call(ticker, k*BRL, maturity)

    scenario_fixed = FinancialDSL.Core.FixedScenario()
    scenario_fixed[FinancialDSL.Core.DiscountFactor(Symbol("UNDERLYING_STOCK DIVIDEND YIELD"), maturity)] = 1.0
    scenario_fixed[FinancialDSL.Core.Stock(:UNDERLYING_STOCK)] = 2.14BRL
    scenario_fixed[FinancialDSL.Core.DiscountFactor(:PRE, maturity)] = 0.7088448249661002
    scenario_fixed[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:UNDERLYING_STOCK))] = 1.01393597487

    pricer = FinancialDSL.Core.compile_pricer(empty_provider, pricing_date, black_scholes_model, contract, attr)
    p = FinancialDSL.Core.price(pricer, scenario_fixed)
    ex = FinancialDSL.Core.exposures(FinancialDSL.Core.DeltaGammaApproxExposuresMethod(), pricer, scenario_fixed)

    @test p ≈ 1.89492553836
    @test qty * p ≈ 643941.1761486751
    @test qty * ex[FinancialDSL.Core.Stock(:UNDERLYING_STOCK)] ≈ 695852.3823994318
    @test qty * ex[FinancialDSL.Core.SecondOrderRiskFactor(FinancialDSL.Core.Stock(:UNDERLYING_STOCK))] ≈ 13844.789831796134
    @test qty * ex[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:UNDERLYING_STOCK))] ≈ 160294.74401978508
    @test qty * ex[FinancialDSL.Core.DiscountFactor(:PRE, Date(2025, 10, 28))] ≈ -51911.20625075656

    # TODO fix volatility exposure
    @test (qty * ex[FinancialDSL.Core.Volatility(FinancialDSL.Core.Stock(:UNDERLYING_STOCK))] / 1.01393597487) == 158091.58368242823
end

ext: jl


text:
## Add Linear constraints

# add single constraint

function add_constr!(model::Model, inds::IVec, coeffs::FVec, rel::Cchar, rhs::Float64)
    length(inds) == length(coeffs) || error("Inconsistent argument dimensions.")
    ret = @grb_ccall(addconstr, Cint, (
        Ptr{Void},    # model
        Cint,         # numnz
        Ptr{Cint},    # cind
        Ptr{Float64}, # cvals
        Cchar,        # sense
        Float64,      # rhs
        Ptr{UInt8}    # name
        ),
        model, length(inds), inds - 1, coeffs, rel, rhs, C_NULL)
    if ret != 0
        throw(GurobiError(model.env, ret))
    end
    nothing
end

function add_constr!(model::Model, inds::Vector, coeffs::Vector, rel::GChars, rhs::Real)
    add_constr!(model, ivec(inds), fvec(coeffs), cchar(rel), Float64(rhs))
end

function add_constr!(model::Model, coeffs::Vector, rel::GChars, rhs::Real)
    inds = find(coeffs)
    vals = coeffs[inds]
    add_constr!(model, inds, vals, rel, rhs)
end

# add multiple constraints

function add_constrs!(model::Model, cbegins::IVec, inds::IVec, coeffs::FVec, rel::CVec, rhs::FVec)
    m = length(cbegins)
    nnz = length(inds)
    (m == length(rel) == length(rhs) && nnz == length(coeffs)) || error("Inconsistent argument dimensions.")

    if m > 0
        ret = @grb_ccall(addconstrs, Cint, (
            Ptr{Void},    # model
            Cint,         # num constraints
            Cint,         # num non-zeros
            Ptr{Cint},    # cbeg
            Ptr{Cint},    # cind
            Ptr{Float64}, # cval
            Ptr{Cchar},   # sense
            Ptr{Float64}, # rhs
            Ptr{UInt8}    # names
            ),
            model, m, nnz, cbegins - 1, inds - 1, coeffs,
            rel, rhs, C_NULL)

        if ret != 0
            throw(GurobiError(model.env, ret))
        end
    end
    nothing
end

function add_constrs!(model::Model, cbeg::Vector, inds::Vector, coeffs::Vector, rel::GCharOrVec, rhs::Vector)
    add_constrs!(model, ivec(cbeg), ivec(inds), fvec(coeffs), cvecx(rel, length(cbeg)), fvec(rhs))
end

function add_constrs_t!(model::Model, At::SparseMatrixCSC{Float64}, rel::GCharOrVec, b::Vector)
    n, m = size(At)
    (m == length(b) && n == num_vars(model)) || error("Incompatible argument dimensions.")
    add_constrs!(model, At.colptr[1:At.n], At.rowval, At.nzval, rel, b)
end

function add_constrs_t!(model::Model, At::Matrix{Float64}, rel::GCharOrVec, b::Vector)
    n, m = size(At)
    (m == length(b) && n == num_vars(model)) || error("Incompatible argument dimensions.")
    add_constrs_t!(model, sparse(At), rel, b)
end

function add_constrs!(model::Model, A::CoeffMat, rel::GCharOrVec, b::Vector{Float64})
    m, n = size(A)
    (m == length(b) && n == num_vars(model)) || error("Incompatible argument dimensions.")
    add_constrs_t!(model, transpose(A), rel, b)
end

# add single range constraint

function add_rangeconstr!(model::Model, inds::IVec, coeffs::FVec, lb::Float64, ub::Float64)
    ret = @grb_ccall(addrangeconstr, Cint, (
        Ptr{Void},    # model
        Cint,         # numnz
        Ptr{Cint},    # cind
        Ptr{Float64}, # cvals
        Float64,      # lower
        Float64,      # upper
        Ptr{UInt8}    # name
        ),
        model, length(inds), inds - 1, coeffs, lb, ub, C_NULL)
    if ret != 0
        throw(GurobiError(model.env, ret))
    end
    nothing
end

function add_rangeconstr!(model::Model, inds::Vector, coeffs::Vector, lb::Real, ub::Real)
    add_rangeconstr!(model, ivec(inds), fvec(coeffs), Float64(lb), Float64(ub))
end

function add_rangeconstr!(model::Model, coeffs::Vector, lb::Real, ub::Real)
    inds = find(coeffs)
    vals = coeffs[inds]
    add_rangeconstr!(model, inds, vals, lb, ub)
end

# add multiple range constraints

function add_rangeconstrs!(model::Model, cbegins::IVec, inds::IVec, coeffs::FVec, lb::FVec, ub::FVec)
    m = length(cbegins)
    nnz = length(inds)
    (m == length(lb) == length(ub) && nnz == length(coeffs)) || error("Incompatible argument dimensions.")

    if m > 0
        ret = @grb_ccall(addrangeconstrs, Cint, (
            Ptr{Void},    # model
            Cint,         # num constraints
            Cint,         # num non-zeros
            Ptr{Cint},    # cbeg
            Ptr{Cint},    # cind
            Ptr{Float64}, # cval
            Ptr{Float64}, # lower
            Ptr{Float64}, # upper
            Ptr{UInt8}    # names
            ),
            model, m, nnz, cbegins - 1, inds - 1, coeffs, lb, ub, C_NULL)

        if ret != 0
            throw(GurobiError(model.env, ret))
        end
    end
    nothing
end

function add_rangeconstrs!(model::Model, cbeg::Vector, inds::Vector, coeffs::Vector, lb::Vector, ub::Vector)
    add_rangeconstrs!(model, ivec(cbeg), ivec(inds), fvec(coeffs), fvec(lb), fvec(ub))
end

function add_rangeconstrs_t!(model::Model, At::SparseMatrixCSC{Float64}, lb::Vector, ub::Vector)
    add_rangeconstrs!(model, At.colptr[1:At.n], At.rowval, At.nzval, lb, ub)
end

function add_rangeconstrs_t!(model::Model, At::Matrix{Float64}, lb::Vector, ub::Vector)
    add_rangeconstrs_t!(model, sparse(At), lb, ub)
end

function add_rangeconstrs!(model::Model, A::CoeffMat, lb::Vector, ub::Vector)
    m, n = size(A)
    (m == length(lb) == length(ub) && n == num_vars(model)) || error("Incompatible argument dimensions.")

    add_rangeconstrs_t!(model, transpose(A), lb, ub)
end

function get_constrmatrix(model::Model)
    nnz = get_intattr(model, "NumNZs")
    m = num_constrs(model)
    n = num_vars(model)
    numnzP = Array(Cint, 1)
    cbeg = Array(Cint, m+1)
    cind = Array(Cint, nnz)
    cval = Array(Cdouble, nnz)
    ret = @grb_ccall(getconstrs, Cint, (
                     Ptr{Void},
                     Ptr{Cint},
                     Ptr{Cint},
                     Ptr{Cint},
                     Ptr{Cdouble},
                     Cint,
                     Cint
                     ),
                     model, numnzP, cbeg, cind, cval, 0, m)
    if ret != 0
        throw(GurobiError(model.env, ret))
    end
    cbeg[end] = nnz
    I = Array(Int64, nnz)
    J = Array(Int64, nnz)
    V = Array(Float64, nnz)
    for i in 1:length(cbeg)-1
        for j in (cbeg[i]+1):cbeg[i+1]
            I[j] = i
            J[j] = cind[j]+1
            V[j] = cval[j]
        end
    end
    return sparse(I, J, V, m, n)
end

const GRB_SOS_TYPE1 = convert(Cint, 1)
const GRB_SOS_TYPE2 = convert(Cint, 2)

function add_sos!(model::Model, sostype::Symbol, idx::Vector{Int}, weight::Vector{Cdouble})
    ((nelem = length(idx)) == length(weight)) || error("Index and weight vectors of unequal length")
    (sostype == :SOS1) ? (typ = GRB_SOS_TYPE1) : ( (sostype == :SOS2) ? (typ = GRB_SOS_TYPE2) : error("Invalid SOS constraint type") )
    ret = @grb_ccall(addsos, Cint, (
                     Ptr{Void},
                     Cint,
                     Cint,
                     Ptr{Cint},
                     Ptr{Cint},
                     Ptr{Cint},
                     Ptr{Cdouble}
                     ),
                     model, convert(Cint, 1), convert(Cint, nelem), Cint[typ], Cint[0], convert(Vector{Cint}, idx-1), weight)
    if ret != 0
        throw(GurobiError(model.env, ret))
    end
end

del_constrs!{T<:Real}(model::Model, idx::T) = del_constrs!(model, Cint[idx])
del_constrs!{T<:Real}(model::Model, idx::Vector{T}) = del_constrs!(model, convert(Vector{Cint},idx))
function del_constrs!(model::Model, idx::Vector{Cint})
    numdel = length(idx)
    ret = @grb_ccall(delconstrs, Cint, (
                     Ptr{Void},
                     Cint,
                     Ptr{Cint}),
                     model, convert(Cint,numdel), idx.-1)
    if ret != 0
        throw(GurobiError(model.env, ret))
    end
end


text:
my_f(x,y) = 2x + 3y
derivative_of_my_f(x,y) = ForwardDiff.derivative(x->my_f(x,y),x)


text:
using PhotoOrganizer

dry_run = false
rm_src = false
dst_root="/home/hertz/mnt/media/Pictures"

#src_dirs = String[]
#push!(src_dirs, "/run/user/1000/gvfs/mtp:host=%5Busb%3A002%2C009%5D/Samsung SD card/CameraZOOM")
#push!(src_dirs, "/run/user/1000/gvfs/mtp:host=%5Busb%3A002%2C009%5D/Samsung SD card/DCIM/Camera")
#push!(src_dirs, "/home/hertz/Documents/Glen/backups/Google/Takeout/Google Photos")
#push!(src_dirs, "/home/hertz/Documents/Erin/backups/Google/Takeout/Google Photos")
#src_dirs = readlines(`/home/hertz/bin/ls_phone_backup_dirs.jl`)

#src_dirs = ["/home/hertz/Documents.local/Pictures"]
#src_root = "/home/hertz/Documents/backups/S7"
#src_root = "/run/user/1000/gvfs/mtp:host=%5Busb%3A003%2C004%5D"
#src_dirs = [
#	    "$src_root/Card/DCIM/Camera",
#            "$src_root/Phone/DCIM/Camera",
#            "$src_root/Phone/DCIM/PhotoScan",
#            "$src_root/Phone/DCIM/Screenshots",
#            "$src_root/Phone/DCIM/Video Editor",
#            "$src_root/Phone/Movies/Instagram",
#            "$src_root/Phone/Pictures",
#            "$src_root/Phone/Snapchat",
#            "$src_root/Phone/Snapseed",
#            "$src_root/Phone/Studio",
#            "$src_root/Phone/Telegram",
#            "$src_root/Phone/WhatsApp/Media"]
#src_dirs = ["/media/hertz/NIKON D5500/DCIM"]
src_root = "/home/hertz/mnt/syncthing/Pixel4a"
src_dirs = [
	    "$src_root/Glen_DCIM"
        "$src_root/Phone/DCIM/Camera",
        "$src_root/Phone/DCIM/PhotoScan",
        "$src_root/Phone/DCIM/Screenshots",
        "$src_root/Phone/DCIM/Video Editor",
        "$src_root/Phone/Movies/Instagram",
        "$src_root/Phone/Pictures",
        "$src_root/Phone/Snapchat",
        "$src_root/Phone/Snapseed",
        "$src_root/Phone/Studio",
        "$src_root/Phone/Telegram",
        "$src_root/Phone/WhatsApp/Media"]

if length(src_dirs) > 0
    organize_photos(src_dirs, dst_root, rm_src, dry_run)
else
    warn("No directories found to backup")
end


text:
#=
const julia_type_for_c_type = Dict(
  :byte => :Int8,
  :double => :Float64,
  :float => :Float64,
  :int => :Int,
  :bool => :Bool,
  :Point3d => :XYZ,
  :Point2d => :XYZ,
  :Vector3d => :VXYZ,
  :string => :ASCIIString,
  :ObjectId => :Int32,
  :Entity => :Int32,
  :BIMLevel => :Int32,
  :FloorFamily => :Int32,
    ) #Either Int32 or Int64, depending on the architecture

julia_type(ctype, is_array) = is_array ? :(Vector{$(julia_type(ctype, false))}) : julia_type_for_c_type[ctype]
=#
export show_rpc, step_rpc
const show_rpc = Parameter(false)
const step_rpc = Parameter(false)
function initiate_rpc_call(conn, opcode, name)
    if step_rpc()
        print(stderr, "About to call $(name) (opcode $(opcode)) [press ENTER]")
        readline()
    end
    if show_rpc()
        print(stderr, "$(name) (opcode $(opcode))")
    end
end
function complete_rpc_call(conn, opcode, result)
    if show_rpc()
        println(stderr, isnothing(result) ? "-> nothing" : "-> $(result)")
    end
    result
end

function my_symbol(prefix, name)
  Symbol(prefix * replace(string(name), ":" => "_"))
end

function translate(sym)
  Symbol(replace(string(sym), r"[:<>]" => "_", ))
end

#=
We parameterize signatures to support different programming languages, e.g.
Val{:CPP} is for C++ functions, while Val{:CS} is for C# static methods

A basic type is represented with an identically-named symbol, e.g.,
int -> :int
An array type is represented with a tuple, e.g.,
int[] -> (:array, :int)
A parametric type is represented with a tuple, e.g.,
Dict<int, string> -> (:Dict, :int, :string)
=#
#=
# C++
parse_signature(::Val{:CPP}, sig::AbstractString) =
  let func_name(name) = replace(name, ":" => "_"),
      type_name(name) = replace(name, r"[:<>]" => "_"),
      m = match(r"^ *(public|) *(\w+) *([\[\]]*) +((?:\w|:|<|>)+) *\( *(.*) *\)", sig),
      ret = type_name(m.captures[2]),
      array_level = count(c -> c=='[', something(m.captures[3], "")),
      name = m.captures[4],
      params = split(m.captures[5], r" *, *", keepempty=false),
      parse_c_decl(decl) =
        let m = match(r"^ *((?:\w|:|<|>)+) *([\[\]]*) *(\w+)$", decl)
          (type_name(m.captures[1]), count(c -> c=='[', something(m.captures[2], "")), Symbol(m.captures[3]))
        end
    (func_name(name), name, [parse_c_decl(decl) for decl in params], (ret, array_level))
  end

# C#
parse_signature(::Val{:CS}, sig::AbstractString) =
  let m = match(r"^ *(public|) *(\w+) *([\[\]]*) +(\w+) *\( *(.*) *\)", sig),
      ret = m.captures[2],
      array_level = count(c -> c=='[', something(m.captures[3], "")),
      name = m.captures[4],
      params = split(m.captures[5], r" *, *", keepempty=false),
      parse_c_decl(decl) =
        let m = match(r"^ *(\w+) *([\[\]]*) *(\w+)$", decl)
          (m.captures[1], count(c -> c=='[', something(m.captures[2], "")), Symbol(m.captures[3]))
    end
    (name, name, [parse_c_decl(decl) for decl in params], (ret, array_level))
  end

# Python
parse_signature(::Val{:PY}, sig::AbstractString) =
  let m = match(r"^ *def +(\w+) *(\(.*\)) *-> *(.+) *: *", sig),
      name = m.captures[1],
      params = m.captures[2],
      ret = m.captures[3],
      parse_type(t) =
        if t isa Symbol
          (t, 0)
        elseif t.head === :ref
          if t.args[1] === :List
            (s, l) = parse_type(t.args[2])
            (s, l + 1)
          elseif t.args[1] === :Tuple
            (Symbol(:T, join(t.args[2:end], "_"), :T), 0)
          else
            error("Uknown expression $(t)")
          end
        else
          error("Uknown expression $(t)")
        end,
      parse_params(ast) =
        let parse_param(p) = (parse_type(p.args[3])..., p.args[2])
          if ast.head === :tuple
            map(parse_param, ast.args)
          elseif ast.head === :call
            [parse_param(ast)]
          else
            error("Uknown expression $(t)")
          end
        end
    (name, name, parse_params(Meta.parse(params)), parse_type(Meta.parse(ret)))
  end
=#
###########################################################
# New version
# C++
parse_signature(::Val{:CPP}, sig::AbstractString) =
  let func_name(name) = replace(name, ":" => "_"),
      type_name(name) = replace(name, r"[:<>]" => "_"),
      packtype(t, n) = foldr((i,v)->Vector{v}, 1:n, init=Val{Symbol(t)}),
      m = match(r"^ *(public|) *(\w+) *([\[\]]*) +((?:\w|:|<|>)+) *\( *(.*) *\)", sig),
      ret = packtype(type_name(m.captures[2]), count(c -> c=='[', something(m.captures[3], ""))),
      name = m.captures[4],
      params = split(m.captures[5], r" *, *", keepempty=false),
      parse_c_decl(decl) =
        let m = match(r"^ *((?:\w|:|<|>)+) *([\[\]]*) *(\w+)$", decl)
          (packtype(type_name(m.captures[1]), count(c -> c=='[', something(m.captures[2], ""))), Symbol(m.captures[3]))
        end
    (func_name(name), name, [parse_c_decl(decl) for decl in params], ret)
  end

# C#
parse_signature(::Val{:CS}, sig::AbstractString) =
  let packtype(t, n) = foldr((i,v)->Vector{v}, 1:n, init=Val{Symbol(t)}),
      m = match(r"^ *(public|) *(\w+) *([\[\]]*) +(\w+) *\( *(.*) *\)", sig),
      ret = packtype(m.captures[2], count(c -> c=='[', something(m.captures[3], ""))),
      name = m.captures[4],
      params = split(m.captures[5], r" *, *", keepempty=false),
      parse_c_decl(decl) =
        let m = match(r"^ *(\w+) *([\[\]]*) *(\w+)$", decl)
          (packtype(m.captures[1], count(c -> c=='[', something(m.captures[2], ""))), Symbol(m.captures[3]))
    end
    (name, name, [parse_c_decl(decl) for decl in params], ret)
  end

# Python
parse_signature(::Val{:PY}, sig::AbstractString) =
  let m = match(r"^ *def +(\w+) *(\(.*\)) *(-> *(.+) *)?: *", sig),
      name = m.captures[1],
      params = m.captures[2],
      ret = m.captures[4],
      parse_type(t) =
        if t isa Symbol
          Val{t}
        elseif t.head === :ref
          if t.args[1] === :List
            Vector{parse_type(t.args[2])}
          elseif t.args[1] === :Tuple
            Tuple{map(parse_type, t.args[2:end])...}
          else
            error("Unknown expression type $(t) in signature $(sig)")
          end
        else
          error("Unknown expression type $(t) in signature $(sig)")
        end,
      parse_params(ast) =
        let parse_param(p) = (parse_type(p.args[3]), p.args[2])
          if ast isa Symbol
            error("Missing type information in parameter $(ast) in signature $(sig)")
          elseif ast.head === :tuple
            map(parse_param, ast.args)
          elseif ast.head === :call
            [parse_param(ast)]
          else
            error("Uknown kind of parameter $(ast) in signature $(sig)")
          end
        end
    isnothing(ret) && error("Missing return type information in signature $(sig)")
    (name, name, parse_params(Meta.parse(params)), parse_type(Meta.parse(ret)))
  end

#=
A remote function encapsulates the information needed for communicating with remote
applications, including the opcode that represents the remote function and that
is generated by the remote application from the remote_name upon request.
=#

mutable struct RemoteFunction{T} <: Function
  namespace::T
  signature::String
  local_name::String
  remote_name::String
  opcode::Int32
  encoder::Function
  buffer::IOBuffer
end

remote_function(ns::T,
                sig::AbstractString,
                local_name::AbstractString,
                remote_name::AbstractString,
                encoder::Function) where {T} =
  RemoteFunction(
    ns,
    string(sig),
    string(local_name),
    string(remote_name),
    Int32(-1),
    encoder,
    IOBuffer())

request_operation(f::RemoteFunction{T}, conn::IO) where {T} = begin
  encode(f.namespace, Val(:int), conn, 0)
  encode(f.namespace, Val(:string), conn, f.remote_name)
  op = decode(f.namespace, Val(:size), conn)
  if op == -1
    error(f.remote_name * " is not available")
  else
    op
  end
end

interrupt_processing(conn::IO) = write(conn, Int32(-1))

ensure_opcode(f::RemoteFunction, conn) =
  f.opcode == -1 ?
    f.opcode = Int32(request_operation(f, conn)) :
    f.opcode

reset_opcode(f::RemoteFunction) =
  f.opcode = -1

call_remote(f::RemoteFunction, conn, args...) =
  f.encoder(ensure_opcode(f, conn), conn, f.buffer, args...)

(f::RemoteFunction)(conn, args...) = call_remote(f, conn, args...)

#=
The most important part is the lang_rpc function. It parses the string describing the
signature of the remote function, generates a function to encode arguments and
retrieve results and, finally, creates the remote_function object.
=#
#=
remote_function_meta_program(nssym, sig, local_name, remote_name, params, ret) =
  let packtype(t, n) =
        let init = t isa Tuple ? :(Tuple{$(t...)}) : t
          foldr((i,v)->:(Vector{$v}), 1:n, init=:(Val{$(QuoteNode(Symbol(t)))}))
        end,
      namespace = :(Val{$(nssym)}())
    :(remote_function(
           $(namespace),
           $(sig),
           $(local_name),
           $(remote_name),
           (opcode, conn, buf, $([:($(p[3])) for p in params]...)) -> begin
              initiate_rpc_call(conn, opcode, $(remote_name))
              take!(buf) # Reset the buffer just in case there was an encoding error on a previous call
              encode($(namespace), Val(:int), buf, opcode)
              $([:(encode($(namespace),
                          $(packtype(p[1], p[2]))(),
                          buf,
                          $(p[3])))
                 for p in params]...)
              write(conn, take!(buf))
              complete_rpc_call(conn, opcode,
                decode($(namespace),
                       $(packtype(ret[1], ret[2]))(), conn))
            end))
  end
=#
###################################################
# New version
type_constructor(t) =
  t <: Tuple ?
    Expr(:tuple, map(type_constructor, t.types)...) :
    Expr(:call, t)

remote_function_meta_program(nssym, sig, local_name, remote_name, params, ret) =
  let namespace = :(Val{$(nssym)}())
    :(remote_function(
           $(namespace),
           $(sig),
           $(local_name),
           $(remote_name),
           (opcode, conn, buf, $([p[2] for p in params]...)) -> begin
              initiate_rpc_call(conn, opcode, $(remote_name))
              take!(buf) # Reset the buffer just in case there was an encoding error on a previous call
              encode($(namespace), Val(:int), buf, opcode)
              $([:(encode($(namespace),
                          $(type_constructor(p[1])),
                          buf,
                          $(p[2])))
                 for p in params]...)
              write(conn, take!(buf))
              #flush(conn)
              complete_rpc_call(conn, opcode,
                decode($(namespace), $(type_constructor(ret)), conn))
            end))
  end

#=
let lang = :CS,
    str = "public Entity Sphere(Point3d c, double r)",
    (local_name, remote_name, params, ret) = parse_signature(Val(lang), str)
  (Symbol(local_name), remote_function_meta_program(lang, str, local_name, remote_name, params, ret))
 end

let lang = :CS,
   str = "public Entity[][] Spheres(Point3d c, double[][] r)",
   (local_name, remote_name, params, ret) = parse_signature(Val(lang), str)
 (Symbol(local_name), remote_function_meta_program(lang, str, local_name, remote_name, params, ret))
end
=#

lang_rpc(lang, sigs) =
  [let (local_name, remote_name, params, ret) =
      try
        parse_signature(Val(lang.value), str)
      catch
        error("Cannot parse in $(lang) the signature $(str)")
      end
    (Symbol(local_name), remote_function_meta_program(lang, str, local_name, remote_name, params, ret))
   end
   for str in split(sigs, "\n") if str != ""]

#=
Given that remote apps might fail, it might be necessary to reset a connection.
This can be done either by resetting the opcodes of all remote functions or by
simply recreating all of them. This second alternative looks better because it
also allows to access multiple instances of the same remote app (as long as it)
support multiple separate connections.

=#

#=
The idea is that a particular remote application will store all of its functions
in a struct, as follows:

@remote_functions :CS """
  int add(int a, int b)
  int sub(int a, int b)
"""
=#

macro remote_functions(lang, str)
  let remotes = lang_rpc(lang, str)
    Expr(:tuple, [Expr(:(=), remote...) for remote in remotes]...)
  end
end

#=
@macroexpand @remote_functions :CS """
  int add(int a, int b)
  int sub(int a, int b)
"""

@macroexpand @remote_functions :PY """
def get_view()->Tuple[Point3d, Point3d, float]:
"""
=#

# We need to detect errors by recognizing a particular value
# Note that this is parametric on namespace ond type
decode_or_error(ns::Val{NS}, t::Val{T}, c::IO, err) where {NS,T} =
  let v = decode(ns, t, c)
    v == err ? backend_error(ns, c) : v
  end

struct BackendError
  msg::String
  backtrace
end

show(io::IO, e::BackendError) =
  print(io, "Backend Error: $(e.msg)")

# In order to read arrays or process error messages,
# we need a few basic types, namely :size and :string,
# that need to be specified for each backend.

backend_error(ns::Val{NS}, c::IO) where {NS} =
  throw(BackendError(decode(ns, Val(:string), c), backtrace()))

## To present errors in the backends that call back to Julia
exception_backtrace(e) = backtrace()
exception_backtrace(e::BackendError) = e.backtrace

export errormsg
errormsg(e) =
  sprint((io, e) -> showerror(io, e, exception_backtrace(e), backtrace=true), e)

# Encoding and decoding vectors
encode(ns::Val{NS}, t::Vector{T}, c::IO, v) where {NS,T} = begin
  sub = T()
  encode(ns, Val(:size), c, length(v))
  for e in v encode(ns, sub, c, e) end
end
decode(ns::Val{NS}, t::Vector{T}, c::IO) where {NS,T} = begin
  sub = T()
  len = decode(ns, Val(:size), c)
  [decode(ns, sub, c) for i in 1:len]
end

# Encoding and decoding vectors of tuples
encode(ns::Val{NS}, t::Vector{Tuple{T1,T2}}, c::IO, v) where {NS,T1,T2} = begin
  sub1 = T1()
  sub2 = T2()
  encode(ns, Val(:size), c, length(v))
  for (e1, e2) in v
    encode(ns, sub1, c, e1)
    encode(ns, sub2, c, e2)
  end
end
decode(ns::Val{NS}, t::Vector{Tuple{T1,T2}}, c::IO) where {NS,T1,T2} = begin
  sub1 = T1()
  sub2 = T2()
  len = decode(ns, Val(:size), c)
  [(decode(ns, sub1, c), decode(ns, sub2, c)) for i in 1:len]
end

# Some generic conversions for C#, C++, and Python
const SizeIsInt = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(ns::SizeIsInt, t::Val{:size}, c::IO, v) =
  encode(ns, Val(:int), c, v)
decode(ns::SizeIsInt, t::Val{:size}, c::IO) =
  decode_or_error(ns, Val(:int), c, -1)
encode(ns::SizeIsInt, t::Val{:address}, c::IO, v) =
  encode(ns, Val(:long), c, v)
decode(ns::SizeIsInt, t::Val{:address}, c::IO) =
  decode_or_error(ns, Val(:long), c, -1)

const BoolIsByte = Union{Val{:CS},Val{:PY}}
encode(ns::BoolIsByte, t::Val{:bool}, c::IO, v::Bool) =
  encode(ns, Val(:byte), c, v ? 1 : 0)
decode(ns::BoolIsByte, t::Val{:bool}, c::IO) =
  decode_or_error(ns, Val(:byte), c, UInt8(127)) == UInt8(1)

const ByteIsUInt8 = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(::ByteIsUInt8, t::Val{:byte}, c::IO, v) =
  write(c, convert(UInt8, v))
decode(::ByteIsUInt8, t::Val{:byte}, c::IO) =
  convert(UInt8, read(c, UInt8))

# Assuming short is two bytes in C++ and Python
const ShortIsInt16 = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(::ShortIsInt16, t::Val{:short}, c::IO, v) =
  write(c, convert(Int16, v))
decode(::ShortIsInt16, t::Val{:short}, c::IO) =
  convert(Int16, read(c, Int16))

# Assuming int is four bytes in C++ and Python
const IntIsInt32 = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(::IntIsInt32, t::Val{:int}, c::IO, v) =
  write(c, convert(Int32, v))
decode(::IntIsInt32, t::Val{:int}, c::IO) =
  convert(Int32, read(c, Int32))

# Assuming long is eight bytes in C++
const LongIsInt64 = Union{Val{:CS},Val{:CPP}}
encode(::LongIsInt64, t::Val{:long}, c::IO, v) =
  write(c, convert(Int64, v))
decode(::Union{Val{:CS},Val{:CPP}}, t::Val{:long}, c::IO) =
  convert(Int64, read(c, Int64))

# Assuming float is four bytes in C++
const FloatIsFloat32 = Union{Val{:CS},Val{:CPP}}
encode(::FloatIsFloat32, t::Val{:float}, c::IO, v) =
  write(c, convert(Float32, v))
decode(ns::FloatIsFloat32, t::Val{:float}, c::IO) =
  let d = read(c, Float32)
    isnan(d) ? backend_error(ns, c) : convert(Float64, d)
  end

# Assuming float is eight bytes in Python
const FloatIsFloat64 = Union{Val{:PY}}
encode(::FloatIsFloat64, t::Val{:float}, c::IO, v) =
  write(c, convert(Float64, v))
decode(ns::FloatIsFloat64, t::Val{:float}, c::IO) =
  let d = read(c, Float64)
    isnan(d) ? backend_error(ns, c) : convert(Float64, d)
  end

# Assuming double is eight bytes in C++
const DoubleIsFloat64 = Union{Val{:CS},Val{:CPP}}
encode(::DoubleIsFloat64, t::Val{:double}, c::IO, v) =
  write(c, convert(Float64, v))
decode(ns::DoubleIsFloat64, t::Val{:double}, c::IO) =
  let d = read(c, Float64)
    isnan(d) ? backend_error(ns, c) : d
  end

# The binary_stream we use with C++ and Python replicates C# behavior
const StringIsCSString = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(::StringIsCSString, ::Union{Val{:string},Val{:String},Val{:str}}, c::IO, v) = begin
  str = string(v)
  size = length(str)
  array = UInt8[]
  while true
    byte = size & 0x7f
    size >>= 7
    if size > 0
      push!(array, byte | 0x80)
    else
      push!(array, byte)
      break
    end
  end
  write(c, array)
  write(c, str)
end

#=
decode(::Val{:CS}, ::Val{:string}, c::IO) = begin
  loop(size::Int, shift::Int) = begin
    b = convert(Int, read(c, UInt8))
    size = size | ((b & 0x7f) << shift)
    if (b & 0x80) == 0
      String(read(c, size))
    else
      loop(size, shift + 7)
    end
  end
  loop(0, 0)
end
=#
decode(ns::StringIsCSString, ::Union{Val{:string},Val{:String},Val{:str}}, c::IO) = begin
  size::Int = 0
  shift::Int = 0
  while true
    b = convert(Int, read(c, UInt8))
    size = size | ((b & 0x7f) << shift)
    if (b & 0x80) == 0
      str = String(read(c, size))
      str == "This an error!" ?
        backend_error(ns, c) :
        return str
    else
      shift += 7
    end
  end
end

const VoidIsByte = Union{Val{:CS},Val{:PY}}
decode(ns::VoidIsByte, t::Union{Val{:void},Val{:None}}, c::IO) =
  decode_or_error(ns, Val(:byte), c, 0x7f) == 0x00

# Useful CS types
export Guid, Guids
const Guid = UInt128
const Guids = Vector{Guid}

encode(::Val{:CS}, ::Val{:Guid}, c::IO, v) =
  write(c, v % UInt128)
decode(ns::Val{:CS}, ::Val{:Guid}, c::IO) =
  let guid = read(c, UInt128)
    iszero(guid) ? backend_error(ns, c) : guid
  end

# It is frequently necessary to encode/decode an abstract type that is
# implemented os some primitive type
macro encode_decode_as(ns, from, to)
  esc(
    quote
      encode(ns::Val{$ns}, ::$from, c::IO, v) = encode(ns, $to(), c, v)
      decode(ns::Val{$ns}, ::$from, c::IO) = decode(ns, $to(), c)
    end)
end

#=
There are lots of cases where doubles need to be encoded/decoded
(e.g. (points, vectors))
encode_double3(c::IO, v0::Real, v1::Real, v2::Real) = begin
    encode_double(c, v0)
    encode_double(c, v1)
    encode_double(c, v2)
end
encode_float3(c::IO, v0::Real, v1::Real, v2::Real) = begin
    encode_float(c, v0)
    encode_float(c, v1)
    encode_float(c, v2)
end
=#

const SupportsTuples = Union{Val{:CS},Val{:CPP},Val{:PY}}
encode(ns::SupportsTuples, t::Tuple{T1,T2}, c::IO, v) where {T1,T2} =
  begin
    encode(ns, T1(), c, v[1])
    encode(ns, T2(), c, v[2])
  end
decode(ns::SupportsTuples, t::Tuple{T1,T2}, c::IO) where {T1,T2} =
  (decode(ns, T1(), c),
   decode(ns, T2(), c))
encode(ns::SupportsTuples, t::Tuple{T1,T2,T3}, c::IO, v) where {T1,T2,T3} =
  begin
    encode(ns, T1(), c, v[1])
    encode(ns, T2(), c, v[2])
    encode(ns, T3(), c, v[3])
  end
decode(ns::SupportsTuples, t::Tuple{T1,T2,T3}, c::IO) where {T1,T2,T3} =
  (decode(ns, T1(), c),
   decode(ns, T2(), c),
   decode(ns, T3(), c))
encode(ns::SupportsTuples, t::Tuple{T1,T2,T3,T4}, c::IO, v) where {T1,T2,T3,T4} =
  begin
    encode(ns, T1(), c, v[1])
    encode(ns, T2(), c, v[2])
    encode(ns, T3(), c, v[3])
    encode(ns, T4(), c, v[4])
  end
decode(ns::SupportsTuples, t::Tuple{T1,T2,T3,T4}, c::IO) where {T1,T2,T3,T4} =
  (decode(ns, T1(), c),
   decode(ns, T2(), c),
   decode(ns, T3(), c),
   decode(ns, T4(), c))

encode(ns::SupportsTuples, ::Val{:float3}, c::IO, v) =
  encode(ns, (Val(:float),Val(:float),Val(:float)), c, v)
decode(ns::SupportsTuples, ::Val{:float3}, c::IO) =
  decode(ns, (Val(:float),Val(:float),Val(:float)), c)

encode(ns::SupportsTuples, ::Val{:double3}, c::IO, v) =
  encode(ns, (Val(:double),Val(:double),Val(:double)), c, v)
decode(ns::SupportsTuples, ::Val{:double3}, c::IO) =
  decode(ns, (Val(:double),Val(:double),Val(:double)), c)

encode(ns::SupportsTuples, ::Val{:RGB}, c::IO, v) =
  encode(ns, (Val(:float),Val(:float),Val(:float)), c, (red(v), green(v), blue(v)))
decode(ns::SupportsTuples, ::Val{:RGB}, c::IO) =
  RGB(decode(ns, (Val(:float),Val(:float),Val(:float)), c)...)

encode(ns::SupportsTuples, ::Val{:RGBA}, c::IO, v) =
  encode(ns, (Val(:float),Val(:float),Val(:float),Val(:float)), c, (red(v), green(v), blue(v), alpha(v)))
decode(ns::SupportsTuples, ::Val{:RGBA}, c::IO) =
  RGBA(decode(ns, (Val(:float),Val(:float),Val(:float),Val(:float)), c)...)

encode(ns::SupportsTuples, ::Val{:Color}, c::IO, v) =
  encode(ns, (Val(:byte),Val(:byte),Val(:byte),Val(:byte)), c,
         (reinterpret(UInt8, v.alpha),
          reinterpret(UInt8, v.r),
          reinterpret(UInt8, v.g),
          reinterpret(UInt8, v.b)))
decode(ns::SupportsTuples, ::Val{:Color}, c::IO) =
  let a = decode(ns, Val(:byte), c),
      r = decode(ns, Val(:byte), c),
      g = decode(ns, Val(:byte), c),
      b = decode(ns, Val(:byte), c)
    RGBA(r, g, b, a)
  end

const one_year_milliseconds = 366*24*60*60*1000

encode(ns::Val{:CS}, ::Val{:DateTime}, c::IO, v) =
  encode(ns, Val(:long), c, (Dates.datetime2epochms(v) - one_year_milliseconds)*10000) # Julia uses milliseconds since 0000-01-01T00:00:00 while C# uses 100s of nanoseconds since 0001-01-01T00:00:00

decode(ns::Val{:CS}, ::Val{:DateTime}, c::IO) =
  Dates.epochms2datetime(decode(ns, Val(:long), c) ÷ 10000 + one_year_milliseconds)


text:
using CGP
using Logging
using ArgParse
using JSON
using Images
using FileIO
using Base.Threads
using Dates
CGP.Config.init("cfg/image_caption.yaml")

function overprint(str)
    print("\e[2K")
    print("\e[1G")
    print(str)
end

function computeParallel(c::Chromosome, startIndex::Int64, endIndex::Int64, data::Array{Any,2}, images::Dict{Any,Any}, nin::Int64, nout::Int64, accuracy::Base.Threads.Atomic{Float64})::Float64
    @threads for d in startIndex:endIndex

        imagePath = string("data/images/",data[d,1],".jpg")
        img = images[imagePath]
        r = Float64.(red.(img))
        g = Float64.(green.(img))
        b = Float64.(blue.(img))

        outputs = process(c,[r,g,b,data[d,2:nin-1]...])
        if indmax(outputs)==data[d,nin-1]
            atomic_add!(accuracy,1.0)
        end
    end
    accuracy[]
end

function blueScore(c::Chromosome, data::Array{Any,2}, nin::Int64, nout::Int64)

    accuracy =  0.0
    batchSize = 1000
    nsamples = size(data, 1)
    nbatches = Int64(ceil(nsamples/batchSize))

    for batch in 1:nbatches

        startIndex = (batch-1)*batchSize + 1
        endIndex = batch*batchSize<nsamples? batch*batchSize:nsamples

        images = Dict()
        prevImgPath = string("data/images/",data[startIndex,1],".jpg")
        img = load(prevImgPath)
        images[prevImgPath] = img

        println("batch:",batch,"/",nbatches)
        println("loading images batch: $(startIndex):$(endIndex)")
        for sample in startIndex:endIndex
            imgPath= string("data/images/$(data[sample,1]).jpg")
            if imgPath != prevImgPath
                img = load(imgPath)
                prevImgPath=imgPath
                images[imgPath] = img
            end
        end
        println("image Batch Loaded Successfully")
        println("<<< Training >>>")

        now = Dates.now()
        accuracy += computeParallel(c, startIndex, endIndex, data, images, nin, nout, Threads.Atomic{Float64}(0.0))
        println("paralle time: $(Dates.value(Dates.now()-now)/1000)s")

    end

    accuracy /= nsamples
    println()
    accuracy
end

function predict(c::Chromosome, path::String, nin::Int64)

        img = load(path)
        r = Float64.(red.(img))
        g = Float64.(green.(img))
        b = Float64.(blue.(img))
        sequence =  zeros(Float64, nin-3)
        sequence[nin-3]=1
        for i = 1:(nin-3)
            outputs = process(c,[r,g,b,sequence...])
            for j = 1:(nin-4)
                sequence[j]=sequence[j+1]
            end
            sequence[nin-3] = indmax(outputs)
            if indmax(outputs) == 7

                break
            end
        end
        println(sequence)

end

function MSE(c::Chromosome, data::Array{Float64}, nin::Int64, nout::Int64)
    accuracy = 0
    nsamples = size(data, 2)
    for d in 1:nsamples
        outputs = process(c, data[d, 2:nin+1])
        if indmax(outputs) == data[d,nin+1]
            accuracy += 1
        end
    end
    accuracy /= nsamples
    accuracy
end

function get_args()
    s = ArgParseSettings()

    @add_arg_table(
        s,
        "--id", arg_type = String, default = "image_caption",
        "--seed", arg_type = Int, default = 0,
        "--log", arg_type = String, default = "image_caption.log",
        "--ea", arg_type = String, default = "oneplus",
        "--chromosome", arg_type = String, default = "CGPChromo",
        "--data", arg_type = String, required = true,
        "--meta", arg_type = String, required = true,
        "--fitness", arg_type = String, default = "blueScore",
    )

    parse_args(CGP.Config.add_arg_settings!(s))
end

function read_data_descriptor(dMeta::String, dData::String)
    j = JSON.parsefile(dMeta)
    seqLen = j["meta"]["Sequence Length"]
    outputSize = j["meta"]["Vocab Size"]
    wordDict = j["dict"]

    data = readdlm(dData, ' ')
    data[:,2:seqLen+1] = Float64.(data[:,2:seqLen+1])
    #seqLen + r+g+b
    return seqLen + 3, outputSize, data, data, wordDict
end

if ~isinteractive()
    args = get_args()
    CGP.Config.init(Dict([k=>args[k] for k in setdiff(
        keys(args), ["seed", "log", "ea", "chromosome", "data", "meta", "fitness","id"])]...))

    srand(args["seed"])
    Logging.configure(filename=args["log"], level=INFO)

    println("Loading Dataset ...")
    nin, nout, train, test, word_indx = read_data_descriptor(args["meta"],args["data"])
    ea = eval(parse(args["ea"]))
    ctype = eval(parse(args["chromosome"]))
    fitness = eval(parse(args["fitness"]))
    trainFit = x->fitness(x, train, nin, nout)

    println("Training ...")
    maxfit, best = ea(nin, nout, trainFit, seed=args["seed"], id=args["id"], ctype=ctype)
    best_ind = ctype(best, nin, nout)
    test_fit = fitness(best_ind, test, nin, nout)

    Logging.info(@sprintf("T: %d %0.8f %0.8f %d %d %s %s",
                        args["seed"], maxfit, test_fit,
                        sum([n.active for n in best_ind.nodes]),
                        length(best_ind.nodes), args["ea"], args["chromosome"]))
    Logging.info(@sprintf("E%0.6f", -maxfit))

    predict(best_ind,"data/images/2a.jpg",nin)
    #render_genes(best, args; ctype=ctype)
end


text:
startswith("abcd","ab")            #returns true
findfirst("ab", "abcd")            #returns 1:2, indices range where string was found
endswith("abcd","zn")              #returns false
match(r"ab","abcd") != Nothing     #returns true where 1st arg is regex string
for r in eachmatch(r"ab","abab")
	println(r.offset)
end                                #returns 1, then 3 matching the two starting indices where the substring was found

max_stars_repo_name: ethansaxenian/RosettaDecode

max_issues_repo_name: ethansaxenian/RosettaDecode

max_forks_repo_name: ethansaxenian/RosettaDecode


text:
# syntax: proto3
using Compat
using ProtoBuf
import ProtoBuf.meta
import Base: hash, isequal, ==

type FieldMask
    paths::Array{AbstractString,1}
    FieldMask(; kwargs...) = (o=new(); fillunset(o); isempty(kwargs) || ProtoBuf._protobuild(o, kwargs); o)
end #type FieldMask
hash(v::FieldMask) = ProtoBuf.protohash(v)
isequal(v1::FieldMask, v2::FieldMask) = ProtoBuf.protoisequal(v1, v2)
==(v1::FieldMask, v2::FieldMask) = ProtoBuf.protoeq(v1, v2)

export FieldMask


text:
abstract type AngularQuadrature end
abstract type UnstructuredMesh_2D end
abstract type LinearUnstructuredMesh_2D <: UnstructuredMesh_2D end
abstract type QuadraticUnstructuredMesh_2D <: UnstructuredMesh_2D end

max_stars_repo_name: KyleVaughn/MOCNeutronTransport

max_issues_repo_name: KyleVaughn/MOCNeutronTransport

max_forks_repo_name: KyleVaughn/MOCNeutronTransport


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

Statistics.mean(x::NDArray; dims = :) = _mean(x, dims)
@_remap _mean(x::NDArray, ::Colon) mean(x)
@_remap _mean(x::NDArray, dims)    mean(x; axis = 0 .- dims, keepdims = true)

Base.sum(x::NDArray; dims = :) = _sum(x, dims)
@_remap _sum(x::NDArray, ::Colon) sum(x)
@_remap _sum(x::NDArray, dims)    sum(x; axis = 0 .- dims, keepdims = true)


text:
struct ParityError <: Exception
    var
    text
end

"""
    Controls the Buffer for Parity Errors
    $(SIGNATURES)

    ´dc´: decoder, struct for all ephemeris data
    ´words_in_subframes´: buffer of 1500 bits, sliced in 5 subframes á 300 bits, which are again sliced in 10 Words á 30 Bits

    # Details
    # Controls every Word for Parity Errors
"""
function buffer_control(dc::GNSSDecoderState, words_in_subframes)

    parity_buffer = vcat(words_in_subframes[1],
                        words_in_subframes[2],
                        words_in_subframes[3],
                        words_in_subframes[4],
                        words_in_subframes[5])

    dc.data_integrity = true
    for i in 1:length(parity_buffer)
        if i == 1
            if dc.prev_30
                parity_buffer[i][1:24] = .!parity_buffer[i][1:24]
            end
            if parity_check(parity_buffer[i], dc.prev_29, dc.prev_30)

            else
                @warn("Parity Error: Word , ", i)
                dc.data_integrity = false
            end
        else
            dc.prev_29 = parity_buffer[i - 1][29]
            dc.prev_30 = parity_buffer[i - 1][30]

            if dc.prev_30
                parity_buffer[i][1:24] = .!parity_buffer[i][1:24]
            end
            if parity_check(parity_buffer[i], dc.prev_29, dc.prev_30)
            else
                @warn("Parity Error: Word , ", i)
                dc.data_integrity = false
            end
        end
    end

    return dc
end

"""
    Controls Data for Errors (IODC and matching IODEs)
    $(SIGNATURES)

    ´TLM_HOW_Data´: Data of TLM and HOW Words of last read in subframe
    ´subframe_1_data´: Data of Subframe 1
    ´subframe_2_data´: Data of Subframe 2
    ´subframe_3_data´: Data of Subframe 3

    # Details
    #checks if IODE of subframe 2 and 3 matches the 8 LSB of th in Subframe 1 transmitted IODC
"""
function control_data(TLM_HOW_Data::TLM_HOW_Data_Struct,
    subfr_1_data::Subframe_1_Data,
    subfr_2_data::Subframe_2_Data,
    subfr_3_data::Subframe_3_Data)

    status = true
    if subfr_2_data.IODE != subfr_1_data.IODC[3:10] # IODE and the 8 LSB of IODC must match
        @info "new data required, IODC and IODE of Subframe 2 do not match (CEI data set cutover)"
        status = false
    end

    if subfr_3_data.IODE != subfr_1_data.IODC[3:10] # IODE and the 8 LSB of IODC must match
        @info "new data required, IODC and IODE of Subframe 3 do not match (CEI data set cutover)"
        status = false
    end

    return status
end

"""
    Decodes words of subframe 1-3, TLM and HOW of subframe 1-5
    $(SIGNATURES)

    ´dc´: decoder, struct for all Satellite Data
    ´words_in_subframes´: buffer of 1500 bits, sliced in 5 subframes á 300 bits, which are again sliced in 10 Words á 30 Bits

    # Details
    # Decodes complete subframe from the buffer, saves data for position computing in ´dc.data´. It returns the decoder, ´dc´.
    #The number of saved Bits is resetted to 2 due to the Buffer length of 1502. Those 2 Bits will be the previous 2 Bits to the next 5 Subframes.
"""
function decode_words(dc::GNSSDecoderState, words_in_subframes)

    dc = buffer_control(dc, words_in_subframes)
    if dc.data_integrity == true
        TLM_HOW, subframe_1_data = decode_subframe_1(words_in_subframes[1])
        dc.subframes_decoded[1] = true
        TLM_HOW, subframe_2_data = decode_subframe_2(words_in_subframes[2])
        dc.subframes_decoded[2] = true
        TLM_HOW, subframe_3_data = decode_subframe_3(words_in_subframes[3])
        dc.subframes_decoded[3] = true
        TLM_HOW = decode_subframe_4(words_in_subframes[4])
        dc.subframes_decoded[4] = true
        TLM_HOW = decode_subframe_5(words_in_subframes[5])
        dc.subframes_decoded[5] = true

        if control_data(TLM_HOW, subframe_1_data, subframe_2_data, subframe_3_data)  # Checks for Data Errors
            data = create_data(TLM_HOW, subframe_1_data, subframe_2_data, subframe_3_data) # Creates GPSData from single, subframe specific data and the TLM Word
            dc.data = data
            println(" DECODING COMPLETED!")
        end
    end

    dc.num_bits_buffered = 2 # Due to Buffer length of 1502 and the need to save the last two bits, only 1500 Bits have to been read in the next loop.
    return dc
end