text:
struct MaybeKnown
    hint::Int
    sym::Symbol
    known::Bool
end
struct Loop
    itersymbol::Symbol
    start::MaybeKnown
    stop::MaybeKnown
    step::MaybeKnown
    rangesym::Symbol
    lensym::Symbol
end

struct UnrollSymbols
    u₁loopsym::Symbol
    u₂loopsym::Symbol
    vloopsym::Symbol
end
struct UnrollArgs
    u₁loop::Loop
    u₂loop::Loop
    vloop::Loop
    u₁::Int
    u₂max::Int
    suffix::Int # -1 means not tiled
end
UnPack.unpack(ua::UnrollArgs, ::Val{:u₁loopsym}) = getfield(getfield(ua, :u₁loop), :itersymbol)
UnPack.unpack(ua::UnrollArgs, ::Val{:u₂loopsym}) = getfield(getfield(ua, :u₂loop), :itersymbol)
UnPack.unpack(ua::UnrollArgs, ::Val{:vloopsym}) = getfield(getfield(ua, :vloop), :itersymbol)
UnPack.unpack(ua::UnrollArgs, ::Val{:u₁step}) = getfield(getfield(ua, :u₁loop), :step)
UnPack.unpack(ua::UnrollArgs, ::Val{:u₂step}) = getfield(getfield(ua, :u₂loop), :step)
UnPack.unpack(ua::UnrollArgs, ::Val{:vstep}) = getfield(getfield(ua, :vloop), :step)

function UnrollArgs(ua::UnrollArgs, u₁::Int)
    @unpack u₁loop, u₂loop, vloop, u₂max, suffix = ua
    UnrollArgs(u₁loop, u₂loop, vloop, u₁, u₂max, suffix)
end
# UnrollSymbols(ua::UnrollArgs) = UnrollSymbols(ua.u₁loopsym, ua.u₂loopsym, ua.vloopsym)

# isfirst(ua::UnrollArgs{Nothing}) = iszero(ua.u₁)
# isfirst(ua::UnrollArgs{Int}) = iszero(ua.u₁) & iszero(ua.suffix)

struct UnrollSpecification
    u₁loopnum::Int
    u₂loopnum::Int
    vloopnum::Int
    u₁::Int
    u₂::Int
end
# UnrollSpecification(ls::LoopSet, u₁loop::Loop, vloopsym::Symbol, u₁, u₂) = UnrollSpecification(ls, u₁loop.itersymbol, vloopsym, u₁, u₂)
function UnrollSpecification(us::UnrollSpecification, u₁, u₂)
    @unpack u₁loopnum, u₂loopnum, vloopnum = us
    UnrollSpecification(u₁loopnum, u₂loopnum, vloopnum, u₁, u₂)
end
# function UnrollSpecification(us::UnrollSpecification; u₁ = us.u₁, u₂ = us.u₂)
#     @unpack u₁loopnum, u₂loopnum, vloopnum = us
#     UnrollSpecification(u₁loopnum, u₂loopnum, vloopnum, u₁, u₂)
# end
isunrolled1(us::UnrollSpecification, n::Int) = us.u₁loopnum == n
isunrolled2(us::UnrollSpecification, n::Int) = !isunrolled1(us, n) && us.u₂loopnum == n
isvectorized(us::UnrollSpecification, n::Int) = us.vloopnum == n
function unrollfactor(us::UnrollSpecification, n::Int)
    @unpack u₁loopnum, u₂loopnum, u₁, u₂ = us
    (u₁loopnum == n) ? u₁ : ((u₂loopnum == n) ? u₂ : 1)
end
function pushexpr!(ex::Expr, mk::MaybeKnown)
    if isknown(mk)
        push!(ex.args, staticexpr(gethint(mk)))
    else
        push!(ex.args, getsym(mk))
    end
    nothing
end
pushexpr!(ex::Expr, x::Union{Symbol,Expr}) = (push!(ex.args, x); nothing)
pushexpr!(ex::Expr, x::Integer) = (push!(ex.args, staticexpr(convert(Int, x))); nothing)
MaybeKnown(x::Integer) = MaybeKnown(convert(Int, x), Symbol("##UNDEFINED##"), true)
MaybeKnown(x::Integer, default::Int) = MaybeKnown(x)
MaybeKnown(x::Symbol, default::Int) = MaybeKnown(default, x, false)

isknown(mk::MaybeKnown) = getfield(mk, :known)
getsym(mk::MaybeKnown) = getfield(mk, :sym)
gethint(mk::MaybeKnown) = getfield(mk, :hint)
Base.isone(mk::MaybeKnown) = isknown(mk) && isone(gethint(mk))
Base.iszero(mk::MaybeKnown) = isknown(mk) && iszero(gethint(mk))

function Loop(
    itersymbol::Symbol, start::Union{Int,Symbol}, stop::Union{Int,Symbol}, step::Union{Int,Symbol},
    rangename::Symbol, lensym::Symbol
)
    Loop(itersymbol, MaybeKnown(start, 1), MaybeKnown(stop, 1024), MaybeKnown(step, 1), rangename, lensym)
end
startstopΔ(loop::Loop) = gethint(last(loop)) - gethint(first(loop))
function Base.length(loop::Loop)
    l = startstopΔ(loop)
    s = gethint(step(loop))
    (isone(s) ? l : cld(l, s)) + 1
end
Base.first(l::Loop) = getfield(l, :start)
Base.last(l::Loop) = getfield(l, :stop)
Base.step(l::Loop) = getfield(l, :step)

isstaticloop(l::Loop) = isknown(first(l)) & isknown(last(l)) & isknown(step(l))
unitstep(l::Loop) = isone(step(l))

function startloop(loop::Loop, itersymbol)
    start = first(loop)
    if isknown(start)
        Expr(:(=), itersymbol, gethint(start) - 1)
    else
        Expr(:(=), itersymbol, Expr(:call, lv(:staticm1), Expr(:call, lv(:unwrap), getsym(start))))
    end
end
# mulexpr(a,b) = Expr(:call, lv(:vmul_fast), a, b)

pushmulexpr!(q, a, b) = (push!(q.args, mulexpr(a, b)); nothing)
function pushmulexpr!(q, a, b::Integer)
    if isone(b)
        push!(q.args, a)
    else
        push!(q.args, mulexpr(a, b))
    end
    nothing
end

# function arithmetic_expr(f, a, b)
#     call = Expr(:call, lv(f))
#     if isa(a, MaybeKnown)
#         pushexpr!(
# end
isknown(x::Union{Symbol,Expr}) = false
isknown(x::Integer) = true
gethint(a::Integer) = a
addexpr(a,b) = arithmeticexpr(+, :vadd_fast, a, b)
subexpr(a,b) = arithmeticexpr(-, :vsub_fast, a, b)
mulexpr(a,b) = arithmeticexpr(*, :vmul_fast, a, b)
lazymulexpr(a,b) = arithmeticexpr(*, :lazymul, a, b)
function arithmeticexpr(op, f, a, b)
    if isknown(a) & isknown(b)
        return staticexpr(op(gethint(a), gethint(b)))
    else
        ex = Expr(:call, lv(f))
        pushexpr!(ex, a)
        pushexpr!(ex, b)
        return ex
    end
end
mulexpr(a,b,c) = arithmeticexpr(*, 1, :vmul_fast, a, b, c)
addexpr(a,b,c) = arithmeticexpr(+, 0, :vadd_fast, a, b, c)
function arithmeticexpr(op, init, f, a, b, c)
    ex = Expr(:call, lv(f))
    p = init
    if isknown(a)
        p = op(p, gethint(a))
        known = 1
    else
        pushexpr!(ex, a)
        known = 0
    end
    if isknown(b)
        p = op(p, gethint(b))
        known += 1
    else
        pushexpr!(ex, b)
    end
    if isknown(c)
        p = op(p, gethint(c))
        known += 1
    elseif known == 0
        ex = Expr(:call, lv(f), ex)
        pushexpr!(ex, c)
    end
    if known == 3
        return staticexpr(p)
    else
        if known == 2
            pushexpr!(ex, p)
            return ex
        elseif known == 1
            return Expr(:call, lv(f), ex, staticexpr(p))
        else#known == 0
            return ex
        end
    end
end

function addexpr(ex, incr::Integer)
    if incr > 0
        f = :vadd_fast
    else
        f = :vsub_fast
        incr = -incr
    end
    expr = Expr(:call, lv(f))
    pushexpr!(expr, ex)
    pushexpr!(expr, convert(Int, incr))
    expr
end

staticmulincr(ptr, incr) = Expr(:call, lv(:staticmul), Expr(:call, :eltype, ptr), incr)
function vec_looprange(loop::Loop, UF::Int, mangledname)
    compexpr = Expr(:call, lv(:vsub_fast))
    pushexpr!(compexpr, last(loop))
    incr = step(loop)
    # if isknown(incr)
    #     UF *= gethint(incr)
    # end
    if isone(UF)
        if isone(incr)
            push!(compexpr.args, VECTORWIDTHSYMBOL)
        else
            push!(compexpr.args, mulexpr(VECTORWIDTHSYMBOL, incr))
        end
    else
        dec = mulexpr(VECTORWIDTHSYMBOL, UF, incr)
        push!(compexpr.args, dec)
    end
    f = if isone(incr)
        :(≤)
    else
        compexpr = addexpr(compexpr, incr)
        :(<)
    end
    Expr(:call, f, mangledname, compexpr)
end

# looprange(stopcon, mangledname) = Expr(:call, :≤, mangledname, stopcon)
function looprange(stopcon::Int, incr::Int, loopstep::Int, mangledname)
    if isone(loopstep)
        Expr(:call, :≤, mangledname, stopcon - incr*loopstep)
    else
        Expr(:call, :<, mangledname, stopcon - incr*loopstep + loopstep)
    end
end
function looprange(stopcon, incr::Int, loopstep::Int, mangledname)
    if isone(loopstep)
        Expr(:call, :≤, mangledname, subexpr(stopcon, incr*loopstep))
    else
        Expr(:call, :<, mangledname, subexpr(stopcon, incr*loopstep - loopstep))
    end
end
function looprange(loop::Loop, incr::Int, mangledname)
    start = first(loop)
    stop = last(loop)
    inc = step(loop)
    if isknown(inc)
        if isknown(stop)
            looprange(gethint(stop), incr, gethint(inc), mangledname)
        else
            looprange(getsym(stop), incr, gethint(inc), mangledname)
        end
    else
        subex = Expr(:call, lv(:vsub_fast))
        pushexpr!(subex, stop)
        incex = Expr(:call, lv(:vmul_fast), staticexpr(incr))
        pushexpr!(incex, inc)
        push!(subex.args, incex)
        Expr(:call, :<, mangledname, addexpr(subex, getsym(inc)))
    end
end

function terminatecondition(
    loop::Loop, us::UnrollSpecification, n::Int, mangledname::Symbol, inclmask::Bool, UF::Int = unrollfactor(us, n)
)
    if !isvectorized(us, n)
        looprange(loop, UF, mangledname)
    elseif inclmask
        looprange(loop, 1, mangledname)
    else
        vec_looprange(loop, UF, mangledname) # may not be u₂loop
    end
end

function incrementloopcounter(us::UnrollSpecification, n::Int, mangledname::Symbol, UF::Int, l::Loop)
    incr = step(l)
    if isknown(incr)
        incrementloopcounter(us, n, mangledname, UF * gethint(incr))
    else
        incrementloopcounter(us, n, mangledname, UF, getsym(incr))
    end
end
function incrementloopcounter(us::UnrollSpecification, n::Int, mangledname::Symbol, UF::Int)
    if isvectorized(us, n)
        if isone(UF)
            Expr(:(=), mangledname, addexpr(VECTORWIDTHSYMBOL, mangledname))
        else
            Expr(:(=), mangledname, addexpr(mulexpr(VECTORWIDTHSYMBOL, staticexpr(UF)), mangledname))
        end
    else
        Expr(:(=), mangledname, addexpr(mangledname, UF))
    end
end
function incrementloopcounter(us::UnrollSpecification, n::Int, mangledname::Symbol, UF::Int, incr::Symbol)
    if isvectorized(us, n)
        if isone(UF)
            Expr(:(=), mangledname, addexpr(mulexpr(VECTORWIDTHSYMBOL, incr), mangledname))
        else
            Expr(:(=), mangledname, addexpr(mulexpr(mulexpr(VECTORWIDTHSYMBOL, staticexpr(UF)), incr), mangledname))
        end
    else
        Expr(:(=), mangledname, addexpr(mangledname, mulexpr(incr, UF)))
    end
end

function incrementloopcounter!(q, us::UnrollSpecification, n::Int, UF::Int, l::Loop)
    incr = step(l)
    if isknown(incr)
        incrementloopcounter!(q, us, n, UF * gethint(incr))
    else
        incrementloopcounter!(q, us, n, UF, getsym(incr))
    end
end
function incrementloopcounter!(q, us::UnrollSpecification, n::Int, UF::Int)
    if isvectorized(us, n)
        if isone(UF)
            push!(q.args, VECTORWIDTHSYMBOL)
        else
            push!(q.args, mulexpr(VECTORWIDTHSYMBOL, staticexpr(UF)))
        end
    else
        push!(q.args, staticexpr(UF))
    end
end
function incrementloopcounter!(q, us::UnrollSpecification, n::Int, UF::Int, incr::Symbol)
    if isvectorized(us, n)
        if isone(UF)
            push!(q.args, mulexpr(VECTORWIDTHSYMBOL, incr))
        else
            push!(q.args, mulexpr(mulexpr(VECTORWIDTHSYMBOL, staticexpr(UF)), incr))
        end
    else
        push!(q.args, mulexpr(staticexpr(UF), incr))
    end
end
# function looplengthexpr(loop::Loop)
#     if loop.stopexact
#         if loop.startexact
#             return length(loop)
#         # elseif loop.rangename === Symbol("")
#             # return Expr(:call, lv(:vsub_fast), loop.stophint + 1, loop.startsym)
#         end
#     elseif loop.startexact
#         if isone(loop.starthint)
#             return loop.stopsym
#         # elseif loop.rangename === Symbol("")
#             # return Expr(:call, lv(:vsub_fast), loop.stopsym, loop.starthint - 1)
#         end
#     # elseif loop.rangename === Symbol("")
#         # return Expr(:call, lv(:vsub_fast), loop.stopsym, Expr(:call, lv(:staticm1), loop.startsym))
#     end
#     Expr(:call, lv(:static_length), loop.rangename)
# end
# use_expect() = false
# use_expect() = true
# function looplengthexpr(loop, n)
#     le = looplengthexpr(loop)
#     # if false && use_expect() && isone(n) && !isstaticloop(loop)
#     #     le = expect(le)
#     #     push!(le.args, Expr(:call, Expr(:curly, :Val, length(loop))))
#     # end
#     le
# end

# load/compute/store × isunrolled × istiled × pre/post loop × Loop number
struct LoopOrder <: AbstractArray{Vector{Operation},5}
    oporder::Vector{Vector{Operation}}
    loopnames::Vector{Symbol}
    bestorder::Vector{Symbol}
end
# function LoopOrder(N::Int)
#     LoopOrder(
#         [ Operation[] for _ ∈ 1:8N ],
#         Vector{Symbol}(undef, N), Vector{Symbol}(undef, N)
#     )
# end
LoopOrder() = LoopOrder(Vector{Operation}[],Symbol[],Symbol[])
Base.empty!(lo::LoopOrder) = foreach(empty!, lo.oporder)
function Base.resize!(lo::LoopOrder, N::Int)
    Nold = length(lo.loopnames)
    resize!(lo.oporder, 8N)
    for n ∈ 8Nold+1:8N
        lo.oporder[n] = Operation[]
    end
    resize!(lo.loopnames, N)
    resize!(lo.bestorder, N)
    lo
end
Base.size(lo::LoopOrder) = (2,2,2,length(lo.loopnames))
Base.@propagate_inbounds Base.getindex(lo::LoopOrder, i::Int) = lo.oporder[i]
Base.@propagate_inbounds Base.getindex(lo::LoopOrder, i::Vararg{Int,K}) where {K} = lo.oporder[LinearIndices(size(lo))[i...]]

@enum NumberType::Int8 HardInt HardFloat IntOrFloat INVALID

struct LoopStartStopManager
    terminators::Vector{Int}
    incrementedptrs::Vector{Vector{ArrayReferenceMeta}}
    uniquearrayrefs::Vector{ArrayReferenceMeta}
end
# Must make it easy to iterate
# outer_reductions is a vector of indices (within operation vectors) of the reduction operation, eg the vmuladd op in a dot product
# O(N) search is faster at small sizes
struct LoopSet
    loopsymbols::Vector{Symbol}
    loopsymbol_offsets::Vector{Int}  # symbol loopsymbols[i] corresponds to loops[lso[i]+1:lso[i+1]] (CartesianIndex handling)
    loops::Vector{Loop}
    opdict::Dict{Symbol,Operation}
    operations::Vector{Operation} # Split them to make it easier to iterate over just a subset
    operation_offsets::Vector{Int}
    outer_reductions::Vector{Int} # IDs of reduction operations that need to be reduced at end.
    loop_order::LoopOrder
    preamble::Expr
    prepreamble::Expr # performs extractions that must be performed first, and don't need further registering
    preamble_symsym::Vector{Tuple{Int,Symbol}}
    preamble_symint::Vector{Tuple{Int,Int}}
    preamble_symfloat::Vector{Tuple{Int,Float64}}
    preamble_zeros::Vector{Tuple{Int,NumberType}}
    preamble_funcofeltypes::Vector{Tuple{Int,Float64}}
    includedarrays::Vector{Symbol}
    includedactualarrays::Vector{Symbol}
    syms_aliasing_refs::Vector{Symbol}
    refs_aliasing_syms::Vector{ArrayReferenceMeta}
    cost_vec::Matrix{Float64}
    reg_pres::Matrix{Float64}
    included_vars::Vector{Bool}
    place_after_loop::Vector{Bool}
    unrollspecification::Base.RefValue{UnrollSpecification}
    loadelimination::Base.RefValue{Bool}
    lssm::Base.RefValue{LoopStartStopManager}
    vector_width::Base.RefValue{Int}
    symcounter::Base.RefValue{Int}
    isbroadcast::Base.RefValue{Bool}
    register_size::Base.RefValue{Int}
    register_count::Base.RefValue{Int}
    cache_linesize::Base.RefValue{Int}
    cache_size::Base.RefValue{Tuple{Int,Int,Int}}
    ureduct::Base.RefValue{Int}
    equalarraydims::Vector{Tuple{Vector{Symbol},Vector{Int}}}
    omop::OffsetLoadCollection
    loopordermap::Vector{Int}
    loopindexesbit::Vector{Bool}
    mod::Symbol
end

function UnrollArgs(ls::LoopSet, u₁::Int, unrollsyms::UnrollSymbols, u₂max::Int, suffix::Int)
    @unpack u₁loopsym, u₂loopsym, vloopsym = unrollsyms
    u₁loop = getloop(ls, u₁loopsym)
    u₂loop = u₂loopsym === Symbol("##undefined##") ? u₁loop : getloop(ls, u₂loopsym)
    vloop = getloop(ls, vloopsym)
    UnrollArgs(u₁loop, u₂loop, vloop, u₁, u₂max, suffix)
end

function cost_vec_buf(ls::LoopSet)
    cv = @view(ls.cost_vec[:,2])
    @inbounds for i ∈ 1:4
        cv[i] = 0.0
    end
    cv
end
function reg_pres_buf(ls::LoopSet)
    ps = @view(ls.reg_pres[:,2])
    @inbounds for i ∈ 1:4
        ps[i] = 0
    end
    ps[4] = reg_count(ls)
    ps
end
function save_tilecost!(ls::LoopSet)
    @inbounds for i ∈ 1:4
        ls.cost_vec[i,1] = ls.cost_vec[i,2]
        ls.reg_pres[i,1] = ls.reg_pres[i,2]
    end
    # ls.reg_pres[5,1] = ls.reg_pres[5,2]
end
function set_hw!(ls::LoopSet, rs::Int, rc::Int, cls::Int, l1::Int, l2::Int, l3::Int)
    ls.register_size[] = rs
    ls.register_count[] = rc
    ls.cache_linesize[] = cls
    ls.cache_size[] = (l1,l2,l3)
    # ls.opmask_register[] = omr
    nothing
end
available_registers() = ifelse(has_opmask_registers(), register_count(), register_count() - One())
function set_hw!(ls::LoopSet)
    set_hw!(
        ls, Int(register_size()), Int(available_registers()), Int(cache_linesize()),
        Int(cache_size(StaticInt(1))), Int(cache_size(StaticInt(2))), Int(cache_size(StaticInt(3)))
    )
end
reg_size(ls::LoopSet) = ls.register_size[]
reg_count(ls::LoopSet) = ls.register_count[]
cache_lnsze(ls::LoopSet) = ls.cache_linesize[]
cache_sze(ls::LoopSet) = ls.cache_size[]
# opmask_reg(ls::LoopSet) = ls.opmask_register[]

pushprepreamble!(ls::LoopSet, ex) = push!(ls.prepreamble.args, ex)
function pushpreamble!(ls::LoopSet, op::Operation, v::Symbol)
    if v !== mangledvar(op)
        push!(ls.preamble_symsym, (identifier(op),v))
    end
    nothing
end
function pushpreamble!(ls::LoopSet, op::Operation, v::Number)
    typ = v isa Integer ? HardInt : HardFloat
    id = identifier(op)
    if iszero(v)
        push!(ls.preamble_zeros, (id, typ))
    elseif isone(v)
        push!(ls.preamble_funcofeltypes, (id, MULTIPLICATIVE_IN_REDUCTIONS))
    elseif v isa Integer
        push!(ls.preamble_symint, (id, convert(Int,v)))
    else
        push!(ls.preamble_symfloat, (id, convert(Float64,v)))
    end
end
pushpreamble!(ls::LoopSet, ex::Expr) = push!(ls.preamble.args, ex)
# function pushpreamble!(ls::LoopSet, op::Operation, RHS::Expr)
#     c = gensym(:licmconst)
#     if RHS.head === :call && first(RHS.args) === :zero
#         push!(ls.preamble_zeros, (identifier(op), IntOrFloat))
#     elseif RHS.head === :call && first(RHS.args) === :one
#         push!(ls.preamble_funcofeltypes, (identifier(op), MULTIPLICATIVE_IN_REDUCTIONS))
#     else
#         pushpreamble!(ls, Expr(:(=), c, RHS))
#         pushpreamble!(ls, op, c)
#     end
#     nothing
# end
function zerotype(ls::LoopSet, op::Operation)
    opid = identifier(op)
    for (id,typ) ∈ ls.preamble_zeros
        id == opid && return typ
    end
    INVALID
end

includesarray(ls::LoopSet, array::Symbol) = array ∈ ls.includedarrays

function LoopSet(mod::Symbol)
    LoopSet(
        Symbol[], [0], Loop[],
        Dict{Symbol,Operation}(),
        Operation[], [0],
        Int[],
        LoopOrder(),
        Expr(:block),Expr(:block),
        Tuple{Int,Symbol}[],
        Tuple{Int,Int}[],
        Tuple{Int,Float64}[],
        Tuple{Int,NumberType}[],Tuple{Int,Symbol}[],
        Symbol[], Symbol[], Symbol[],
        ArrayReferenceMeta[],
        Matrix{Float64}(undef, 4, 2), # cost_vec
        Matrix{Float64}(undef, 4, 2), # reg_pres
        Bool[], Bool[], Ref{UnrollSpecification}(),
        Ref(false), Ref{LoopStartStopManager}(),
        Ref(0), Ref(0), Ref(false),
        Ref(0), Ref(0), Ref(0), Ref((0,0,0)),# hw params
        Ref(-1), # Ureduct
        Tuple{Vector{Symbol},Vector{Int}}[],
        OffsetLoadCollection(),
        Int[], Bool[],
        mod
    )
end

"""
Used internally to create symbols unique for this loopset.
This is used so that identical loops will create identical `_avx_!` calls in the macroexpansions, hopefully reducing recompilation.
"""
gensym!(ls::LoopSet, s) = Symbol("###$(s)###$(ls.symcounter[] += 1)###")

function cacheunrolled!(ls::LoopSet, u₁loop::Symbol, u₂loop::Symbol, vloopsym::Symbol)
    vloop = getloop(ls, vloopsym)
    for op ∈ operations(ls)
        setunrolled!(op, u₁loop, u₂loop, vloopsym)
        empty!(children(op))
        for opp ∈ parents(op)
            push!(children(opp), op)
        end
        if accesses_memory(op)
            rc = rejectcurly(ls, op, u₁loop, vloopsym)
            op.rejectcurly = rc
            if rc
                op.rejectinterleave = true
            else
                omop = ls.omop
                batchid, opind = omop.batchedcollectionmap[identifier(op)]
                op.rejectinterleave = ((batchid == 0) || (!isvectorized(op))) || rejectinterleave(ls, op, vloop, omop.batchedcollections[batchid])
            end
        end
    end
end
rejectcurly(op::Operation) = op.rejectcurly
rejectinterleave(op::Operation) = op.rejectinterleave
num_loops(ls::LoopSet) = length(ls.loops)
function oporder(ls::LoopSet)
    N = length(ls.loop_order.loopnames)
    reshape(ls.loop_order.oporder, (2,2,2,N))
end
names(ls::LoopSet) = ls.loop_order.loopnames
reversenames(ls::LoopSet) = ls.loop_order.bestorder
function getloopid_or_nothing(ls::LoopSet, s::Symbol)
    # @show ls.loopsymbols, s
    for (loopnum,sym) ∈ enumerate(ls.loopsymbols)
        s === sym && return loopnum
    end
end

getloopid(ls::LoopSet, s::Symbol) = getloopid_or_nothing(ls, s)::Int
# function getloopid(ls::LoopSet, s::Symbol)::Int
#     @show ls.loops
#     id = getloopid_or_nothing(ls, s)
#     @show id
#     id
# end
# getloop(ls::LoopSet, i::Integer) = getloop(ls, names(ls)[i])
getloop(ls::LoopSet, i::Integer) = ls.loops[ls.loopordermap[i]] # takes nest level after reordering
getloop_from_id(ls::LoopSet, i::Integer) = ls.loops[i] # takes w/ respect to original loop order.
getloop(ls::LoopSet, s::Symbol) = getloop_from_id(ls, getloopid(ls, s))
getloopsym(ls::LoopSet, i::Integer) = ls.loopsymbols[i]
Base.length(ls::LoopSet, s::Symbol) = length(getloop(ls, s))
function init_loop_map!(ls::LoopSet)
    @unpack loopordermap = ls
    order = names(ls)
    resize!(loopordermap, length(order))
    for (i,o) ∈ enumerate(order)
        loopordermap[i] = getloopid(ls,o)
    end
    nothing
end

# isstaticloop(ls::LoopSet, s::Symbol) = isstaticloop(getloop(ls,s))
# looprangehint(ls::LoopSet, s::Symbol) = length(getloop(ls, s))
# looprangesym(ls::LoopSet, s::Symbol) = getloop(ls, s).rangesym

"""
getop only works while construction a LoopSet object. You cannot use it while lowering.
"""
getop(ls::LoopSet, var::Number, elementbytes) = add_constant!(ls, var, elementbytes)
function getop(ls::LoopSet, var::Symbol, elementbytes::Int)
    get!(ls.opdict, var) do
        add_constant!(ls, var, elementbytes)
    end
end
function getop(ls::LoopSet, var::Symbol, deps, elementbytes::Int)
    get!(ls.opdict, var) do
        add_constant!(ls, var, deps, gensym!(ls, "constant"), elementbytes)
    end
end
getop(ls::LoopSet, i::Int) = ls.operations[i]

findop(ls::LoopSet, s::Symbol) = findop(operations(ls), s)
function findop(ops::Vector{Operation}, s::Symbol)
    for op ∈ ops
        name(op) === s && return op
    end
    throw(ArgumentError("Symbol $s not found."))
end

# """
# Returns an operation with the same name as `s`.
# """
# function getoperation(ls::LoopSet, s::Symbol)
#     for op ∈ Iterators.Reverse(operations(ls))
#         name(op) === s && return op
#     end
#     throw("Symbol $s not found among operations(ls).")
# end

function Operation(
    ls::LoopSet, variable, elementbytes, instruction,
    node_type, dependencies, reduced_deps, parents, ref = NOTAREFERENCE
)
    Operation(
        length(operations(ls)), variable, elementbytes, instruction,
        node_type, dependencies, reduced_deps, parents, ref
    )
end
function Operation(ls::LoopSet, variable, elementbytes, instr, optype, mpref::ArrayReferenceMetaPosition)
    Operation(length(operations(ls)), variable, elementbytes, instr, optype, mpref)
end

operations(ls::LoopSet) = ls.operations
function pushop!(ls::LoopSet, op::Operation, var::Symbol = name(op))
    for opp ∈ operations(ls)
        if matches(op, opp)
            ls.opdict[var] = opp
            return opp
        end
    end
    push!(ls.operations, op)
    ls.opdict[var] = op
    op
end
function add_block!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
    for x ∈ ex.args
        x isa Expr || continue # be that general?
        x.head === :inbounds && continue
        push!(ls, x, elementbytes, position)
    end
end
function maybestatic!(expr::Expr)
    if expr.head === :call
        f = first(expr.args)
        if f === :length
            expr.args[1] = lv(:maybestaticlength)
        elseif f === :size && length(expr.args) == 3
            i = expr.args[3]
            if i isa Integer
                expr.args[1] = lv(:maybestaticsize)
                expr.args[3] = Expr(:call, Expr(:curly, :Val, convert(Int, i)))
            end
        else
            static_literals!(expr)
        end
    end
    expr
end
add_loop_bound!(ls::LoopSet, itersym::Symbol, bound::Union{Integer,Symbol}, upper::Bool, step::Bool)::MaybeKnown = MaybeKnown(bound, upper ? 1024 : 1)
function add_loop_bound!(ls::LoopSet, itersym::Symbol, bound::Expr, upper::Bool, step::Bool)::MaybeKnown
    maybestatic!(bound)
    N = gensym!(ls, string(itersym) * (upper ? "_loop_upper_bound" : (step ? "_loop_step" : "_loop_lower_bound")))
    pushprepreamble!(ls, Expr(:(=), N, bound))
    MaybeKnown(N, upper ? 1024 : 1)
end
static_literals!(s::Symbol) = s
function static_literals!(q::Expr)
    for (i,ex) ∈ enumerate(q.args)
        if ex isa Number
            q.args[i] = staticexpr(ex)
        elseif ex isa Expr
            static_literals!(ex)
        end
    end
    q
end
function range_loop!(ls::LoopSet, itersym::Symbol, l::MaybeKnown, u::MaybeKnown, s::MaybeKnown)
    rangename = gensym!(ls, "range"); lenname = gensym!(ls, "length")
    range = Expr(:call, :(:))
    pushexpr!(range, l)
    isone(s) || pushexpr!(range, s)
    pushexpr!(range, u)
    pushprepreamble!(ls, Expr(:(=), rangename, range))
    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, lv(:maybestaticlength), rangename)))
    Loop(itersym, l, u, s, rangename, lenname)
end
function range_loop!(ls::LoopSet, r::Expr, itersym::Symbol)::Loop
    lower = r.args[2]
    sii::Bool = if length(r.args) == 3
        step = 1
        upper = r.args[3]
        true
    elseif length(r.args) == 4
        step = r.args[3]
        upper = r.args[4]
        isa(step, Integer)
    else
        throw("Literal ranges must have either 2 or 3 arguments.")
    end
    lii::Bool = lower isa Integer
    uii::Bool = upper isa Integer
    l::MaybeKnown = add_loop_bound!(ls, itersym, lower, false, false)
    u::MaybeKnown = add_loop_bound!(ls, itersym, upper, true, false)
    s::MaybeKnown = add_loop_bound!(ls, itersym, step, false, true)
    range_loop!(ls, itersym, l, u, s)
end
function oneto_loop!(ls::LoopSet, r::Expr, itersym::Symbol)::Loop
    otN = r.args[2]
    l = MaybeKnown(1, 0)
    s = MaybeKnown(1, 0)
    u::MaybeKnown = if otN isa Integer
        rangename = lensym = Symbol("")
        MaybeKnown(convert(Int, otN)::Int, 0)
    else
        otN isa Expr && maybestatic!(otN)
        lensym = N = gensym!(ls, "loop" * string(itersym))
        rangename = gensym!(ls, "range");
        pushprepreamble!(ls, Expr(:(=), N, otN))
        pushprepreamble!(ls, Expr(:(=), rangename, Expr(:call, :(:), staticexpr(1), N)))
        MaybeKnown(N, 1024)
    end
    Loop(itersym, l, u, s, rangename, lensym)
end

@inline _reverse(r) = maybestaticlast(r):-static_step(r):maybestaticfirst(r)
@inline canonicalize_range(r::OptionallyStaticUnitRange) = r
@inline function canonicalize_range(r::OptionallyStaticRange, ::StaticInt{S}) where {S}
    ifelse(ArrayInterface.gt(StaticInt{S}(), Zero()), r, _reverse(r))
end
@inline canonicalize_range(r::OptionallyStaticRange, s::Integer) = s > 0 ? r : _reverse(r)
@inline canonicalize_range(r::CloseOpen) = r
@inline canonicalize_range(r::AbstractUnitRange) = maybestaticfirst(r):maybestaticlast(r)
@inline canonicalize_range(r::OptionallyStaticRange) = canonicalize_range(r, static_step(r))
@inline canonicalize_range(r::AbstractRange) = canonicalize_range(maybestaticfirst(r):static_step(r):maybestaticlast(r))
@inline canonicalize_range(r::CartesianIndices) = CartesianIndices(map(canonicalize_range, r.indices))

function misc_loop!(ls::LoopSet, r::Union{Expr,Symbol}, itersym::Symbol, staticstepone::Bool)::Loop
    rangename = gensym!(ls, "looprange" * string(itersym)); lenname = gensym!(ls, "looplen" * string(itersym));
    pushprepreamble!(ls, Expr(:(=), rangename, Expr(:call, lv(:canonicalize_range), :(@inbounds $(static_literals!(r))))))
    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, lv(:maybestaticlength), rangename)))
    L = add_loop_bound!(ls, itersym, Expr(:call, lv(:maybestaticfirst), rangename), false, false)
    U = add_loop_bound!(ls, itersym, Expr(:call, lv(:maybestaticlast), rangename), true, false)
    if staticstepone
        Loop(itersym, L, U, MaybeKnown(1), rangename, lenname)
    else
        S = add_loop_bound!(ls, itersym, Expr(:call, lv(:static_step), rangename), false, true)
        Loop(itersym, L, U, S, rangename, lenname)
    end
end

function indices_loop!(ls::LoopSet, r::Expr, itersym::Symbol)::Loop
    if length(r.args) == 3
        arrays = r.args[2]
        dims = r.args[3]
        if isexpr(arrays, :tuple) && length(arrays.args) > 1 && all(s -> s isa Symbol, arrays.args)
            narrays =  length(arrays.args)::Int
            if dims isa Integer
                # ids = Vector{NTuple{2,Int}}(undef, narrays)
                vptrs = Vector{Symbol}(undef, narrays)
                mdims = fill(dims::Int, narrays)
                # _d::Int = dims
                for n ∈ 1:narrays
                    a_s::Symbol = arrays.args[n]
                    vptrs[n] = vptr(a_s)
                end
                push!(ls.equalarraydims, (vptrs, mdims))
            elseif isexpr(dims, :tuple) && length(dims.args) == narrays && all(i -> i isa Integer, dims.args)
                # ids = Vector{NTuple{2,Int}}(undef, narrays)
                vptrs = Vector{Symbol}(undef, narrays)
                mdims = Vector{Int}(undef, narrays)
                for n ∈ 1:narrays
                    a_s::Symbol = arrays.args[n]
                    vptrs[n] = vptr(a_s)
                    mdims[n] = dims.args[n]
                end
                push!(ls.equalarraydims, (vptrs, mdims))
                # push!(ls.equalarraydims, ids)
            end
        end
    end
    misc_loop!(ls, r, itersym, true)
end

"""
This function creates a loop, while switching from 1 to 0 based indices
"""
function register_single_loop!(ls::LoopSet, looprange::Expr)
    itersym = (looprange.args[1])::Symbol
    r = looprange.args[2]
    loop = if isexpr(r, :call)
        r = r::Expr        # julia#37342
        f = first(r.args)
        if f === :(:)
            range_loop!(ls, r, itersym)
        elseif f === :OneTo || isscopedname(f, :Base, :OneTo)
            oneto_loop!(ls, r, itersym)
        elseif f === :indices || (isexpr(f, :(.), 2) && (f.args[2] === QuoteNode(:indices)) && ((f.args[1] === :ArrayInterface) || (f.args[1] === :LoopVectorization)))
            indices_loop!(ls, r, itersym)
        else
            misc_loop!(ls, r, itersym, (f === :eachindex) | (f === :axes))
        end
    elseif isa(r, Symbol)
        misc_loop!(ls, r, itersym, false)
    else
        throw(LoopError("Unrecognized loop range type: $r."))
    end
    add_loop!(ls, loop, itersym)
    nothing
end
function register_loop!(ls::LoopSet, looprange::Expr)
    if looprange.head === :block # multiple loops
        for lr ∈ looprange.args
            register_single_loop!(ls, lr::Expr)
        end
    else
        @assert looprange.head === :(=)
        register_single_loop!(ls, looprange)
    end
end
function add_loop!(ls::LoopSet, q::Expr, elementbytes::Int)
    register_loop!(ls, q.args[1]::Expr)
    body = q.args[2]::Expr
    position = length(ls.loopsymbols)
    if body.head === :block
        add_block!(ls, body, elementbytes, position)
    else
        push!(ls, q, elementbytes, position)
    end
end
function add_loop!(ls::LoopSet, loop::Loop, itersym::Symbol = loop.itersymbol)
    push!(ls.loopsymbols, itersym)
    push!(ls.loops, loop)
    nothing
end

# function instruction(x)
#     x isa Symbol ? x : last(x.args).value
# end
# instruction(ls::LoopSet, f::Symbol) = instruction!(ls, f)
function instruction!(ls::LoopSet, x::Expr)
    # x isa Symbol && return x
    if x.head === :$
        _x = only(x.args)
        _x isa Symbol && return instruction!(ls, _x)
        @assert _x isa Expr
        x = _x
    end
    instr = last(x.args).value
    if instr ∉ keys(COST)
        instr = gensym!(ls, "f")
        pushpreamble!(ls, Expr(:(=), instr, x))
        Instruction(Symbol(""), instr)
    else
        Instruction(:LoopVectorization, instr)
    end
end
instruction!(ls::LoopSet, x::Symbol) = instruction(x)
function instruction!(ls::LoopSet, f::F) where {F <: Function}
    get(FUNCTIONSYMBOLS, F) do
        instr = gensym!(ls, "f")
        pushpreamble!(ls, Expr(:(=), instr, f))
        Instruction(Symbol(""), instr)
    end
end

function maybe_const_compute!(ls::LoopSet, LHS::Symbol, op::Operation, elementbytes::Int, position::Int)
    # return op
    if iscompute(op) && iszero(length(loopdependencies(op)))
        ls.opdict[LHS] = add_constant!(ls, LHS, ls.loopsymbols[1:position], gensym!(ls, instruction(op).instr), elementbytes, :numericconstant)
    else
        op
    end
end
strip_op_linenumber_nodes(q::Expr) = only(filter(x -> !isa(x, LineNumberNode), q.args))

function add_operation!(ls::LoopSet, LHS::Symbol, RHS::Symbol, elementbytes::Int, position::Int)
    add_constant!(ls, RHS, ls.loopsymbols[1:position], LHS, elementbytes)
end
function add_operation!(
    ls::LoopSet, LHS::Symbol, RHS::Expr, elementbytes::Int, position::Int
)
    if RHS.head === :ref
        add_load_ref!(ls, LHS, RHS, elementbytes)
    elseif RHS.head === :call
        f = first(RHS.args)
        if f === :getindex
            add_load_getindex!(ls, LHS, RHS, elementbytes)
        elseif f === :zero || f === :one
            c = gensym!(ls, f)
            op = add_constant!(ls, c, ls.loopsymbols[1:position], LHS, elementbytes, :numericconstant)
            if f === :zero
                push!(ls.preamble_zeros, (identifier(op), IntOrFloat))
            else
                push!(ls.preamble_funcofeltypes, (identifier(op), MULTIPLICATIVE_IN_REDUCTIONS))
            end
            op
        else
            # maybe_const_compute!(ls, add_compute!(ls, LHS, RHS, elementbytes, position), elementbytes, position)
            add_compute!(ls, LHS, RHS, elementbytes, position)
        end
    elseif RHS.head === :if
        add_if!(ls, LHS, RHS, elementbytes, position)
    elseif RHS.head === :block
        add_operation!(ls, LHS, strip_op_linenumber_nodes(RHS), elementbytes, position)
    else
        throw(LoopError("Expression not recognized.", RHS))
    end
end
add_operation!(ls::LoopSet, RHS::Expr, elementbytes::Int, position::Int) = add_operation!(ls, gensym!(ls, "LHS"), RHS, elementbytes, position)
function add_operation!(
    ls::LoopSet, LHS_sym::Symbol, RHS::Expr, LHS_ref::ArrayReferenceMetaPosition, elementbytes::Int, position::Int
)
    if RHS.head === :ref# || (RHS.head === :call && first(RHS.args) === :getindex)
        array, rawindices = ref_from_expr!(ls, RHS)
        RHS_ref = array_reference_meta!(ls, array, rawindices, elementbytes, gensym!(ls, LHS_sym))
        op = add_load!(ls, RHS_ref, elementbytes)
        iop = add_compute!(ls, LHS_sym, :identity, [op], elementbytes)
        # pushfirst!(LHS_ref.parents, iop)
    elseif RHS.head === :call
        f = first(RHS.args)
        if f === :getindex
            add_load!(ls, LHS_sym, LHS_ref, elementbytes)
        elseif f === :zero || f === :one
            c = gensym!(ls, f)
            op = add_constant!(ls, c, ls.loopsymbols[1:position], LHS_sym, elementbytes, :numericconstant)
            # op = add_constant!(ls, c, Symbol[], LHS_sym, elementbytes, :numericconstant)
            if f === :zero
                push!(ls.preamble_zeros, (identifier(op), IntOrFloat))
            else
                push!(ls.preamble_funcofeltypes, (identifier(op), MULTIPLICATIVE_IN_REDUCTIONS))
            end
            op
        else
            add_compute!(ls, LHS_sym, RHS, elementbytes, position, LHS_ref)
        end
    elseif RHS.head === :if
        add_if!(ls, LHS_sym, RHS, elementbytes, position, LHS_ref)
    elseif RHS.head === :block
        add_operation!(ls, LHS, strip_op_linenumber_nodes(RHS), elementbytes, position)
    else
        throw(LoopError("Expression not recognized.", RHS))
    end
end

function prepare_rhs_for_storage!(ls::LoopSet, RHS::Union{Symbol,Expr}, array, rawindices, elementbytes::Int, position::Int)
    RHS isa Symbol && return add_store!(ls, RHS, array, rawindices, elementbytes)
    mpref = array_reference_meta!(ls, array, rawindices, elementbytes)
    cachedparents = copy(mpref.parents)
    ref = mpref.mref.ref
    lrhs = gensym!(ls, "RHS")
    mpref.varname = lrhs
    add_operation!(ls, lrhs, RHS, mpref, elementbytes, position)
    mpref.parents = cachedparents
    add_store!(ls, mpref, elementbytes)
end

function Base.push!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
    if ex.head === :call
        finex = first(ex.args)::Symbol
        if finex === :setindex!
            array, rawindices = ref_from_setindex!(ls, ex)
            prepare_rhs_for_storage!(ls, ex.args[3]::Union{Symbol,Expr}, array, rawindices, elementbytes, position)
        else
            error("Function $finex not recognized.")
        end
    elseif ex.head === :(=)
        LHS = ex.args[1]
        RHS = ex.args[2]
        if LHS isa Symbol
            if RHS isa Expr
                maybe_const_compute!(ls, LHS, add_operation!(ls, LHS, RHS, elementbytes, position), elementbytes, position)
            else
                add_constant!(ls, RHS, ls.loopsymbols[1:position], LHS, elementbytes)
            end
        elseif LHS isa Expr
            if LHS.head === :ref
                if RHS isa Symbol
                    add_store_ref!(ls, RHS, LHS, elementbytes)
                elseif RHS isa Expr
                    # need to check if LHS appears in RHS
                    # assign RHS to lrhs
                    array, rawindices = ref_from_expr!(ls, LHS)
                    prepare_rhs_for_storage!(ls, RHS, array, rawindices, elementbytes, position)
                else
                    add_store_ref!(ls, RHS, LHS, elementbytes)  # is this necessary? (Extension API?)
                end
            elseif LHS.head === :tuple
                @assert length(LHS.args) ≤ 9 "Functions returning more than 9 values aren't currently supported."
                lhstemp = gensym!(ls, "lhstuple")
                vparents = Operation[maybe_const_compute!(ls, lhstemp, add_operation!(ls, lhstemp, RHS, elementbytes, position), elementbytes, position)]
                for i ∈ eachindex(LHS.args)
                    f = (:first,:second,:third,:fourth,:fifth,:sixth,:seventh,:eighth,:ninth)[i]
                    lhsi = LHS.args[i]
                    if lhsi isa Symbol
                        add_compute!(ls, lhsi, f, vparents, elementbytes)
                    elseif lhsi isa Expr && lhsi.head === :ref
                        tempunpacksym = gensym!(ls, "tempunpack")
                        add_compute!(ls, tempunpacksym, f, vparents, elementbytes)
                        add_store_ref!(ls, tempunpacksym, lhsi, elementbytes)
                    else
                        throw(LoopError("Unpacking the above expression in the left hand side was not understood/supported.", lhsi))
                    end
                end
                first(vparents)
            else
                throw(LoopError("LHS not understood; only `:ref`s and `:tuple`s are currently supported.", LHS))
            end
        else
            throw(LoopError("LHS not understood.", LHS))
        end
    elseif ex.head === :block
        add_block!(ls, ex, elementbytes, position)
    elseif ex.head === :for
        add_loop!(ls, ex, elementbytes)
    elseif ex.head === :&&
        add_andblock!(ls, ex, elementbytes, position)
    elseif ex.head === :||
        add_orblock!(ls, ex, elementbytes, position)
    elseif ex.head === :local # Handle locals introduced by `@inbounds`; using `local` with `@avx` is not recomended (nor is `@inbounds`; which applies automatically regardless)
        @assert length(ex.args) == 1 # TODO replace assert + first with "only" once support for Julia < 1.4 is dropped
        localbody = first(ex.args)
        @assert localbody.head === :(=)
        @assert length(localbody.args) == 2
        LHS = (localbody.args[1])::Symbol
        RHS = push!(ls, (localbody.args[2]), elementbytes, position)
        if isstore(RHS)
            RHS
        else
            add_compute!(ls, LHS, :identity, [RHS], elementbytes)
        end
    else
        throw(LoopError("Don't know how to handle expression.", ex))
    end
end

function UnrollSpecification(ls::LoopSet, u₁loop::Symbol, u₂loop::Symbol, vloopsym::Symbol, u₁, u₂)
    order = names(ls)
    nu₁ = findfirst(Base.Fix2(===,u₁loop), order)::Int
    nu₂ = u₂ == -1 ? nu₁ : findfirst(Base.Fix2(===,u₂loop), order)::Int
    nv = findfirst(Base.Fix2(===,vloopsym), order)::Int
    UnrollSpecification(nu₁, nu₂, nv, u₁, u₂)
end

"""
  looplengthprod(ls::LoopSet)

Convert to `Float64` for the sake of non-64 bit platforms.
"""
function looplengthprod(ls::LoopSet)
    l = 1.0
    for loop ∈ ls.loops
        l *= Float64(length(loop))
    end
    l
end
    # prod(Float64 ∘ length, ls.loops)

function looplength(ls::LoopSet, s::Symbol)
    # search_tree(parents(operations(ls)[i]), name(op)) && return true
    id = getloopid_or_nothing(ls, s)
    if id === nothing
        l = 0.0
        # TODO: we could double count a loop.
        for op ∈ operations(ls)
            name(op) === s || continue
            for opp ∈ parents(op)
                if isloopvalue(opp)
                    oppname = first(loopdependencies(opp))
                    l += looplength(ls, oppname)
                elseif iscompute(opp)
                    oppname = name(opp)
                    l += looplength(ls, oppname)
                    # TODO elseif isconstant(opp)
                end
            end
            l += 1 - length(parents(op))
        end
        l
    else
        Float64(length(ls.loops[id]))
    end
end

# function getunrolled(ls::LoopSet)
#     order = names(ls)
#     us = ls.unrollspecification[]
#     @unpack u₁loopnum, u₂loopnum = us
#     order[u₁loopnum], order[u₂loopnum]
# end
offsetloadcollection(ls::LoopSet) = ls.omop
function fill_offset_memop_collection!(ls::LoopSet)
    omop = offsetloadcollection(ls)
    ops = operations(ls)
    num_ops = length(ops)
    @unpack opids, opidcollectionmap, batchedcollections, batchedcollectionmap = omop
    length(opidcollectionmap) == 0 || return
    resize!(opidcollectionmap, num_ops)
    fill!(opidcollectionmap, (0,0));
    resize!(batchedcollectionmap, num_ops)
    fill!(batchedcollectionmap, (0,0))
    empty!(opids);# empty!(offsets);
    for i ∈ 1:num_ops
        op = ops[i]
        isconditionalmemop(op) && continue # not supported yet
        opref = op.ref.ref
        opisload = isload(op)
        (opisload | isstore(op)) || continue
        opidcollectionmap[i] === (0,0) || continue # if not -1, we already handled
        isdiscontiguous(op) && continue
        collectionsize = 0
        for j ∈ i+1:num_ops
            opp = ops[j]
            isconditionalmemop(opp) && continue # not supported yet
            if opisload # Each collection is either entirely loads or entirely stores
                isload(opp) || continue
            else
                isstore(opp) || continue
            end
            # @show op opp
            oppref = opp.ref.ref
            sameref(opref, oppref) || continue
            if collectionsize == 0
                push!(opids, [identifier(op), identifier(opp)])
                # push!(offsets, [opref.offsets, oppref.offsets])
                opidcollectionmap[identifier(op)] = (length(opids),1)
            else
                push!(last(opids), identifier(opp))
                # push!(last(offsets), oppref.offsets)
            end
            opidcollectionmap[identifier(opp)] = (length(opids),length(last(opids)))
            collectionsize += 1
        end
    end
    for (collectionid,opidc) ∈ enumerate(opids)
        length(opidc) > 1 || continue

        # we check if we can turn the offsets into an unroll
        # we have up to `length(opidc)` loads to do, so we allocate that many "base" vectors
        # then we iterate through them, adding them to collections as appropriate
        # inner vector tuple is of (op_pos_w/in collection,o)
        unroll_collections = Vector{Vector{Tuple{Int,Int}}}(undef, length(opidc))
        num_unroll_collections = 0
        # num_ops_considered = length(opidc)
        r = 2:length(getindices(ops[first(opidc)]))

        for (i,opid) ∈ enumerate(opidc)
            op = ops[opid]
            offset = getoffsets(op)
            o = offset[1]
            v = view(offset, r)
            found_match = false
            for j ∈ 1:num_unroll_collections
                collectionⱼ = unroll_collections[j]
                # giet id (`first`) of first item in collection to get base offsets for comparison
                # @show op, opid ops[opidc[first(first(collectionⱼ))], first(first(collectionⱼ))
                if view(getoffsets(ops[opidc[first(first(collectionⱼ))]]), r) == v
                    found_match = true
                    push!(collectionⱼ, (i, o))
                end
            end
            # @show opid, found_match
            if !found_match
                num_unroll_collections += 1 # the `i` points to position within `opidc`
                unroll_collections[num_unroll_collections] = [(i,o)]
            end
        end
        for j ∈ 1:num_unroll_collections
            collectionⱼ = unroll_collections[j]
            collen = length(collectionⱼ)
            collen ≤ 1 && continue
            # we have multiple, easiest to process if we sort them
            sort!(collectionⱼ, by=last)
            istart = 1; ostart = last(first(collectionⱼ))
            oprev = ostart
            for i ∈ 2:collen
                onext = last(collectionⱼ[i])
                if onext == oprev + 1
                    oprev = onext
                    continue
                end
                # we skipped one, so we must now lower all previous
                if oprev ≠ ostart # it's just 1
                    pushbatchedcollection!(batchedcollections, batchedcollectionmap, opidc, ops, collectionⱼ, istart, i-1)
                end
                # restart istart and ostart
                istart = i
                ostart = onext
                oprev = onext
            end
            if istart ≠ collen
                pushbatchedcollection!(batchedcollections, batchedcollectionmap, opidc, ops, collectionⱼ, istart, collen)
            end
        end
    end
end

function pushbatchedcollection!(batchedcollections, batchedcollectionmap, opidc, ops, collectionⱼ, istart, istop)
    colview = view(collectionⱼ, istart:istop)
    push!(batchedcollections, colview)
    bclen = length(batchedcollections)
    for (i,(k,_)) ∈ enumerate(colview)
        # batchedcollectionmap[identifier(op)] gives index into `batchedcollections` containing `colview`
        batchedcollectionmap[identifier(ops[opidc[k]])] = (bclen,i)
    end
end

"""
Returns `0` if the op is the declaration of the constant outerreduction variable.
Returns `n`, where `n` is the constant declarations's index among parents(op), if op is an outter reduction.
Returns `-1` if not an outerreduction.
"""
function isouterreduction(ls::LoopSet, op::Operation)
    if isconstant(op) # equivalent to checking if length(loopdependencies(op)) == 0
        op.instruction === LOOPCONSTANT ? 0 : -1
    elseif iscompute(op)
        var = op.variable
        for opid ∈ ls.outer_reductions
            rop = operations(ls)[opid]
            if rop === op
                for (n,opp) ∈ enumerate(parents(op))
                    opp.variable === var && return n
                end
            else
                for (n,opp) ∈ enumerate(parents(op))
                    opp === rop && return n
                    search_tree(parents(opp), rop.variable) && return n
                end
            end
        end
        -1
    else
        -1
    end
end

struct LoopError <: Exception
    msg
    ex
    LoopError(msg, ex=nothing) = new(msg, ex)
end

function Base.showerror(io::IO, err::LoopError)
    printstyled(io, err.msg; color = :red)
    err.ex === nothing || printstyled(io, '\n', err.ex)
end


text:
function profile(y, params);

    Uj = params.Uj
    Lj = params.Lj
    Fr = params.Fr

    Ψ  = -Uj * tanh.(y/Lj);
    U  =  Uj * sech.(y/Lj).^2;
    Qy = Fr*U .- Uj*(4 * sinh.(y/Lj).^2 .- 2) .* sech.(y/Lj).^4;

    return Qy,U,Ψ
end

# Q    = Ψ_yy - Fr Ψ
# Qy   = Ψ_yyy - Fr Ψ_y = - U_yy + Fr U = Fr U - U_yy
# U_yy = Uj * ( 4*tanh.(y/Lj).^2*sech.(y/Lj).^2 - sech.(y/Lj).^4)

ext: jl


text: @warn "`neural/layers.jl` is not tested."


text:
module MLJBase

# ===================================================================
# IMPORTS

import Base: ==, precision, getindex, setindex!
import Base.+, Base.*

# Scitype
import ScientificTypes: TRAIT_FUNCTION_GIVEN_NAME
using MLJScientificTypes
using MLJModelInterface
import MLJModelInterface: fit, update, update_data, transform,
                          inverse_transform, fitted_params, predict,
                          predict_mode, predict_mean, predict_median,
                          evaluate, clean!

# Containers & data manipulation
using Tables
import PrettyTables
using DelimitedFiles
using OrderedCollections
using CategoricalArrays
import InvertedIndices: Not

# Distributed computing
using Distributed
using ComputationalResources
using ComputationalResources: CPUProcesses
using ProgressMeter
import .Threads

# Operations & extensions
import LossFunctions
import LossFunctions: DistanceLoss, MarginLoss, SupervisedLoss
import StatsBase
import StatsBase: fit!, mode, countmap
import Missings: levels
import Distributions
import Distributions: pdf

# from Standard Library:
using Statistics, LinearAlgebra, Random, InteractiveUtils

# ===================================================================
## METHOD EXPORTS

# -------------------------------------------------------------------
# re-exports from MLJModelInterface, (MLJ)ScientificTypes
# NOTE: MLJBase does **not** re-export UnivariateFinite to avoid
# ambiguities between the raw constructor (MLJBase.UnivariateFinite)
# and the general method (MLJModelInterface.UnivariateFinite)

# MLJ model hierarchy
export MLJType, Model, Supervised, Unsupervised,
       Probabilistic, Deterministic, Interval, Static,
       UnivariateFinite

# model constructor + metadata
export @mlj_model, metadata_pkg, metadata_model

# model api
export fit, update, update_data, transform, inverse_transform,
       fitted_params, predict, predict_mode, predict_mean, predict_median,
       evaluate, clean!

# model traits
export input_scitype, output_scitype, target_scitype,
       is_pure_julia, package_name, package_license,
       load_path, package_uuid, package_url,
       is_wrapper, supports_weights, supports_online,
       docstring, name, is_supervised,
       prediction_type, implemented_methods, hyperparameters,
       hyperparameter_types, hyperparameter_ranges

# data operations
export matrix, int, classes, decoder, table,
       nrows, selectrows, selectcols, select

# re-exports from (MLJ)ScientificTypes
export Scientific, Found, Unknown, Known, Finite, Infinite,
       OrderedFactor, Multiclass, Count, Continuous, Textual,
       Binary, ColorImage, GrayImage, Image, Table
export scitype, scitype_union, elscitype, nonmissing, trait
export coerce, coerce!, autotype, schema, info

# -------------------------------------------------------------------
# exports from MLJBase

export DeterministicNetwork, ProbabilisticNetwork, UnsupervisedNetwork,
       best, @load

# computational_resources.jl:
export default_resource

# equality.jl:
export is_same_except

# one_dimensional_ranges.jl:
export ParamRange, NumericRange, NominalRange, iterator, scale

# parameter_inspection.jl:
export params # note this is *not* an extension of StatsBase.params

# data.jl:
export reconstruct, levels_seen, container_type,
       partition, unpack, complement, restrict, corestrict

# utilities.jl:
export @set_defaults, flat_values, recursive_setproperty!,
       recursive_getproperty, pretty, unwind

# show.jl
export HANDLE_GIVEN_ID, @more, @constant, color_on, color_off

# distributions.jl:
export average

# tasks.jl:
export SupervisedTask, UnsupervisedTask, MLJTask,
       X_and_y, X_, y_, nfeatures

# info_dict.jl:
export info_dict

# datasets.jl:
export load_boston, load_ames, load_iris,
       load_reduced_ames, load_crabs,
       @load_boston, @load_ames, @load_iris,
       @load_reduced_ames, @load_crabs

# machines.jl:
export machine, Machine, AbstractMachine, fit!, report

# networks.jl:
export NodalMachine,  machines, source, node,sources, origins,
    rebind!, nodes, freeze!, thaw!, models, Node, AbstractNode, Source

# datasets_synthetics.jl
export make_blobs, make_moons, make_circles, make_regression

# composites.jl:
export machines, sources, anonymize!, @from_network, fitresults

# pipelines.jl:
export @pipeline

# resampling.jl:
export ResamplingStrategy, Holdout, CV, StratifiedCV,
       evaluate!, Resampler, PerformanceEvaluation

# openml.jl:
export OpenML

# -------------------------------------------------------------------
# exports from MLJBase specific to Measure (these may go in their
# specific MLJMeasureInterface package in some future)

# measures/registry.jl:
export measures, metadata_measure

# measure/measures.jl:
export orientation, reports_each_observation,
       is_feature_dependent, aggregation,
       aggregate, default_measure, value

# measures/continuous.jl:
export mav, mae, rms, rmsl, rmslp1, rmsp, l1, l2

# measures/confusion_matrix.jl:
export confusion_matrix, confmat

# measures/finite.jl
export cross_entropy, BrierScore, brier_score,
       misclassification_rate, mcr, accuracy,
       balanced_accuracy, bacc, bac,
       matthews_correlation, mcc

# measures/finite.jl -- binary order independent:
export auc, area_under_curve, roc_curve, roc

# measures/finite.jl -- binary order dependent:
export TruePositive, TrueNegative, FalsePositive, FalseNegative,
       TruePositiveRate, TrueNegativeRate, FalsePositiveRate,
       FalseNegativeRate, FalseDiscoveryRate, Precision, NPV, FScore,
       # standard synonyms
       TPR, TNR, FPR, FNR, FDR, PPV,
       Recall, Specificity, BACC,
       # instances and their synonyms
       truepositive, truenegative, falsepositive, falsenegative,
       true_positive, true_negative, false_positive, false_negative,
       truepositive_rate, truenegative_rate, falsepositive_rate,
       true_positive_rate, true_negative_rate, false_positive_rate,
       falsenegative_rate, negativepredictive_value,
       false_negative_rate, negative_predictive_value,
       positivepredictive_value, positive_predictive_value,
       tpr, tnr, fpr, fnr,
       falsediscovery_rate, false_discovery_rate, fdr, npv, ppv,
       recall, sensitivity, hit_rate, miss_rate,
       specificity, selectivity, f1score, fallout

# -------------------------------------------------------------------
# re-export from Random, StatsBase, Statistics, Distributions,
# CategoricalArrays, InvertedIndices:
export pdf, mode, median, mean, shuffle!, categorical, shuffle,
       levels, levels!, std, Not

# ===================================================================
## CONSTANTS

# the directory containing this file: (.../src/)
const MODULE_DIR = dirname(@__FILE__)

# horizontal space for field names in `MLJType` object display:
const COLUMN_WIDTH = 24
# how deep to display fields of `MLJType` objects:
const DEFAULT_SHOW_DEPTH = 0
const DEFAULT_AS_CONSTRUCTED_SHOW_DEPTH = 2
const INDENT = 4

const CategoricalElement = Union{CategoricalValue,CategoricalString}

const Arr = AbstractArray
const Vec = AbstractVector

const MMI = MLJModelInterface
const FI  = MLJModelInterface.FullInterface

# ===================================================================
# Computational Resource
# default_resource allows to switch the mode of parallelization

default_resource()    = DEFAULT_RESOURCE[]
default_resource(res) = (DEFAULT_RESOURCE[] = res)

# stub for @load (extended by MLJModels)
macro load end

# ===================================================================
# Includes

include("init.jl")
include("utilities.jl")
include("parameter_inspection.jl")
include("equality.jl")
include("show.jl")
include("info_dict.jl")

include("interface/data_utils.jl")
include("interface/model_api.jl")
include("interface/univariate_finite.jl")

include("distributions.jl")

include("machines.jl")

include("composition/networks.jl")
include("composition/composites.jl")
include("composition/pipelines.jl")
include("composition/pipeline_static.jl")
VERSION ≥ v"1.3.0-" && include("composition/arrows.jl")

include("operations.jl")

include("resampling.jl")

include("hyperparam/one_dimensional_ranges.jl")
include("hyperparam/one_dimensional_range_methods.jl")

include("data/data.jl")
include("data/datasets.jl")
include("data/datasets_synthetic.jl")

include("measures/measures.jl")
include("measures/registry.jl")

include("openml.jl")

end # module


text:
# Database
"""
    dbconnect()::Connection
Returns a connection to the burning_glass database in postgis_2.
"""
dbconnect() =
    Connection("""
               host = $db_host
               port = $db_port
               dbname = $dbname
               user = $db_user
               password = $db_pwd
               """)

"""
    varschema(variable)::String
Returns a suitable schema for the database declaration from the table information.
"""
function varschema(variable)
    varname = variable.column_id
    format = variable.format
    optional = variable.optional ? "" : "not null"
    if occursin("Character Varying", format)
        datatype = "varchar($(match(r"\d+", format).match))"
    elseif occursin("Character", format)
        datatype = "char($(match(r"\d+", format).match))"
    elseif occursin("Date", format)
        datatype = "date"
    elseif occursin("Float", format)
        precision, scale = getproperty.(eachmatch(r"\d+", format), :match)
        datatype = "numeric($precision, $scale)"
    elseif occursin("Integer", format)
        datatype = "numeric($(match(r"\d+", format).match))"
    end
    "$varname $datatype $optional"
end

"""
    drop_duplicates!(name::AbstractString)
Drop duplicate records from the table.
"""
function drop_duplicates!(name::AbstractString)
    conn = dbconnect()
    execute(conn, "create table onet.$(name)_clean (like onet.$name including all);")
    execute(conn, "insert into onet.$(name)_clean select distinct * from onet.$name;")
    execute(conn, "drop table onet.$(name)")
    execute(conn, "alter table onet.$(name)_clean rename to onet.$name")
    close(conn)
end

"""
    isdone(name::AbstractString)::Bool
Are all available records for that table in the database?
"""
function isdone(name::AbstractString)
    conn = dbconnect()
    output = execute(conn, "select count(*) from (select * from onet.$name) as tmp;") |>
        (tbl -> getproperty.(tbl, :count) |> first) |>
        (n -> n == tbls.count[findfirst(isequal(name), tbls.name)])
    close(conn)
    output
end


text:
using Distributed
addprocs(3)
@assert nprocs() == 4

@everywhere import Primes
using Primes

future = remotecall(factor, 2, 21883298135690819)
@assert isa(future, Future)
factors = fetch(future)
@assert collect(factors) == [93234719=>1,234711901=>1]

max_stars_repo_name: idrougge/ple

max_issues_repo_name: idrougge/ple

max_forks_repo_name: idrougge/ple


text:
"""
Module FiniteDifferences1D

Provides macros for 1-D finite differences computations. The dimensions x refers to the 1st and only array dimension. The module was designed to enable a math-close notation and, as a consequence, it does not allow for a nested invocation of the provided macros as, e.g., `@d(@d(A))` (instead, one can simply write `@d2(A)`). Additional macros can be straightforwardly added in the caller module to cover eventual missing functionality. The module is intended for usage with the module ParallelStencil.

# Usage
    using ParallelStencil.FiniteDifferences1D

# Macros

###### Differences
- [`@d`](@ref)
- [`@d2`](@ref)

###### Selection
- [`@all`](@ref)
- [`@inn`](@ref)

###### Averages
- [`@av`](@ref)

###### Others
- [`@maxloc`](@ref)
- [`@minloc`](@ref)

To see a description of a macro type `?<macroname>` (including the `@`).
"""
module FiniteDifferences1D
export @d, @d2
export @all, @inn
export @av
export @maxloc, @minloc
export @within

@doc "`@d(A)`: Compute differences between adjacent elements of `A`." :(@d)
@doc "`@d2(A)`: Compute the 2nd order differences between adjacent elements of `A`." :(@d2)
@doc "`@all(A)`: Select all elements of `A`. Corresponds to `A[:]`." :(@all)
@doc "`@inn(A)`: Select the inner elements of `A`. Corresponds to `A[2:end-1]`" :(@inn)
@doc "`@av(A)`: Compute averages between adjacent elements of `A`." :(@av)
@doc "`@maxloc(A)`: Compute the maximum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@maxloc)
@doc "`@minloc(A)`: Compute the minimum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@minloc)

import ..ParallelStencil: INDICES, WITHIN_DOC
const ix = INDICES[1]
const ixi = :($ix+1)

macro      d(A::Symbol)  esc(:( $A[$ix+1] - $A[$ix] )) end
macro     d2(A::Symbol)  esc(:( ($A[$ixi+1] - $A[$ixi])  -  ($A[$ixi] - $A[$ixi-1]) )) end
macro    all(A::Symbol)  esc(:( $A[$ix  ] )) end
macro    inn(A::Symbol)  esc(:( $A[$ixi ] )) end
macro     av(A::Symbol)  esc(:(($A[$ix] + $A[$ix+1] )*0.5 )) end
macro maxloc(A::Symbol)  esc(:( max( max($A[$ixi-1], $A[$ixi+1]), $A[$ixi] ) )) end
macro minloc(A::Symbol)  esc(:( min( min($A[$ixi-1], $A[$ixi+1]), $A[$ixi] ) )) end

@doc WITHIN_DOC
macro within(macroname::String, A::Symbol)
    if     macroname == "@all"  esc(  :($ix<=size($A,1)  )  )
    elseif macroname == "@inn"  esc(  :($ix<=size($A,1)-2)  )
    else error("unkown macroname: $macroname. If you want to add your own assignement macros, overwrite the macro 'within(macroname::String, A::Symbol)'; to still use the exising macro within as well call ParallelStencil.FiniteDifferences{1|2|3}D.@within(macroname, A) at the end.")
    end
end

end # Module FiniteDifferences1D

"""
Module FiniteDifferences2D

Provides macros for 2-D finite differences computations. The dimensions x and y refer to the 1st and 2nd array dimensions, respectively. The module was designed to enable a math-close notation and, as a consequence, it does not allow for a nested invocation of the provided macros as, e.g., `@inn_y(@d_xa(A))` (instead, one can simply write `@d_xi(A)`). Additional macros can be straightforwardly added in the caller module to cover eventual missing functionality. The module is intended for usage with the module ParallelStencil.

# Usage
    using ParallelStencil.FiniteDifferences2D

# Macros

###### Differences
- [`@d_xa`](@ref)
- [`@d_ya`](@ref)
- [`@d_xi`](@ref)
- [`@d_yi`](@ref)
- [`@d2_xi`](@ref)
- [`@d2_yi`](@ref)

###### Selection
- [`@all`](@ref)
- [`@inn`](@ref)
- [`@inn_x`](@ref)
- [`@inn_y`](@ref)

###### Averages
- [`@av`](@ref)
- [`@av_xa`](@ref)
- [`@av_ya`](@ref)
- [`@av_xi`](@ref)
- [`@av_yi`](@ref)

###### Others
- [`@maxloc`](@ref)
- [`@minloc`](@ref)

To see a description of a macro type `?<macroname>` (including the `@`).
"""
module FiniteDifferences2D
export @d_xa, @d_ya, @d_xi, @d_yi, @d2_xi, @d2_yi
export @all, @inn, @inn_x, @inn_y
export @av, @av_xa, @av_ya, @av_xi, @av_yi
export @maxloc, @minloc
export @within

@doc "`@d_xa(A)`: Compute differences between adjacent elements of `A` along the dimension x." :(@d_xa)
@doc "`@d_ya(A)`: Compute differences between adjacent elements of `A` along the dimension y." :(@d_ya)
@doc "`@d_xi(A)`: Compute differences between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_y(@d_xa(A))`." :(@d_xi)
@doc "`@d_yi(A)`: Compute differences between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_x(@d_ya(A))`." :(@d_yi)
@doc "`@d2_xi(A)`: Compute the 2nd order differences between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_y(@d2_xa(A))`." :(@d2_xi)
@doc "`@d2_yi(A)`: Compute the 2nd order differences between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_x(@d2_ya(A))`." :(@d2_yi)
@doc "`@all(A)`: Select all elements of `A`. Corresponds to `A[:,:]`." :(@all)
@doc "`@inn(A)`: Select the inner elements of `A`. Corresponds to `A[2:end-1,2:end-1]`." :(@inn)
@doc "`@inn_x(A)`: Select the inner elements of `A` in dimension x. Corresponds to `A[2:end-1,:]`." :(@inn_x)
@doc "`@inn_y(A)`: Select the inner elements of `A` in dimension y. Corresponds to `A[:,2:end-1]`." :(@inn_y)
@doc "`@av(A)`: Compute averages between adjacent elements of `A` along the dimensions x and y." :(@av)
@doc "`@av_xa(A)`: Compute averages between adjacent elements of `A` along the dimension x." :(@av_xa)
@doc "`@av_ya(A)`: Compute averages between adjacent elements of `A` along the dimension y." :(@av_ya)
@doc "`@av_xi(A)`: Compute averages between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_y(@av_xa(A))`." :(@av_xi)
@doc "`@av_yi(A)`: Compute averages between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_x(@av_ya(A))`." :(@av_yi)
@doc "`@maxloc(A)`: Compute the maximum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@maxloc)
@doc "`@minloc(A)`: Compute the minimum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@minloc)

import ..ParallelStencil: INDICES, WITHIN_DOC
ix, iy = INDICES[1], INDICES[2]
ixi, iyi = :($ix+1), :($iy+1)

macro   d_xa(A::Symbol)  esc(:( $A[$ix+1,$iy  ] - $A[$ix  ,$iy ] )) end
macro   d_ya(A::Symbol)  esc(:( $A[$ix  ,$iy+1] - $A[$ix  ,$iy ] )) end
macro   d_xi(A::Symbol)  esc(:( $A[$ix+1,$iyi ] - $A[$ix  ,$iyi] )) end
macro   d_yi(A::Symbol)  esc(:( $A[$ixi ,$iy+1] - $A[$ixi ,$iy ] )) end
macro  d2_xi(A::Symbol)  esc(:( ($A[$ixi+1,$iyi  ] - $A[$ixi ,$iyi])  -  ($A[$ixi ,$iyi] - $A[$ixi-1,$iyi  ]) )) end
macro  d2_yi(A::Symbol)  esc(:( ($A[$ixi  ,$iyi+1] - $A[$ixi ,$iyi])  -  ($A[$ixi ,$iyi] - $A[$ixi  ,$iyi-1]) )) end
macro    all(A::Symbol)  esc(:( $A[$ix  ,$iy  ] )) end
macro    inn(A::Symbol)  esc(:( $A[$ixi ,$iyi ] )) end
macro  inn_x(A::Symbol)  esc(:( $A[$ixi ,$iy  ] )) end
macro  inn_y(A::Symbol)  esc(:( $A[$ix  ,$iyi ] )) end
macro     av(A::Symbol)  esc(:(($A[$ix  ,$iy  ] + $A[$ix+1,$iy  ] + $A[$ix,$iy+1] + $A[$ix+1,$iy+1])*0.25 )) end
macro  av_xa(A::Symbol)  esc(:(($A[$ix  ,$iy  ] + $A[$ix+1,$iy  ] )*0.5 )) end
macro  av_ya(A::Symbol)  esc(:(($A[$ix  ,$iy  ] + $A[$ix  ,$iy+1] )*0.5 )) end
macro  av_xi(A::Symbol)  esc(:(($A[$ix  ,$iyi ] + $A[$ix+1,$iyi ] )*0.5 )) end
macro  av_yi(A::Symbol)  esc(:(($A[$ixi ,$iy  ] + $A[$ixi ,$iy+1] )*0.5 )) end
macro maxloc(A::Symbol)  esc(:( max( max( max($A[$ixi-1,$iyi  ], $A[$ixi+1,$iyi  ])  , $A[$ixi  ,$iyi  ] ),
                                          max($A[$ixi  ,$iyi-1], $A[$ixi  ,$iyi+1]) ) )) end
macro minloc(A::Symbol)  esc(:( min( min( min($A[$ixi-1,$iyi  ], $A[$ixi+1,$iyi  ])  , $A[$ixi  ,$iyi  ] ),
                                          min($A[$ixi  ,$iyi-1], $A[$ixi  ,$iyi+1]) ) )) end

@doc WITHIN_DOC
macro within(macroname::String, A::Symbol)
    if     macroname == "@all"    esc(  :($ix<=size($A,1)   && $iy<=size($A,2)  )  )
    elseif macroname == "@inn"    esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)-2)  )
    elseif macroname == "@inn_x"  esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)  )  )
    elseif macroname == "@inn_y"  esc(  :($ix<=size($A,1)   && $iy<=size($A,2)-2)  )
    else error("unkown macroname: $macroname. If you want to add your own assignement macros, overwrite the macro 'within(macroname::String, A::Symbol)'; to still use the exising macro within as well call ParallelStencil.FiniteDifferences{1|2|3}D.@within(macroname, A) at the end.")
    end
end

end # Module FiniteDifferences2D

"""
Module FiniteDifferences3D

Provides macros for 3-D finite differences computations. The dimensions x, y and z refer to the 1st, 2nd and 3rd array dimensions, respectively. The module was designed to enable a math-close notation and, as a consequence, it does not allow for a nested invocation of the provided macros as, e.g., `@inn_z(@inn_y(@d_xa(A)))` (instead, one can simply write `@d_xi(A)`). Additional macros can be straightforwardly added in the caller module to cover eventual missing functionality. The module is intended for usage with the module ParallelStencil.

# Usage
    using ParallelStencil.FiniteDifferences3D

# Macros

###### Differences
- [`@d_xa`](@ref)
- [`@d_ya`](@ref)
- [`@d_za`](@ref)
- [`@d_xi`](@ref)
- [`@d_yi`](@ref)
- [`@d_zi`](@ref)
- [`@d2_xi`](@ref)
- [`@d2_yi`](@ref)
- [`@d2_zi`](@ref)

###### Selection
- [`@all`](@ref)
- [`@inn`](@ref)
- [`@inn_x`](@ref)
- [`@inn_y`](@ref)
- [`@inn_z`](@ref)
- [`@inn_xy`](@ref)
- [`@inn_xz`](@ref)
- [`@inn_yz`](@ref)

###### Averages
- [`@av`](@ref)
- [`@av_xa`](@ref)
- [`@av_ya`](@ref)
- [`@av_za`](@ref)
- [`@av_xi`](@ref)
- [`@av_yi`](@ref)
- [`@av_zi`](@ref)
- [`@av_xya`](@ref)
- [`@av_xza`](@ref)
- [`@av_yza`](@ref)
- [`@av_xyi`](@ref)
- [`@av_xzi`](@ref)
- [`@av_yzi`](@ref)

###### Others
- [`@maxloc`](@ref)
- [`@minloc`](@ref)

To see a description of a macro type `?<macroname>` (including the `@`).
"""
module FiniteDifferences3D
export @d_xa, @d_ya, @d_za, @d_xi, @d_yi, @d_zi, @d2_xi, @d2_yi, @d2_zi
export @all, @inn, @inn_x, @inn_y, @inn_z, @inn_xy, @inn_xz, @inn_yz
export @av, @av_xa, @av_ya, @av_za, @av_xi, @av_yi, @av_zi, @av_xya, @av_xza, @av_yza, @av_xyi, @av_xzi, @av_yzi #, @av_xya2, @av_xza2, @av_yza2
export @maxloc, @minloc
export @within

@doc "`@d_xa(A)`: Compute differences between adjacent elements of `A` along the dimension x." :(@d_xa)
@doc "`@d_ya(A)`: Compute differences between adjacent elements of `A` along the dimension y." :(@d_ya)
@doc "`@d_za(A)`: Compute differences between adjacent elements of `A` along the dimension z." :(@d_za)
@doc "`@d_xi(A)`: Compute differences between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_yz(@d_xa(A))`." :(@d_xi)
@doc "`@d_yi(A)`: Compute differences between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xz(@d_ya(A))`." :(@d_yi)
@doc "`@d_zi(A)`: Compute differences between adjacent elements of `A` along the dimension z and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xy(@d_za(A))`." :(@d_zi)
@doc "`@d2_xi(A)`: Compute the 2nd order differences between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_yz(@d2_xa(A))`." :(@d2_xi)
@doc "`@d2_yi(A)`: Compute the 2nd order differences between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xz(@d2_ya(A))`." :(@d2_yi)
@doc "`@d2_zi(A)`: Compute the 2nd order differences between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xy(@d2_za(A))`." :(@d2_zi)
@doc "`@all(A)`: Select all elements of `A`. Corresponds to `A[:,:,:]`." :(@all)
@doc "`@inn(A)`: Select the inner elements of `A`. Corresponds to `A[2:end-1,2:end-1,2:end-1]`." :(@inn)
@doc "`@inn_x(A)`: Select the inner elements of `A` in dimension x. Corresponds to `A[2:end-1,:,:]`." :(@inn_x)
@doc "`@inn_y(A)`: Select the inner elements of `A` in dimension y. Corresponds to `A[:,2:end-1,:]`." :(@inn_y)
@doc "`@inn_z(A)`: Select the inner elements of `A` in dimension z. Corresponds to `A[:,:,2:end-1]`." :(@inn_z)
@doc "`@inn_xy(A)`: Select the inner elements of `A` in dimensions x and y. Corresponds to `A[2:end-1,2:end-1,:]`." :(@inn_xy)
@doc "`@inn_xz(A)`: Select the inner elements of `A` in dimensions x and z. Corresponds to `A[2:end-1,:,2:end-1]`." :(@inn_xz)
@doc "`@inn_yz(A)`: Select the inner elements of `A` in dimensions y and z. Corresponds to `A[:,2:end-1,2:end-1]`." :(@inn_yz)
@doc "`@av(A)`: Compute averages between adjacent elements of `A` along the dimensions x and y and z." :(@av)
@doc "`@av_xa(A)`: Compute averages between adjacent elements of `A` along the dimension x." :(@av_xa)
@doc "`@av_ya(A)`: Compute averages between adjacent elements of `A` along the dimension y." :(@av_ya)
@doc "`@av_za(A)`: Compute averages between adjacent elements of `A` along the dimension z." :(@av_za)
@doc "`@av_xi(A)`: Compute averages between adjacent elements of `A` along the dimension x and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_yz(@av_xa(A))`." :(@av_xi)
@doc "`@av_yi(A)`: Compute averages between adjacent elements of `A` along the dimension y and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xz(@av_ya(A))`." :(@av_yi)
@doc "`@av_zi(A)`: Compute averages between adjacent elements of `A` along the dimension z and select the inner elements of `A` in the remaining dimensions. Corresponds to `@inn_xy(@av_za(A))`." :(@av_zi)
@doc "`@av_xya(A)`: Compute averages between adjacent elements of `A` along the dimensions x and y." :(@av_xya)
@doc "`@av_xza(A)`: Compute averages between adjacent elements of `A` along the dimensions x and z." :(@av_xza)
@doc "`@av_yza(A)`: Compute averages between adjacent elements of `A` along the dimensions y and z." :(@av_yza)
@doc "`@av_xyi(A)`: Compute averages between adjacent elements of `A` along the dimensions x and y and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_z(@av_xya(A))`." :(@av_xyi)
@doc "`@av_xzi(A)`: Compute averages between adjacent elements of `A` along the dimensions x and z and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_y(@av_xza(A))`." :(@av_xzi)
@doc "`@av_yzi(A)`: Compute averages between adjacent elements of `A` along the dimensions y and z and select the inner elements of `A` in the remaining dimension. Corresponds to `@inn_x(@av_yza(A))`." :(@av_yzi)
@doc "`@maxloc(A)`: Compute the maximum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@maxloc)
@doc "`@minloc(A)`: Compute the minimum between 2nd order adjacent elements of `A`, using a moving window of size 3." :(@minloc)

import ..ParallelStencil: INDICES, WITHIN_DOC
ix, iy, iz = INDICES[1], INDICES[2], INDICES[3]
ixi, iyi, izi = :($ix+1), :($iy+1), :($iz+1)

macro   d_xa(A::Symbol)  esc(:( $A[$ix+1,$iy  ,$iz  ] - $A[$ix  ,$iy  ,$iz  ] )) end
macro   d_ya(A::Symbol)  esc(:( $A[$ix  ,$iy+1,$iz  ] - $A[$ix  ,$iy  ,$iz  ] )) end
macro   d_za(A::Symbol)  esc(:( $A[$ix  ,$iy  ,$iz+1] - $A[$ix  ,$iy  ,$iz  ] )) end
macro   d_xi(A::Symbol)  esc(:( $A[$ix+1,$iyi ,$izi ] - $A[$ix  ,$iyi ,$izi ] )) end
macro   d_yi(A::Symbol)  esc(:( $A[$ixi ,$iy+1,$izi ] - $A[$ixi ,$iy  ,$izi ] )) end
macro   d_zi(A::Symbol)  esc(:( $A[$ixi ,$iyi ,$iz+1] - $A[$ixi ,$iyi ,$iz  ] )) end
macro  d2_xi(A::Symbol)  esc(:( ($A[$ixi+1,$iyi  ,$izi  ] - $A[$ixi ,$iyi ,$izi ])  -  ($A[$ixi ,$iyi ,$izi ] - $A[$ixi-1,$iyi  ,$izi  ]) )) end
macro  d2_yi(A::Symbol)  esc(:( ($A[$ixi  ,$iyi+1,$izi  ] - $A[$ixi ,$iyi ,$izi ])  -  ($A[$ixi ,$iyi ,$izi ] - $A[$ixi  ,$iyi-1,$izi  ]) )) end
macro  d2_zi(A::Symbol)  esc(:( ($A[$ixi  ,$iyi  ,$izi+1] - $A[$ixi ,$iyi ,$izi ])  -  ($A[$ixi ,$iyi ,$izi ] - $A[$ixi  ,$iyi  ,$izi-1]) )) end
macro    all(A::Symbol)  esc(:( $A[$ix  ,$iy  ,$iz  ] )) end
macro    inn(A::Symbol)  esc(:( $A[$ixi ,$iyi ,$izi ] )) end
macro  inn_x(A::Symbol)  esc(:( $A[$ixi ,$iy  ,$iz  ] )) end
macro  inn_y(A::Symbol)  esc(:( $A[$ix  ,$iyi ,$iz  ] )) end
macro  inn_z(A::Symbol)  esc(:( $A[$ix  ,$iy  ,$izi ] )) end
macro inn_xy(A::Symbol)  esc(:( $A[$ixi ,$iyi ,$iz  ] )) end
macro inn_xz(A::Symbol)  esc(:( $A[$ixi ,$iy  ,$izi ] )) end
macro inn_yz(A::Symbol)  esc(:( $A[$ix  ,$iyi ,$izi ] )) end
macro     av(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix+1,$iy  ,$iz  ] +
                                $A[$ix+1,$iy+1,$iz  ] + $A[$ix+1,$iy+1,$iz+1] +
                                $A[$ix  ,$iy+1,$iz+1] + $A[$ix  ,$iy  ,$iz+1] +
                                $A[$ix+1,$iy  ,$iz+1] + $A[$ix  ,$iy+1,$iz  ] )*0.125)) end
macro  av_xa(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix+1,$iy  ,$iz  ] )*0.5 )) end
macro  av_ya(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix  ,$iy+1,$iz  ] )*0.5 )) end
macro  av_za(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix  ,$iy  ,$iz+1] )*0.5 )) end
macro  av_xi(A::Symbol)  esc(:(($A[$ix  ,$iyi ,$izi ] + $A[$ix+1,$iyi ,$izi ] )*0.5 )) end
macro  av_yi(A::Symbol)  esc(:(($A[$ixi ,$iy  ,$izi ] + $A[$ixi ,$iy+1,$izi ] )*0.5 )) end
macro  av_zi(A::Symbol)  esc(:(($A[$ixi ,$iyi ,$iz  ] + $A[$ixi ,$iyi ,$iz+1] )*0.5 )) end
macro av_xya(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix+1,$iy  ,$iz  ] +
                                $A[$ix  ,$iy+1,$iz  ] + $A[$ix+1,$iy+1,$iz  ] )*0.25 )) end
macro av_xza(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix+1,$iy  ,$iz  ] +
                                $A[$ix  ,$iy  ,$iz+1] + $A[$ix+1,$iy  ,$iz+1] )*0.25 )) end
macro av_yza(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$iz  ] + $A[$ix  ,$iy+1,$iz  ] +
                                $A[$ix  ,$iy  ,$iz+1] + $A[$ix  ,$iy+1,$iz+1] )*0.25 )) end
macro av_xyi(A::Symbol)  esc(:(($A[$ix  ,$iy  ,$izi ] + $A[$ix+1,$iy  ,$izi ] +
                                $A[$ix  ,$iy+1,$izi ] + $A[$ix+1,$iy+1,$izi ] )*0.25 )) end
macro av_xzi(A::Symbol)  esc(:(($A[$ix  ,$iyi ,$iz  ] + $A[$ix+1,$iyi ,$iz  ] +
                                $A[$ix  ,$iyi ,$iz+1] + $A[$ix+1,$iyi ,$iz+1] )*0.25 )) end
macro av_yzi(A::Symbol)  esc(:(($A[$ixi ,$iy  ,$iz  ] + $A[$ixi ,$iy+1,$iz  ] +
                                $A[$ixi ,$iy  ,$iz+1] + $A[$ixi ,$iy+1,$iz+1] )*0.25 )) end
macro maxloc(A::Symbol)  esc(:( max( max( max( max($A[$ixi-1,$iyi  ,$izi  ], $A[$ixi+1,$iyi  ,$izi  ])  , $A[$ixi  ,$iyi  ,$izi  ] ),
                                               max($A[$ixi  ,$iyi-1,$izi  ], $A[$ixi  ,$iyi+1,$izi  ]) ),
                                               max($A[$ixi  ,$iyi  ,$izi-1], $A[$ixi  ,$iyi  ,$izi+1]) ) )) end
macro minloc(A::Symbol)  esc(:( min( min( min( min($A[$ixi-1,$iyi  ,$izi  ], $A[$ixi+1,$iyi  ,$izi  ])  , $A[$ixi  ,$iyi  ,$izi  ] ),
                                               min($A[$ixi  ,$iyi-1,$izi  ], $A[$ixi  ,$iyi+1,$izi  ]) ),
                                               min($A[$ixi  ,$iyi  ,$izi-1], $A[$ixi  ,$iyi  ,$izi+1]) ) )) end

@doc WITHIN_DOC
macro within(macroname::String, A::Symbol)
    if     macroname == "@all"     esc(  :($ix<=size($A,1)   && $iy<=size($A,2)   && $iz<=size($A,3)  )  )
    elseif macroname == "@inn"     esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)-2 && $iz<=size($A,3)-2)  )
    elseif macroname == "@inn_x"   esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)   && $iz<=size($A,3)  )  )
    elseif macroname == "@inn_y"   esc(  :($ix<=size($A,1)   && $iy<=size($A,2)-2 && $iz<=size($A,3)  )  )
    elseif macroname == "@inn_z"   esc(  :($ix<=size($A,1)   && $iy<=size($A,2)   && $iz<=size($A,3)-2)  )
    elseif macroname == "@inn_xy"  esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)-2 && $iz<=size($A,3)  )  )
    elseif macroname == "@inn_xz"  esc(  :($ix<=size($A,1)-2 && $iy<=size($A,2)   && $iz<=size($A,3)-2)  )
    elseif macroname == "@inn_yz"  esc(  :($ix<=size($A,1)   && $iy<=size($A,2)-2 && $iz<=size($A,3)-2)  )
    else error("unkown macroname: $macroname. If you want to add your own assignement macros, overwrite the macro 'within(macroname::String, A::Symbol)'; to still use the exising macro within as well call ParallelStencil.FiniteDifferences{1|2|3}D.@within(macroname, A) at the end.")
    end
end

end # Module FiniteDifferences3D


text:
struct TransBoolVariate <: AbstractBoolVariate
    name::String
    baseboolvar::AbstractBoolVariate
    transform::Function
end

Base.length(var::TransBoolVariate) = length(var.baseboolvar)

function slice(boolvar::TransBoolVariate, fromobs::Integer, toobs::Integer, slicelength::Integer)
    baseboolvar = boolvar.baseboolvar
    f = boolvar.transform
    slicelength = verifyslicelength(fromobs, toobs, slicelength)
    slices = slice(baseboolvar, fromobs, toobs, slicelength)
    mapslice(f, slices, slicelength, Bool)
end

function Base.broadcast(f, baseboolvar::AbstractBoolVariate)
    Trans2BoolVariate("$(f)($(getname(baseboolvar)))", baseboolvar, f)
end

max_stars_repo_name: Statfactory/JuML

max_issues_repo_name: Statfactory/JuML

max_forks_repo_name: Statfactory/JuML


text:
using AdventOfCode2018
using AdventOfCode2018.Day11

input = get_aoc_input(11)
serial = parse(Int, input)

println("Solution part 1", part_1(serial))
println("Solution part 2", part_2(serial))


text:
using MacroTools

# Threading macros

import Base: replace

export @>, @>>, @as, @_, @switch, @or, @dotimes, @oncethen, @defonce, @with, @errs,
  @forward, @iter

"""
A switch statement of sorts:

    @switch x begin
      1; "x equals one!"
      2; "x equals two!"
      "x equals something else!"
    end

However, it's a bit more general than a regular switch in that
you can test more than just equality:

    @switch isa(x, _) begin
      Integer; "x is an integer!"
      FloatingPoint; "x is a float!"
      "x is something else!"
    end

    @switch _ begin
      a > b;  "more than!"
      a < b;  "less than!"
      a == b; "equal!"       # Note that this level of enthusiasm is not mandatory.
    end

Where `_` is replaced by the value for testing in each case. The final
expression, if there is one, is used as the default value; if there is
no default and nothing matches an error will be thrown.
"""
macro switch(args...)
  test, exprs = splitswitch(args...)
  length(exprs) == 0 && return nothing
  length(exprs) == 1 && return esc(exprs[1])

  test_expr(test, val) =
    test == :_      ? val :
    isa(test, Expr) ? :(let _ = $val; $test; end) :
                      :($test==$val)

  thread(val, yes, no) = :($(test_expr(test, val)) ? $yes : $no)
  thread(val, yes) = thread(val, yes, :(error($"No match for $test in @switch")))
  thread(val, yes, rest...) = thread(val, yes, thread(rest...))

  esc(thread(exprs...))
end

function splitswitch(test, exprs)
  @assert isexpr(exprs, :block) "@switch requires a begin block"
  test, rmlines(exprs).args
end

function splitswitch(ex)
  test = ex.args[1]
  exprs = c()
  for ex in ex.args[2:end]
    isexpr(ex, :->) ?
      push!(exprs, map(unblock, ex.args)...) :
      push!(exprs, ex)
  end
  test, exprs
end

"""
The threading macro is like a more flexible version of the `|>` operator.

    @> x f = f(x)
    @> x g f == f(g(x))
    @> x a b c d e == e(d(c(b(a(x)))))

Unlike |>, functions can have arguments - the value
preceding a function will be treated as its first argument

    @> x g(y, z) f == f(g(x, y, z))

    @> x g f(y, z) == f(g(x), y, z)

See also `@>>`, `@as`.
"""
macro >(exs...)
  thread(x) = isexpr(x, :block) ? thread(rmlines(x).args...) : x

  @static if VERSION < v"0.7"

    thread(x, ex) =
    isexpr(ex, :call, :macrocall) ? Expr(ex.head, ex.args[1], x, ex.args[2:end]...) :
    @capture(ex, f_.(xs__))       ? :($f.($x, $(xs...))) :
    isexpr(ex, :block)            ? thread(x, rmlines(ex).args...) :
    Expr(:call, ex, x)

  else

    thread(x, ex) =
    isexpr(ex, :macrocall)        ? Expr(ex.head, ex.args[1], ex.args[2], x, ex.args[3:end]...) :
    isexpr(ex, :call,)            ? Expr(ex.head, ex.args[1], x, ex.args[2:end]...) :
    @capture(ex, f_.(xs__))       ? :($f.($x, $(xs...))) :
    isexpr(ex, :block)            ? thread(x, rmlines(ex).args...) :
    Expr(:call, ex, x)

  end

  thread(x, exs...) = reduce(thread, exs, init=x)

  esc(thread(exs...))
end

"""
Same as `@>`, but threads the last argument.

  @>> x g(y, z) f == f(g(y, z, x))

  @>> x g f(y, z) == f(y, z, g(x))
"""
macro >>(exs...)
  thread(x) = isexpr(x, :block) ? thread(rmlines(x).args...) : x

  thread(x, ex) =
    isexpr(ex, Symbol, :->)       ? Expr(:call, ex, x) :
    isexpr(ex, :call, :macrocall) ? Expr(ex.head, ex.args..., x) :
    @capture(ex, f_.(xs__))       ? :($f.($(xs...), $x)) :
    isexpr(ex, :block)            ? thread(x, rmlines(ex).args...) :
                                    error("Unsupported expression $ex in @>>")

  thread(x, exs...) = reduce(thread, exs; init=x)

  esc(thread(exs...))
end

"""
# @as lets you name the threaded argmument
@as _ x f(_, y) g(z, _) == g(z, f(x, y))

# All threading macros work over begin blocks

@as x 2 begin
 x^2
 x+2
end == 6

`@_` is a version of `@as` which defaults to `_` as the argument name.
"""
macro as(as, exs...)
  thread(x) = isexpr(x, :block) ? thread(rmlines(x).args...) : x

  thread(x, ex) =
    isexpr(ex, Symbol, :->) ? Expr(:call, ex, x) :
    isexpr(ex, :block)      ? thread(x, rmlines(ex).args...) :
    :(let $as = $x
        $ex
      end)

  thread(x, exs...) = reduce((x, ex) -> thread(x, ex), exs, init=x)

  esc(thread(exs...))
end

@static if VERSION < v"0.7"
  """
  Same as `@as` but uses `_` as the argmument name.
  """
  macro _(args...)
    :(@as $(esc(:_)) $(map(esc, args)...))
  end
end

macro or(exs...)
  thread(x) = isexpr(x, :block) ? thread(rmlines(x).args...) : esc(x)

  thread(x, xs...) =
    :(let x = $(esc(x))
        !(x == nothing || x == false) ? x : $(thread(xs...))
      end)

  thread(exs...)
end

"Repeat `body` `n` times."
macro dotimes(n, body)
  quote
    for i = 1:$(esc(n))
      $(esc(body))
    end
  end
end

"""
A do-while loop – executes the while loop once regardless of the
condition, then tests the condition before subsequen iterations.
"""
macro oncethen(expr::Expr)
  @assert expr.head == :while
  esc(quote
    $(expr.args[2]) # body of loop
    $expr # loop
  end)
end

"""
Stop Julia from complaining about redifined consts/types –

    @defonce type MyType
      ...
    end
    or
    @defonce const pi = 3.14
"""
macro defonce(def)
  name = namify(isexpr(def, :type) ? def.args[2] : def)

  :(if !isdefined($(Expr(:quote, name)))
      $(esc(def))
    end)
end

"""
End-less let block, e.g.

    @with (x = 1, y = 2),
      x+y
"""
macro with(ex)
  @capture(ex, ((bindings__,), body_)) || error("Invalid expression @with $ex")
  ex = :(let $(bindings...)
           $body
         end)
  return esc(ex)
end

# Other syntax

export c, s, d, @d
c(xs...) = Any[xs...]
s(xs...) = Set{Any}(xs)
d(xs...) = Dict{Any, Any}(xs...)

"""
Creates a typed dictionary, e.g.

   julia> @d(a=>1,b=>2)
   Dict{Any,Any} with 2 entries:
     :a => 1
     :b => 2
"""
macro d(xs...)
  :(Dict{Any, Any}($(map(esc, xs)...)))
end

macro errs(ex)
  :(try $(esc(ex))
    catch e
      showerror(stderr, e, catch_backtrace())
      println(stderr)
    end)
end

"""
    @forward T.x functions...

Define methods for `functions` on type `T`, which call the relevant function
on the field `x`.

# Example

```julia
struct Wrapper
    x
end

@forward Wrapper.x  Base.sqrt                                  # now sqrt(Wrapper(4.0)) == 2.0
@forward Wrapper.x  Base.length, Base.getindex, Base.iterate   # several forwarded functions are put in a tuple
@forward Wrapper.x (Base.length, Base.getindex, Base.iterate)  # equivalent to above
```
"""
macro forward(ex, fs)
  @capture(ex, T_.field_) || error("Syntax: @forward T.x f, g, h")
  T = esc(T)
  fs = isexpr(fs, :tuple) ? map(esc, fs.args) : [esc(fs)]
  :($([:($f(x::$T, args...) = (Base.@_inline_meta; $f(x.$field, args...)))
       for f in fs]...);
    nothing)
end

# Forwarding iteration

struct SubIter{I,S}
  iter::I
  state::S
end

# Julia#16096

macro iter(ex)
  @capture(ex, x_::T_ -> it_) || error("Use @iter x::T -> y ...")
  @capture(it, $x.f_) &&
    return :(@forward $(esc(T)).$f Base.iterate, Base.iterate)
  quote
    @inline function Base.iterate($x::$T)
      it = $it
      Lazy.SubIter(it, Base.iterate(it))
    end
    @inline function Base.iterate(::$T, sub::Lazy.SubIter)
      next, state = Base.iterate(sub.iter, sub.state)
      next == nothing && return nothing
      next, Lazy.SubIter(sub.iter, state)
    end
  end |> esc
end

# Init macro

export @init

function initm(ex)
  quote

    if !isdefined(@__MODULE__, :__inits__)
      const $(esc(:__inits__)) = Function[]
    end
    if !isdefined(@__MODULE__, :__init__)
      function $(esc(:__init__))()
        for f in $(esc(:__inits__))
          f()
        end
      end
    end

    push!($(esc(:__inits__)), () -> $(esc(ex)))
    nothing
  end
end

macro init(args...)
  initm(args...)
end


text:
module ZulipSnippetBot

using HTTP
using JSON3
using StructTypes
using Dates
using SQLite
using DBInterface

include("migrations.jl")
export setupbot!

const SYMBOLS = ('A':'Z'..., 'a':'z'..., '0':'9'...)
codeid() = rand(SYMBOLS, 10) |> join
currentts() = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS")

struct Opts
    token::String
    host::String
    port::Int
end
const OPTS = Ref(Opts("", "127.0.0.1", 9174))

function setupbot!(; token = OPTS[].token,
                     host = OPTS[].host,
                     port = OPTS[].port)
    OPTS[] = Opts(token, host, port)
end

########################################
# Incoming messages processing
########################################
mutable struct Message
    sender_id::Int
end
Message() = Message(-1)
StructTypes.StructType(::Type{Message}) = StructTypes.Mutable()

mutable struct ZulipRequest
    data::String
    token::String
    message::Message
end
ZulipRequest() = ZulipRequest("", "", Message())
StructTypes.StructType(::Type{ZulipRequest}) = StructTypes.Mutable()

########################################
# Database functionality
########################################

function get_by_code(db, code_id)
    query = """
    SELECT snippet
    FROM snippets
    WHERE code = ?
    """

    stmt = SQLite.Stmt(db, query)
    DBInterface.execute(stmt, (code_id, ))
end

function store_snippet(db, user_id, code_id, snippet, tags)
    query = """
    INSERT INTO snippets(code, user_id, snippet, tags, created)
    VALUES (?, ?, ?, ?, ?)
    """

    ts = currentts()
    stmt = SQLite.Stmt(db, query)
    DBInterface.execute(stmt, (code_id, user_id, snippet, tags, ts))
end

function load_snippets(db, user_id)
    query = """
    SELECT code, snippet, tags
    FROM snippets
    WHERE user_id = ?
    """

    stmt = SQLite.Stmt(db, query)
    DBInterface.execute(stmt, (user_id, ))
end

########################################
# Processing
########################################
function process(obj::ZulipRequest, db, opts)
    obj.data = strip(obj.data)
    if obj.message.sender_id < 0 || isempty(obj.data) || isempty(obj.token)
        return JSON3.write((; content = "Wrong message"))
    end
    if obj.token != opts.token
        return JSON3.write((; content = "Incorrect token, verify SnippetBot server configuration"))
    end

    if obj.data[1] == '@'
        m = match(r"^@[^\s]+\s+(.*)$"s, obj.data)
        isnothing(m) && return JSON3.write((; content = "Wrong message"))
        obj.data = m[1]
    end

    resp = if startswith(obj.data, "show")
        process_show(obj, db, opts)
    elseif startswith(obj.data, "list")
        process_list(obj, db, opts)
    elseif startswith(obj.data, "save")
        process_save(obj, db, opts)
    elseif startswith(obj.data, "help")
        process_help(obj, db, opts)
    else
        "Unknown command"
    end

    return JSON3.write((; content = resp))
end

function process_show(obj::ZulipRequest, db, opts)
    @info "show"
    m = match(r"^show\s+([A-Za-z0-9]+)$", obj.data)
    if isnothing(m)
        return "No codeid is given in `show` command"
    end
    try
        r = map(x -> x.snippet, get_by_code(db, m[1]))
        if isempty(r)
            return "Codeid $(m[1]) is not found"
        end
        return r[1]
    catch err
        @error exception=(err, catch_backtrace())
        return "Server error, please try one more time. Later. Sorry."
    end
end

function process_save(obj::ZulipRequest, db, opts)
    @info "save"
    hashtags = String[]
    length(obj.data) < 6 && return "Not enough arguments in `save` command"
    s = obj.data[6:end]

    while true
        m = match(r"^\s*(#[^\s]+)\s+(.*)"s, s)
        isnothing(m) && break
        push!(hashtags, m[1])
        s = m[2]
    end

    hashtags = join(hashtags, " ")
    token = codeid()
    try
        store_snippet(db, obj.message.sender_id, token, s, hashtags)
        return "Snippet codeid: `$token`"
    catch err
        @error "DB Error" exception=(err, catch_backtrace())
        return "Server error, please try one more time. Later. Sorry."
    end
end

function process_list(obj::ZulipRequest, db, opts)
    @info "list"
    snippets = load_snippets(db, obj.message.sender_id)
    snippets = map(x -> (; snippet = x.snippet, code = x.code, hashtags = x.tags, tags = split(x.tags, " ")), snippets)
    if length(obj.data) >= 6
        hashtags = String[]
        s = obj.data[6:end]

        while true
            m = match(r"^\s*(#[^\s]+)\s*(.*)"s, s)
            isnothing(m) && break
            push!(hashtags, m[1])
            s = m[2]
        end
        snippets = filter(x -> !isempty(intersect(hashtags, x.tags)), snippets)
    end

    isempty(snippets) && return "No snippets found"

    io = IOBuffer()
    for snippet in snippets
        println(io, "**codeid**: ", snippet.code, ", **tags**: ", snippet.hashtags)
        println(io, snippet.snippet, "\n")
    end
    res = strip(String(take!(io)))
    return res
end

function process_help(obj::ZulipRequest, db, opts)
    @info "help"
    return """
Currently following commands are supported

1. `show <codeid>` - show snippet with the required `codeid`
2. `list <#hashtag1> <#hashtag2>` - shows all snippets of the author with corresponding hashtags. If no hashtags is provided returns all snippets.
3. `save <#hashtag1> <#hashtag2> <snippet>` - saves snippet in database with provided hashtags. In response server returns code_id, that can be used for snippet search.
4. `help` - this message
    """
end

########################################
# Server
########################################
function run(db, opts = OPTS[])
    HTTP.serve(opts.host, opts.port) do http
        obj = JSON3.read(HTTP.payload(http), ZulipRequest)
        resp = process(obj, db, opts)

        return HTTP.Response(resp)
    end
end

end # module

max_stars_repo_name: Arkoniak/ZulipSnippetBot

max_issues_repo_name: Arkoniak/ZulipSnippetBot

max_forks_repo_name: Arkoniak/ZulipSnippetBot


text:
using Pkg

Pkg.develop(PackageSpec(path=pwd()))
Pkg.instantiate()

using Documenter, BioAlignments

makedocs(
    format = Documenter.HTML(
        edit_link = :commit
    ),
    modules = [BioAlignments],
    sitename = "BioAlignments.jl",
    pages = [
        "Home" => "index.md",
        "Alignment representation" => "alignments.md",
        "Pairwise alignment" => "pairalign.md",
        "API Reference" => "references.md"
    ],
    authors = replace(join(Pkg.TOML.parsefile("Project.toml")["authors"], ", "), r" <.*?>" => "" ) * ", The BioJulia Organisation, and other contributors."
)

deploydocs(
    repo = "github.com/BioJulia/BioAlignments.jl.git",
    devbranch = "develop",
    push_preview = true
)


text:
  # US Equities from A to E (so far)
  const AAPL = readtimearray(Pkg.dir("MarketData/data/AAPL.csv"), meta = "AAPL")
  const BA   = readtimearray(Pkg.dir("MarketData/data/BA.csv"),   meta = "BA")
  const CAT  = readtimearray(Pkg.dir("MarketData/data/CAT.csv"),  meta = "CAT")
  const DELL = readtimearray(Pkg.dir("MarketData/data/DELL.csv"), meta = "DELL")
  const EBAY = readtimearray(Pkg.dir("MarketData/data/EBAY.csv"), meta = "EBAY")
  const F    = readtimearray(Pkg.dir("MarketData/data/F.csv"), meta = "F")
  const GE   = readtimearray(Pkg.dir("MarketData/data/GE.csv"), meta = "GE")

  # smaller datasets used for testing time-related packages
  const cl        = AAPL["Close"][Date(2000,1,1):Date(2001,12,31)]
  const op        = AAPL["Open"][Date(2000,1,1):Date(2001,12,31)]
  const ohlc      = AAPL["Open", "High", "Low", "Close"][Date(2000,1,1):Date(2001,12,31)]
  const ohlcv     = AAPL["Open", "High", "Low", "Close", "Volume"][Date(2000,1,1):Date(2001,12,31)]
  const datetime1 = readtimearray(Pkg.dir("MarketData/data/datetime1.csv"))
  const datetime2 = readtimearray(Pkg.dir("MarketData/data/datetime2.csv"), format="yyyy-mm-dd HH:MM:SS")
  const mdata     = TimeArray(cl.timestamp, cl.values, cl.colnames, "Apple") # data with meta field occupied


text:
module TestWrapper

using Test
using UrlDownload
using DataFrames
using CSV

wrapper(x; kw...) = DataFrame(CSV.File(IOBuffer(x); kw...))

@testset "wrappers" begin
    df = DataFrame(x = [1, 3], y = [2, 4])

    url = "https://raw.githubusercontent.com/Arkoniak/UrlDownload.jl/master/data/ext.csv"
    res = urldownload(url, parser = x -> DataFrame(CSV.File(IOBuffer(x))))
    @test res == df

    url = "https://raw.githubusercontent.com/Arkoniak/UrlDownload.jl/master/data/semicolon.csv"
    res = urldownload(url, parser = wrapper, delim = ';')
    @test res == df
end

end # module