text:
module ANSIColoredPrinters

import Base: ==, show, showable

export PlainTextPrinter, HTMLPrinter

abstract type AbstractPrinter end

"""
    StackModelPrinter

An abstract printer type for stack-based or tree model formats.
"""
abstract type StackModelPrinter <: AbstractPrinter end

"""
    FlatModelPrinter

An abstract printer type for non-stack-based formats.
"""
abstract type FlatModelPrinter <: AbstractPrinter end

struct SGRColor
    class::String
    hex::String
    SGRColor(class::AbstractString="", hex::AbstractString="") = new(class, hex)
end

mutable struct SGRContext
    fg::SGRColor
    bg::SGRColor
    flags::BitVector
    SGRContext() = new(SGRColor(), SGRColor(), falses(128))
end

include("colors.jl")
include("plain.jl")
include("html.jl")

==(a::SGRContext, b::SGRContext) = a.fg == b.fg && a.bg == b.bg && a.flags == b.flags

isnormal(ctx::SGRContext) = isnormal(ctx.fg) && isnormal(ctx.bg) && !any(ctx.flags)

function reset_color(ctx::SGRContext)
    ctx.fg = SGRColor()
    ctx.bg = SGRColor()
end

function reset(ctx::SGRContext)
    reset_color(ctx)
    ctx.flags .= false
end

function reset(printer::AbstractPrinter)
    seekstart(printer.buf)
    if printer isa StackModelPrinter
        while !isempty(printer.stack)
            pop!(printer.stack)
        end
    end
    reset(printer.ctx)
    reset(printer.prevctx)
end

function copy!(dest::SGRContext, src::SGRContext)
    dest.fg = src.fg
    dest.bg = src.bg
    dest.flags .= src.flags
end

escape_char(printer::AbstractPrinter, c::Char) = nothing

function show_body(io::IO, printer::AbstractPrinter)
    reset(printer)
    buf = printer.buf
    ctx_changed = false
    while !eof(buf)
        c = read(buf, Char)
        if c !== '\e'
            if ctx_changed
                apply_changes(io, printer)
                copy!(printer.prevctx, printer.ctx)
                ctx_changed = false
            end
            ec = escape_char(printer, c)
            write(io, ec === nothing ? c : ec)
            continue
        end
        ansiesc = IOBuffer()
        c = read(buf, Char)
        c === '[' || continue
        while !eof(buf)
            c = read(buf, Char)
            if '0' <= c <= '9' || c === ';' # strip spaces
                write(ansiesc, c)
            elseif c >= Char(0x40)
                break
            end
        end
        astr = String(take!(ansiesc))
        if c === 'm'
            while true
                astr = parse_sgrcodes(printer.ctx, astr)
                isempty(astr) && break
            end
        end
        ctx_changed = printer.prevctx != printer.ctx
    end

    finalize(io, printer)
end

function parse_sgrcodes(ctx::SGRContext, astr::AbstractString)
    if (m = match(r"^0?(?:;|$)", astr)) !== nothing
        reset(ctx)
    elseif (m = match(r"^22;?", astr)) !== nothing
        reset_color(ctx)
        ctx.flags[1:2] .= false
    elseif (m = match(r"^2([3-57-9]);?", astr)) !== nothing
        di = parse(Int, m.captures[1])
        ctx.flags[di] = false
        ctx.flags[di + (di === 5)] = false
    elseif (m = match(r"^39;?", astr)) !== nothing
        ctx.fg = SGRColor()
    elseif (m = match(r"^49;?", astr)) !== nothing
        ctx.bg = SGRColor()
    elseif (m = match(r"^([349][0-7]|10[0-7]);?", astr)) !== nothing
        set_16colors!(ctx, m.captures[1])
    elseif (m = match(r"^([345]8);5;(\d{0,3});?", astr)) !== nothing
        d, col = m.captures
        d != "58" && set_256colors!(ctx, d, col) # code 58 is not yet supported
    elseif (m = match(r"^([345]8);2;(\d{0,3});(\d{0,3});(\d{0,3});?", astr)) !== nothing
        d, rs, gs, bs = m.captures
        d != "58" && set_24bitcolors!(ctx, d, rs, gs, bs) # code 58 is not yet supported
    elseif (m = match(r"^(\d);?", astr)) !== nothing
        di = parse(Int, m.captures[1])
        if di === 1 || di === 2
            ctx.flags[1:2] .= (di === 1, di === 2)
        elseif di === 5 || di === 6
            ctx.flags[5:6] .= (di === 5, di === 6)
        else
            ctx.flags[di] = true
        end
    elseif (m = match(r"^(\d+);?", astr)) !== nothing
        # unsupported
    else # unknown
        return ""
    end
    return astr[m.offset + lastindex(m.match):end]
end

function apply_changes(io::IO, printer::StackModelPrinter)
    stack = printer.stack
    ctx = printer.ctx
    prevctx = printer.prevctx
    invert = prevctx.flags[7] != ctx.flags[7]

    marks = zeros(Bool, length(stack))
    nstack = String[]

    for di = 1:9
        if prevctx.flags[di] != ctx.flags[di]
            class = string(di)
            marks .|= map(c -> c == class, stack)
            ctx.flags[di] && push!(nstack, class)
        end
    end
    if prevctx.fg != ctx.fg || invert
        marks .|= map(c -> occursin(r"^(?:3[0-7]|9[0-7]|38_[25])$", c), stack)
        isnormal(ctx.fg) || push!(nstack, ctx.fg.class)
    end
    if prevctx.bg != ctx.bg || invert
        marks .|= map(c -> occursin(r"^(?:4[0-7]|10[0-7]|48_[25])$", c), stack)
        isnormal(ctx.bg) || push!(nstack, ctx.bg.class)
    end
    poplevel = findfirst(marks)
    if poplevel !== nothing
        while length(stack) >= poplevel
            end_current_state(io, printer)
            class = pop!(stack)
            pop!(marks) || push!(nstack, class)
        end
    end
    while !isempty(nstack)
        class = pop!(nstack)
        push!(stack, class)
        start_new_state(io, printer)
    end
end

function apply_changes(io::IO, printer::FlatModelPrinter)
    ansicodes = Int[]

    prevctx = printer.prevctx
    ctx = printer.ctx
    prevflags = prevctx.flags
    flags = ctx.flags

    prevctx == ctx && return

    if isnormal(ctx)
        change_state(io, printer, ansicodes)
        return
    end
    if prevflags[1] & !flags[1] || prevflags[2] & !flags[2]
        if !flags[1] && !flags[2]
            push!(ansicodes, 22)
            parse_sgrcodes(prevctx, "22")
        end
    end
    if prevflags[5] & !flags[5] || prevflags[6] & !flags[6]
        if !flags[5] && !flags[6]
            push!(ansicodes, 25)
            parse_sgrcodes(prevctx, "25")
        end
    end
    prevctx.fg != ctx.fg && append!(ansicodes, isnormal(ctx.fg) ? (39,) : codes(ctx.fg))
    prevctx.bg != ctx.bg && append!(ansicodes, isnormal(ctx.bg) ? (49,) : codes(ctx.bg))
    for i in 1:length(flags)
        prevflags[i] === flags[i] && continue
        if 1 <= i <= 2 || 5 <= i <= 6
            flags[i] && push!(ansicodes, i)
        elseif i <= 9
            push!(ansicodes, flags[i] ? i : i + 20)
        end
    end

    isempty(ansicodes) || change_state(io, printer, ansicodes)
end

function finalize(io::IO, printer::StackModelPrinter)
    while !isempty(printer.stack) # force closing
        end_current_state(io, printer)
        pop!(printer.stack)
    end
end

function finalize(io::IO, printer::FlatModelPrinter)
    reset(printer.ctx)
    printer.prevctx != printer.ctx && apply_changes(io, printer)
end

function Base.show(io::IO, ::MIME"text/plain", printer::AbstractPrinter)
    reset(printer)
    if get(io, :color, false)::Bool
        write(io, printer.buf)
    end
end

end # module


text:
using GeometryBasics
using AbstractPlotting: Axis

@cell "Axis theming" begin
    scene = Scene()
    points = decompose(Point2f0, Circle(Point2f0(10), 10f0), 9)
    lines!(scene, points, linewidth=8, color=:black)
    axis = scene[Axis] # get axis
    axis[:frame][:linewidth] = 5
    axis[:grid][:linewidth] = (1, 5)
    axis[:grid][:linecolor] = ((:red, 0.3), (:blue, 0.5))
    axis[:names][:axisnames] = ("x", "y   ")
    axis[:ticks][:title_gap] = 1
    axis[:names][:rotation] = (0.0, -3 / 8 * pi)
    axis[:names][:textcolor] = ((:red, 1.0), (:blue, 1.0))
    axis[:ticks][:font] = ("Dejavu Sans", "Helvetica")
    axis[:ticks][:rotation] = (0.0, -pi / 2)
    axis[:ticks][:textsize] = (3, 7)
    axis[:ticks][:gap] = 5
    scene
end

@cell "Legend" begin
    scene = Scene(resolution=(500, 500))
    x = map([:dot, :dash, :dashdot], [2, 3, 4]) do ls, lw
        linesegments!(
            range(1, stop=5, length=100), RNG.rand(100), RNG.rand(100),
            linestyle=ls, linewidth=lw,
            color=RNG.rand(RGBAf0)
        )[end]
    end
    x = [x..., scatter!(range(1, stop=5, length=100), RNG.rand(100), RNG.rand(100))[end]]
    center!(scene)
    ls = AbstractPlotting.legend(x, ["attribute $i" for i in 1:4], camera=campixel!, raw=true)
    l = ls[end]
    l[:strokecolor] = RGBAf0(0.8, 0.8, 0.8)
    l[:gap] = 15
    l[:textsize] = 12
    l[:linepattern] = Point2f0[(0, -0.2), (0.5, 0.2), (0.5, 0.2), (1.0, -0.2)]
    l[:scatterpattern] = decompose(Point2f0, Circle(Point2f0(0.5, 0), 0.3f0), 9)
    l[:markersize] = 2f0
    vbox(scene, ls)
end

ext: jl


text:
function get_results(data_dir, results_dir, filename; Ncourse=21,
                                                      threshold_level=-0.05,
                                                      verbose=true)

    location = data_dir * "/" * filename
    file = jldopen(location)

    t_vec, r_vec, y_vec, z_vec, e_vec = get_data(file,
                                                 Ncourse,
                                                 threshold_level,
                                                 verbose)

    save_results(results_dir, filename, t_vec, r_vec, y_vec, z_vec, e_vec)
    return [t_vec r_vec y_vec z_vec e_vec]
end
function get_info(file)
    Ly = file["grid/Ly"]
    Lz = file["grid/Lz"]
    Ny = file["grid/Ny"]
    Nz = file["grid/Nz"]
    return Ly, Lz, Ny, Nz
end
function get_data(file, Ncourse, threshold_level, verbose)

    iterations = parse.(Int, keys(file["timeseries/t"]))

    Nt = length(iterations)
    t_vec = zeros(Nt)
    r_vec = zeros(Nt)
    y_vec = zeros(Nt)
    z_vec = zeros(Nt)
    ∂z∂t_vec = zeros(Nt-1)
    e_vec = zeros(Nt)

    Ly, Lz, Ny, Nz = get_info(file)
    ynodes = range(-Ly/2,Ly/2,length=Ny)
    znodes = range(-Lz,0,length=Nz)
    guess_cache = circ_guess.(ynodes, znodes',0,0,0)
    gridCoarse = ParameterGrid2D(Ly,Lz,Ncourse)

    threshold(x) = x < threshold_level

    for (i, iter) in enumerate(iterations)
        # retrieve solution from JLD2 file
        if verbose println("Step: ", i, "/", Nt) end
        b_bitmap = threshold.(file["timeseries/b/$iter"][1, :, :])
        res = coarse_fine_search(gridCoarse, b_bitmap, guess_cache, ynodes, znodes)

        r_vec[i] = res[1][1]
        y_vec[i] = res[1][2]
        z_vec[i] = res[1][3]
        e_vec[i] = res[2]

        t = file["timeseries/t/$iter"]
        t_vec[i] = t
    end
    return t_vec, r_vec, y_vec, z_vec, e_vec
end
function save_results(results_dir, filename, t_vec, r_vec, y_vec, z_vec, e_vec)
    txt_filename = filename[1:end-4] * "txt"
    location = results_dir * "/" * txt_filename
    open(location, "w") do io
        writedlm(io, [t_vec r_vec y_vec z_vec e_vec])
    end
end


text:
struct BaroneAdesiWhaleyApproximationEngine{B <: AbstractBlackScholesProcess} <: PricingEngine
  process::B
end

function get_critical_price_params(::Call, payoff::StrikedTypePayoff, n::Float64, m::Float64,
                                  variance::Float64, bT::Float64)
  qu = (-(n - 1.0) + sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * m)) / 2.0
  Su = payoff.strike / (1.0 - 1.0 / qu)
  h = -(bT + 2.0 * sqrt(variance)) * payoff.strike / (Su - payoff.strike)
  Si = payoff.strike + (Su - payoff.strike) * (-expm1(h))

  return qu, Su, h, Si
end

function get_critical_price_params(::Put, payoff::StrikedTypePayoff, n::Float64, m::Float64,
                                  variance::Float64, bT::Float64)
  qu = (-(n - 1.0) - sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * m)) / 2.0
  Su = payoff.strike / (1.0 - 1.0 / qu)
  h = (bT - 2.0 * sqrt(variance)) * payoff.strike / (payoff.strike - Su)
  Si = Su + (payoff.strike - Su) * exp(h)

  return qu, Su, h, Si
end

function evolve_Si(::Call, payoff::StrikedTypePayoff, Si::Float64, n::Float64, K::Float64, dividendDiscount::Float64,
                  riskFreeDiscount::Float64, tolerance::Float64, variance::Float64, temp::Float64, d1::Float64)
  w = Normal()
  Q = (-(n - 1.0) + sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * K)) / 2.0
  LHS = Si - payoff.strike
  RHS = temp + (1.0 - dividendDiscount * cdf(w, d1)) * Si / Q
  bi = dividendDiscount * cdf(w, d1) * (1.0 - 1.0 / Q) + (1.0 - dividendDiscount * distribution_derivative(w, d1) /
        sqrt(variance)) / Q
  while abs(LHS - RHS) / payoff.strike > tolerance
    Si = (payoff.strike + RHS - bi * Si) / (1.0 - bi)
    forwardSi = Si * dividendDiscount / riskFreeDiscount
    d1 = (log(forwardSi / payoff.strike) + 0.5 * variance) / sqrt(variance)
    LHS = Si - payoff.strike
    temp2 = black_formula(payoff.optionType, payoff.strike, forwardSi, sqrt(variance)) * riskFreeDiscount
    RHS = temp2 + (1.0 - dividendDiscount * cdf(w, d1)) * Si / Q
    bi = dividendDiscount * cdf(w, d1) * (1.0 - 1.0 / Q) + (1.0 - dividendDiscount * distribution_derivative(w, d1) /
          sqrt(variance)) / Q
  end

  return Si
end

function evolve_Si(::Put, payoff::StrikedTypePayoff, Si::Float64, n::Float64, K::Float64, dividendDiscount::Float64,
                  riskFreeDiscount::Float64, tolerance::Float64, variance::Float64, temp::Float64, d1::Float64)
  w = Normal()
  Q = (-(n - 1.0) - sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * K)) / 2.0
  LHS = payoff.strike - Si
  RHS = temp - (1.0 - dividendDiscount * cdf(w, -d1)) * Si / Q
  bi = -dividendDiscount * cdf(w, -d1) * (1.0 - 1.0 / Q) - (1.0 + dividendDiscount * distribution_derivative(w, -d1) /
        sqrt(variance)) / Q
  while abs(LHS - RHS) / payoff.strike > tolerance
    Si = (payoff.strike - RHS + bi * Si) / (1.0 + bi)
    forwardSi = Si * dividendDiscount / riskFreeDiscount
    d1 = (log(forwardSi / payoff.strike) + 0.5 * variance) / sqrt(variance)
    LHS = payoff.strike - Si
    temp2 = black_formula(payoff.optionType, payoff.strike, forwardSi, sqrt(variance)) * riskFreeDiscount
    RHS = temp2 - (1.0 - dividendDiscount * cdf(w, -d1)) * Si / Q
    bi = -dividendDiscount * cdf(w, -d1) * (1.0 - 1.0 / Q) - (1.0 + dividendDiscount * distribution_derivative(w, -d1) /
          sqrt(variance)) / Q
  end

  return Si
end

function critical_price(payoff::StrikedTypePayoff, riskFreeDiscount::Float64, dividendDiscount::Float64,
                        variance::Float64, tolerance::Float64)
  # calculation of seed values, Si
  bT = log(dividendDiscount / riskFreeDiscount)
  n = 2.0 * bT / variance
  m = -2.0 * log(riskFreeDiscount) / variance

  qu, Su, h, Si = get_critical_price_params(payoff.optionType, payoff, n, m, variance, bT)

  # Newton - Raphson algo for finding critical price Si
  forwardSi = Si * dividendDiscount / riskFreeDiscount
  d1 = (log(forwardSi / payoff.strike) + 0.5 * variance) / sqrt(variance)
  K = ~is_close(riskFreeDiscount, 1.0, 1000) ? -2.0 * log(riskFreeDiscount) / (variance * (1.0 - riskFreeDiscount)) :
                                              2.0 / variance

  temp = black_formula(payoff.optionType, payoff.strike, forwardSi, sqrt(variance)) * riskFreeDiscount

  Si = evolve_Si(payoff.optionType, payoff, Si, n, K, dividendDiscount, riskFreeDiscount, tolerance,
                variance, temp, d1)

  return Si
end

function early_exericse_calc(::Call, payoff::StrikedTypePayoff, n::Float64, K::Float64, Sk::Float64,
                            dividendDiscount::Float64, d1::Float64, spot::Float64, black_val::Float64)
  w = Normal()
  Q = (-(n - 1.0) + sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * K)) / 2.0
  a = (Sk / Q) * (1.0 - dividendDiscount * cdf(w, d1))
  if spot < Sk
    return black_val + a * ^((spot / Sk), Q)
  else
    return spot - payoff.strike
  end
end

function early_exericse_calc(::Put, payoff::StrikedTypePayoff, n::Float64, K::Float64, Sk::Float64,
                            dividendDiscount::Float64, d1::Float64, spot::Float64, black_val::Float64)
  w = Normal()
  Q = (-(n - 1.0) - sqrt(((n - 1.0) * (n - 1.0)) + 4.0 * K)) / 2.0
  a = -(Sk / Q) * (1.0 - dividendDiscount * cdf(w, -d1))
  if spot > Sk
    return black_val + a * ^((spot / Sk), Q)
  else
    return payoff.strike - spot
  end
end

function _calculate!(pe::BaroneAdesiWhaleyApproximationEngine, opt::AmericanOption)
  # stuff
  ex = opt.exercise
  payoff = opt.payoff

  variance = black_variance(pe.process.blackVolatility, ex.dates[end], payoff.strike)

  dividendDiscount = discount(pe.process.dividendYield, ex.dates[end])
  riskFreeDiscount = discount(pe.process.riskFreeRate, ex.dates[end])

  spot = state_variable(pe.process).value
  spot > 0.0 || error("negative or null underlying given")
  forwardPrice = spot *  dividendDiscount / riskFreeDiscount

  black = BlackCalculator(payoff, forwardPrice, sqrt(variance), riskFreeDiscount)

  if dividendDiscount > 1.0 && isa(payoff.optionType, Call)
    # early exercise never optimal
    opt.results.value = value(black)
    opt.results.delta = delta(black, spot)
    opt.results.deltaForward = delta_forward(black)
    opt.results.elasticity = elasticity(black, spot)
    opt.results.gamma = gamma(black, spot)

    rfdc = pe.process.riskFreeRate.dc
    divdc = pe.process.dividendYield.dc
    voldc = pe.process.blackVolatility.dc

    t = year_fraction(rfdc, reference_date(pe.process.riskFreeRate), opt.exercise.dates[end])
    opt.results.rho = rho(black, t)

    t = year_fraction(divdc, reference_date(pe.process.dividendYield), opt.exercise.dates[end])
    opt.results.dividendRho = dividend_rho(black, t)

    t = year_fraction(voldc, reference_date(pe.process.blackVolatility), opt.exercise.dates[end])
    opt.results.vega = vega(black, t)

    opt.results.theta = theta(black, spot, t)
    opt.results.thetaPerDay = theta_per_day(black, spot, t)

    opt.results.strikeSensitivity = strike_sensitivity(black)
    opt.results.itmCashProbability = itm_cash_probability(black)
  else
    # early exercise can be optimal
    tolerance = 1e-6
    Sk = critical_price(payoff, riskFreeDiscount, dividendDiscount, variance, tolerance)
    forwardSk = Sk * dividendDiscount / riskFreeDiscount
    d1 = (log(forwardSk / payoff.strike) + 0.5 * variance) / sqrt(variance)
    n = 2.0 * log(dividendDiscount / riskFreeDiscount)/variance
    K = ~is_close(riskFreeDiscount, 1.0, 1000) ? -2.0 * log(riskFreeDiscount) / (variance * (1.0 - riskFreeDiscount)) :
                                                2.0 / variance
    opt.results.value = early_exericse_calc(payoff.optionType, payoff, n, K, Sk, dividendDiscount, d1,
                          spot, value(black))
  end
end


text:
struct Var
    comp::Symbol
    var::Symbol
end

struct Component
    name::Symbol
    node::EzXML.Node
    deps::Set{Symbol}
end

nameof(comp::Component) = comp.name

struct Connection
    comp_1::Symbol
    comp_2::Symbol
    vars::Array{Pair{Var}}
end

components(conn::Connection) = (conn.comp_1, conn.comp_2)
variables(conn::Connection) = conn.vars

struct Document
    path::AbstractString
    xmls::Array{EzXML.Document}
    comps::Array{Component}
    conns::Array{Connection}
end

components(doc::Document) = doc.comps
connections(doc::Document) = doc.conns

struct CellModel
    doc::Document
    sys::ODESystem
end


text:
module TestXGBoost

using MLJ
using Test
using Pkg
Pkg.clone("https://github.com/dmlc/XGBoost.jl")
import XGBoost

train_X, train_Y = readlibsvm("../data/agaricus.txt.train", (6513, 126))
test_X, test_Y = readlibsvm("../data/agaricus.txt.test", (1611, 126))

dtrain = XGBoost.DMatrix("../data/agaricus.txt.train")
dtest = XGBoost.DMatrix("../data/agaricus.txt.test")

#import XGBoost_
bareboost = XGBoostRegressor(num_round=6)
bfit,= MLJ.fit(bareboost,1,train_X,train_Y)

num_round=6
bst = XGBoost.xgboost(train_X, num_round, label = train_Y)
bst_pred = XGBoost.predict(bst,test_X)
bst_DMatrix, = MLJ.fit(bareboost,1,dtrain)

bpredict_fulltree = MLJ.predict(bareboost,bfit,test_X)
bpredict_onetree = MLJ.predict(bareboost,bfit,test_X,ntree_limit=1)
bpredict_DMatrix = MLJ.predict(bareboost,bst_DMatrix,dtest)

@test  bpredict_fulltree !=bpredict_onetree
@test  bpredict_fulltree ≈ bpredict_DMatrix atol=0.000001
@test_logs (:warn,"updater has been changed to shotgun, the default option for booster=\"gblinear\"") XGBoostRegressor(num_round=1,booster="gblinear",updater="grow_colmaker")
@test_logs (:warn,"\n num_class has been changed to 2") XGBoostClassifier(eval_metric="mlogloss",objective="multi:softprob")
@test bst_pred ≈ bpredict_fulltree atol=0.000001
end
true


text:
# Exercise

#TODO: Write a simulation with 500,1000,1500,2000 agents
#      that where in the beginning every agent has value 0
#      except agent[1] has value 1
#      in every simulation step, every agent gets the
#      mean value of its neighbors
#      agents[2] = (agents[1] + agents[3]) /2
#      important to note is that agent[1] has only one neigbors
#      agents[agent_count] also always stays 0
#       mean(agents)
#      steps 5:10 steps

using Statistics
using Distributed
 function simulate(agent_count :: Integer, step_count :: Integer)
    agents = zeros(agent_count)
    @debug "We are currently simulation $agent_count agents."
    agents[1] = 1
    for step in 1:step_count, current_agent in 2:(agent_count-1)
        agents[current_agent] = (agents[current_agent-1] + agents[current_agent+1]) /2
    end
    mean(agents)
end

println("-------")

using Distributed

@time Threads.@threads for agent_count in 500:500:2000
    for steps in [10000,50000,100000]
        result = simulate(agent_count, steps)
        @info "The result for $agent_count agents and $steps steps is $result it ran on $(Threads.threadid())."
    end
end


text:
export Quaternion, rotationmatrix_normalized, qrotation_x, qrotation_y, qrotation_z

import Base: *, ≈

@doc raw"""
A structure representing a quaternion.

Stripeline uses quaternions to encode rotations when generating pointings. The
fields of `Quaternion` are:

- `s`
- `v1`
- `v2`
- `v3`

It is possible to multiply quaternions using the `*` operator. Constructors for
rotation quaternions are provided by `qrotation_x`, `qrotation_y`, and
`qrotation_z`.

`Quaternion` implements the `≈` binary relationship, in order to ease the
implementation of tests.
"""
struct Quaternion
    s::Float64
    v1::Float64
    v2::Float64
    v3::Float64
end

(*)(q::Quaternion, w::Quaternion) = Quaternion(q.s * w.s - q.v1 * w.v1 - q.v2 * w.v2 - q.v3 * w.v3,
                                               q.s * w.v1 + q.v1 * w.s + q.v2 * w.v3 - q.v3 * w.v2,
                                               q.s * w.v2 - q.v1 * w.v3 + q.v2 * w.s + q.v3 * w.v1,
                                               q.s * w.v3 + q.v1 * w.v2 - q.v2 * w.v1 + q.v3 * w.s)

(≈)(q::Quaternion, w::Quaternion) = (q.s ≈ w.s) && (q.v1 ≈ w.v1) && (q.v2 ≈ w.v2) && (q.v3 ≈ w.v3)

@doc raw"""

    rotationmatrix_normalized(q::Quaternion)

Return a 3×3 matrix representing the rotation encoded by quaternion `q`.
The function assumes that the quaternion is normalized. This is the
case if `q` is a composition of rotations built using the functions
`qrotation_x`, `qrotation_y`, and `qrotation_z`.
"""
function rotationmatrix_normalized(q::Quaternion)
    sx, sy, sz = 2q.s * q.v1, 2q.s * q.v2, 2q.s * q.v3
    xx, xy, xz = 2q.v1^2, 2q.v1 * q.v2, 2q.v1 * q.v3
    yy, yz, zz = 2q.v2^2, 2q.v2 * q.v3, 2q.v3^2
    Float64[(1 - (yy + zz))  (xy - sz)        (xz + sy);
            (xy + sz)        (1 - (xx + zz))  (yz - sx);
            (xz - sy)        (yz + sx)        (1 - (xx + yy))]
end

qrotation_x(theta) = Quaternion(cos(theta / 2), sin(theta / 2), 0.0, 0.0)
qrotation_y(theta) = Quaternion(cos(theta / 2), 0.0, sin(theta / 2), 0.0)
qrotation_z(theta) = Quaternion(cos(theta / 2), 0.0, 0.0, sin(theta / 2))

"""
    qrotation_x(theta)
    qrotation_y(theta)
    qrotation_z(theta)

Return a `Quaternion` object representing a rotation around the ``e_x``,
``e_y``, or ``e_z`` axis by an angle `theta` (in radians). The quaternions
returned by these functions are already normalized and can be used with the
function `rotationmatrix_normalized`.
"""
qrotation_x, qrotation_y, qrotation_z


text:
#=
Created on Friday 15 November 2019
Last update: -

@author: Michiel Stock
michielfmstock@gmail.com

Simulation for 1D CA.
=#

abstract type CARule end

struct DetCARule <: CARule
    rulenr::Int8
    rulestr::BitArray
end

function CARule(rulenr::Int)
    @assert 0 ≤ rulenr < 2^8
    rulenr = Int8(rulenr)
    bstr = bitstring(rulenr)
    rulestr = reverse([c=='1' for c in bstr])
    return DetCARule(rulenr, rulestr)
end

nextstate(x::Bool, y::Bool, z::Bool, rule::CARule) = rule.rulestr[4x+2y+1z+1]

function update!(x, xnew, rule::DetCARule)
    xnew[1] = nextstate(x[end], x[1], x[2], rule)
    @simd for i in 2:length(x)-1
        @inbounds xnew[i] = nextstate(x[i-1], x[i], x[i+1], rule)
    end
    xnew[1] = nextstate(x[end-1], x[end], x[1], rule)
    return xnew
end

function simulate(x₀, rule::CARule; nsteps::Integer=100)
    n = length(x₀)
    X = Array{Bool,2}(undef, nsteps+1, n)
    X[1,:] .= x₀
    x = copy(x₀)
    xnew = similar(x₀)
    for t in 1:nsteps
        update!(x, xnew, rule)
        X[t+1,:] = xnew
        x = xnew
    end
    return X
end

using Plots

x₀ = zeros(Bool, 20001)

x₀[10001] = true

#x₀ = rand(Bool, 2001)

rule = CARule(110)

X = simulate(x₀, rule, nsteps=1000)
heatmap(1X)


text:
struct TimelineColumn
    name::Symbol
    shifts::Int # negative for leads, positive for lags, 0 for normal
    TimelineColumn(n::Symbol) = new(n, 0)
    TimelineColumn(n::Symbol, s::Int) = new(n, s)
end

struct DictIndex
    cols::Vector{TimelineColumn}
    lookup::Dict{Int, TimelineColumn}# reverse of other itmes since data is just stored in vectors
end

DictIndex(cols::Vector{Symbol}) = DictIndex(cols, zeros(Int, length(cols)))

TimelineColumn(x::TimelineColumn) = x
TimelineColumn(x, s=0) = TimelineColumn(Symbol(x), Int(s))

function DictIndex(cols::Vector{Symbol}, shifts::Vector{Int})
    DictIndex(TimelineColumn.(cols, shifts))
end

function DictIndex(cols::Vector{TimelineColumn})
    DictIndex(
        cols,
        Dict(1:length(cols) .=> cols)
    )
end

function Base.getindex(x::DictIndex, i::Int)
    x.lookup[i]
end

function Base.push!(x::DictIndex, n::Symbol)
    n = TimelineColumn(n)
    push!(x.cols, n)
    x.lookup[length(x.cols)] = n
end

Base.length(x::DictIndex) = length(x.cols)
Base.names(x::DictIndex) = x.cols

Base.Symbol(x::TimelineColumn) = x.name
function Base.String(x::TimelineColumn)
    if x.shifts == 0
        x.name |> string
    elseif x.shifts < 0
        v = x.shifts < -1 ? ", $(-1 * x.shifts)" : ""
        "lead($(x.name)$v)"
    else
        v = x.shifts > 1 ? ", $(x.shifts)" : ""
        "lag($(x.name)$v)"
    end
end

function Base.string(x::TimelineColumn)
    String(x)
end

Base.isequal(x::TimelineColumn, y::TimelineColumn) = x.name == y.name && x.shifts == y.shifts
Base.isequal(x::DictIndex, y::DictIndex) = all(isequal.(names(x), names(y)))

Base.:(==)(x::TimelineColumn, y::TimelineColumn) = isequal(x, y)
Base.:(==)(x::DictIndex, y::DictIndex) = isequal(x, y)

Base.isless(x::TimelineColumn, y::TimelineColumn) = isless(x.name, y.name) && isless(x.shifts, y.shifts)

Base.show(io::IO, x::DictIndex) = show(io, names(x))

Base.show(io::IO, x::TimelineColumn) = print(io, String(x))

ShiftedArrays.lead(x::Symbol, n::Int=1) = TimelineColumn(x, -n)
ShiftedArrays.lag(x::Symbol, n::Int=1) = TimelineColumn(x, n)

function Base.iterate(x::DictIndex, i=1)
    if i > length(x)
        nothing
    else
        (names(x)[i], i+1)
    end
end

Base.convert(::Type{Symbol}, x::TimelineColumn) = Symbol(String(x))
shift_count(x::TimelineColumn) = x.shifts


text:
# ------------------------------------------------------------------
# Licensed under the ISC License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    Curve(x, y, z, ...)

A curve along parametric coordinates `x`, `y`, `z`, ...

    Curve(coords)

Alternatively, construct curve from `coords` matrix. The number of rows
of the matrix is the dimensionality of the curve whereas the number
of columns is the number of points.

## Examples

A 3D borehole or directional well can be represented with a curve
where `x`, `y`, and `z` are the parametric coordinates `(x(t), y(t), z(t))`
with parameter `t ∈ R`:

```julia
julia> Curve(x, y, z)
```
"""
struct Curve{T,N} <: AbstractDomain{T,N}
  coords::Matrix{T}
end

function Curve(coordarrays::Vararg{<:AbstractVector{T},N}) where {N,T}
  npts = length.(coordarrays)
  @assert length(unique(npts)) == 1 "coordinates arrays must have the same dimensions"

  coords = Matrix{T}(undef, N, npts[1])
  for (i, array) in enumerate(coordarrays)
    coords[i,:] = array
  end

  Curve{T,N}(coords)
end

Curve(coords::AbstractMatrix{T}) where {T} = Curve{T,size(coords,1)}(coords)

npoints(curve::Curve) = size(curve.coords, 2)

function coordinates!(buff::AbstractVector{T}, curve::Curve{T,N},
                      location::Int) where {N,T}
  for i in 1:N
    @inbounds buff[i] = curve.coords[i,location]
  end
end

georef(data::OrderedDict{Symbol,<:AbstractArray}, curve::Curve{T,N}) where {N,T} =
  CurveData{T,N}(data, curve)

# ------------
# IO methods
# ------------
function Base.show(io::IO, curve::Curve{T,N}) where {N,T}
  npts = size(curve.coords, 2)
  print(io, "$npts Curve{$T,$N}")
end

function Base.show(io::IO, ::MIME"text/plain", curve::Curve{T,N}) where {N,T}
  println(io, curve)
  Base.print_array(io, curve.coords)
end


text:
module Mitosis

using UnPack
using Statistics, StatsBase
using LinearAlgebra
using Random
using MeasureTheory
import MeasureTheory: kernel

"""
    kernel(f, M)
    kernel((f1, f2, ...), M)

A kernel `κ = kernel(f, M)` returns a wrapper around
a function `f` giving the parameters for a measure of type `M`,
such that `κ(x) = M(f(x)...)`
respective `κ(x) = M(f1(x), f2(x), ...)`.

If the argument is a named tuple `(;a=f1, b=f1)`, `κ(x)` is defined as
`M(;a=f(x),b=g(x))`.

# Reference

* https://en.wikipedia.org/wiki/Markov_kernel
"""
kernel

using MeasureTheory: AbstractMeasure
import MeasureTheory: density, logdensity
import Base: iterate, length
import Random.rand
import StatsBase.sample

export Gaussian, Copy, fuse, weighted
export Traced, BFFG, left′, right′, forward, backward, backwardfilter, forwardsampler
export BF, density, logdensity, ⊕, kernel, correct, Kernel, WGaussian, Gaussian, ConstantMap, AffineMap, LinearMap, GaussKernel

function independent_sum
end
const ⊕ = independent_sum

const ∅ = nothing

abstract type Context end

"""
    BFFG()

Backward filter forward guiding context for non-linear Gaussian
systems with `h` parametrized by `WGaussian{(:F,:Γ,:c)}`` (see Theorem 7.1 [Automatic BFFG].)
"""
struct BFFG <: Context
end

"""
    BF()

Backward filter for linear Gaussian systems parametrized
by mean and covariance of the backward filtered marginal distribution.
"""
struct BF <: Context
end

include("linearalgebra.jl")

macro F(f) :(::typeof($f)) end
struct Leaf{T}
    y::T
end
Base.getindex(y::Leaf) = y.y

struct Copy{N}
end
(a::Copy{2})(x) = (x, x)

function forward
end
function backward
end
const left′ = forward
const right′ = backward

backwardfilter(k, a; kargs...) = backward(BFFG(), k, a; kargs...)
forwardsampler(k, m, x; kargs...) = forward(BFFG(), k, m, x; kargs...)

fuse(a; kargs...) = backward(BFFG(), Copy{1}(), a; kargs...)
fuse(a, b; kargs...) = backward(BFFG(), Copy{2}(), a, b; kargs...)
fuse(a, b, c; kargs...) = backward(BFFG(), Copy{3}(), a, b, c; kargs...)

struct Traced{T}
    x::T
end

struct Weighted{S,T}
    x::S
    ll::T
end
weighted(x) = Weighted(x, 0.0)
weighted(x, ll) = Weighted(x, ll)
weighted(x::Weighted, ll) = Weighted(x.x, ll + x.ll)
Base.getindex(x::Weighted) = x.x
include("gauss.jl")
include("wgaussian.jl")
include("markov.jl")
include("rules.jl")
include("regression.jl")

end # module


text:
using Random
using .Models

"""
A finite size PEPS type.
"""
struct PEPS
  data::Matrix{ITensor}
end

PEPS(Nx::Int, Ny::Int) = PEPS(Matrix{ITensor}(undef, Nx, Ny))

"""
    PEPS([::Type{ElT} = Float64, sites; linkdims=1)
Construct an PEPS filled with Empty ITensors of type `ElT` from a collection of indices.
Optionally specify the link dimension with the keyword argument `linkdims`, which by default is 1.
"""
function PEPS(::Type{T}, sites::Matrix{<:Index}; linkdims::Integer=1) where {T<:Number}
  Ny, Nx = size(sites)
  tensor_grid = Matrix{ITensor}(undef, Ny, Nx)
  # we assume the PEPS at least has size (2,2). Can generalize if necessary
  @assert(Nx >= 2 && Ny >= 2)

  lh = Matrix{Index}(undef, Ny, Nx - 1)
  for ii in 1:(Nx - 1)
    for jj in 1:(Ny)
      lh[jj, ii] = Index(linkdims, "Lh-$jj-$ii")
    end
  end
  lv = Matrix{Index}(undef, Ny - 1, Nx)
  for ii in 1:(Nx)
    for jj in 1:(Ny - 1)
      lv[jj, ii] = Index(linkdims, "Lv-$jj-$ii")
    end
  end

  # boundary cases
  tensor_grid[1, 1] = ITensor(T, lh[1, 1], lv[1, 1], sites[1, 1])
  tensor_grid[1, Nx] = ITensor(T, lh[1, Nx - 1], lv[1, Nx], sites[1, Nx])
  tensor_grid[Ny, 1] = ITensor(T, lh[Ny, 1], lv[Ny - 1, 1], sites[Ny, 1])
  tensor_grid[Ny, Nx] = ITensor(T, lh[Ny, Nx - 1], lv[Ny - 1, Nx], sites[Ny, Nx])
  for ii in 2:(Nx - 1)
    tensor_grid[1, ii] = ITensor(T, lh[1, ii], lh[1, ii - 1], lv[1, ii], sites[1, ii])
    tensor_grid[Ny, ii] = ITensor(
      T, lh[Ny, ii], lh[Ny, ii - 1], lv[Ny - 1, ii], sites[Ny, ii]
    )
  end

  # inner sites
  for jj in 2:(Ny - 1)
    tensor_grid[jj, 1] = ITensor(T, lh[jj, 1], lv[jj, 1], lv[jj - 1, 1], sites[jj, 1])
    tensor_grid[jj, Nx] = ITensor(
      T, lh[jj, Nx - 1], lv[jj, Nx], lv[jj - 1, Nx], sites[jj, Nx]
    )
    for ii in 2:(Nx - 1)
      tensor_grid[jj, ii] = ITensor(
        T, lh[jj, ii], lh[jj, ii - 1], lv[jj, ii], lv[jj - 1, ii], sites[jj, ii]
      )
    end
  end

  return PEPS(tensor_grid)
end

PEPS(sites::Matrix{<:Index}, args...; kwargs...) = PEPS(Float64, sites, args...; kwargs...)

function Random.randn!(P::PEPS)
  randn!.(P.data)
  normalize!.(P.data)
  return P
end

Base.:+(A::PEPS, B::PEPS) = broadcast_add(A, B)

broadcast_add(A::PEPS, B::PEPS) = PEPS(A.data .+ B.data)

broadcast_minus(A::PEPS, B::PEPS) = PEPS(A.data .- B.data)

broadcast_mul(c::Number, A::PEPS) = PEPS(c .* A.data)

broadcast_inner(A::PEPS, B::PEPS) = mapreduce(v -> v[], +, A.data .* B.data)

ITensors.prime(P::PEPS, n::Integer=1) = PEPS(map(x -> prime(x, n), P.data))

# prime a PEPS with specified indices
function ITensors.prime(indices::Array{<:Index,1}, P::PEPS, n::Integer=1)
  return PEPS(prime(indices, P.data, n))
end

# prime linkinds of a PEPS
function ITensors.prime(::typeof(linkinds), P::PEPS, n::Integer=1)
  return PEPS(mapinds(x -> prime(x, n), linkinds, P.data))
end

function ITensors.addtags(::typeof(linkinds), P::PEPS, args...)
  return PEPS(addtags(linkinds, P.data, args...))
end

function ITensors.removetags(::typeof(linkinds), P::PEPS, args...)
  return PEPS(removetags(linkinds, P.data, args...))
end

ITensors.data(P::PEPS) = P.data

split_network(P::PEPS, rotation=false) = PEPS(split_network(data(P); rotation=rotation))

function ITensors.commoninds(p1::PEPS, p2::PEPS)
  return commoninds(p1.data, p2.data)
end

function flatten(v::Array{<:PEPS})
  tensor_list = [vcat(peps.data...) for peps in v]
  return vcat(tensor_list...)
end

function rayleigh_quotient(inners::Array)
  self_inner = inners[end][]
  expectations = sum(inners[1:(end - 1)])[]
  return expectations / self_inner
end

include("tree.jl")
include("inner_networks.jl")
include("projectors.jl")
include("chain_rules.jl")


text:
using Documenter
using LearningHorse.Preprocessing
using LearningHorse.NeuralNetwork
using LearningHorse.Regression
using LearningHorse.LossFunction
using LearningHorse.Classification
using LearningHorse.Tree

makedocs(;
    sitename="LearningHorse",
    pages = [
        "Home" => "index.md",
        "Tutorial" => [
            "Welcome to LearningHorse" => "Tutorial/Welcome.md",
            "Getting Started" => "Tutorial/Getting_Started.md",
            "Preprocessing" => "Tutorial/Preprocessing.md",
            "Classifiers" => "Tutorial/Classifiers.md",
            "Tree" => "Tutorial/Tree.md",
            "NeuralNetwork" => "Tutorial/NeuralNetwork.md"
        ],
        "Manual" => [
            "Regression" => "Manual/Regression.md",
            "Classification" => "Manual/Classification.md",
            "Preprocessing" => "Manual/Preprocessing.md",
            "LossFunction" => "Manual/LossFunction.md",
            "NeuralNetwork" => "Manual/NeuralNetwork.md",
        ]
    ]
)

deploydocs(
    repo = "github.com/MommaWatasu/LearningHorse.jl.git",
)


text:
### Reference state
using DocStringExtensions
using ..TemperatureProfiles
export NoReferenceState, HydrostaticState

using CLIMAParameters.Planet: R_d, MSLP, cp_d, grav, T_surf_ref, T_min_ref

"""
    ReferenceState

Reference state, for example, used as initial
condition or for linearization.
"""
abstract type ReferenceState end

vars_state_conservative(m::ReferenceState, FT) = @vars()
vars_state_gradient(m::ReferenceState, FT) = @vars()
vars_state_gradient_flux(m::ReferenceState, FT) = @vars()
vars_state_auxiliary(m::ReferenceState, FT) = @vars()
atmos_init_aux!(
    ::ReferenceState,
    ::AtmosModel,
    aux::Vars,
    geom::LocalGeometry,
) = nothing

"""
    NoReferenceState <: ReferenceState

No reference state used
"""
struct NoReferenceState <: ReferenceState end

"""
    HydrostaticState{P,T} <: ReferenceState

A hydrostatic state specified by a virtual temperature profile and relative humidity.
"""
struct HydrostaticState{P, FT} <: ReferenceState
    virtual_temperature_profile::P
    relative_humidity::FT
end
function HydrostaticState(
    virtual_temperature_profile::TemperatureProfile{FT},
) where {FT}
    return HydrostaticState{typeof(virtual_temperature_profile), FT}(
        virtual_temperature_profile,
        FT(0),
    )
end

vars_state_auxiliary(m::HydrostaticState, FT) =
    @vars(ρ::FT, p::FT, T::FT, ρe::FT, ρq_tot::FT)

function atmos_init_aux!(
    m::HydrostaticState{P, F},
    atmos::AtmosModel,
    aux::Vars,
    geom::LocalGeometry,
) where {P, F}
    z = altitude(atmos, aux)
    T_virt, p = m.virtual_temperature_profile(atmos.param_set, z)
    FT = eltype(aux)
    _R_d::FT = R_d(atmos.param_set)

    ρ = p / (_R_d * T_virt)
    aux.ref_state.ρ = ρ
    aux.ref_state.p = p
    # We evaluate the saturation vapor pressure, approximating
    # temperature by virtual temperature
    # ts = TemperatureSHumEquil(atmos.param_set, T_virt, ρ, FT(0))
    ts = PhaseDry_given_ρT(atmos.param_set, ρ, T_virt)
    q_vap_sat = q_vap_saturation(ts)

    ρq_tot = ρ * relative_humidity(m) * q_vap_sat
    aux.ref_state.ρq_tot = ρq_tot

    q_pt = PhasePartition(ρq_tot)
    R_m = gas_constant_air(atmos.param_set, q_pt)
    T = T_virt * R_m / _R_d
    aux.ref_state.T = T
    aux.ref_state.ρe = ρ * internal_energy(atmos.param_set, T, q_pt)

    e_kin = F(0)
    e_pot = gravitational_potential(atmos.orientation, aux)
    aux.ref_state.ρe = ρ * total_energy(atmos.param_set, e_kin, e_pot, T, q_pt)
end

"""
    relative_humidity(hs::HydrostaticState{P,FT})

Here, we enforce that relative humidity is zero
for a dry adiabatic profile.
"""
relative_humidity(hs::HydrostaticState{P, FT}) where {P, FT} =
    hs.relative_humidity
relative_humidity(hs::HydrostaticState{DryAdiabaticProfile, FT}) where {FT} =
    FT(0)


text:
################################################################################
#
#  Pure and Kummer extensions
#
################################################################################

@doc Markdown.doc"""
    isradical_extension(L::SimpleNumField) -> Bool

Tests if $L/K$ is pure, that is, if the defining polynomial is of the form
$x^n - b$ for some $b \in K$.
"""
function isradical_extension(K::SimpleNumField)
  if !ismonic(K.pol)
    return false
  end
  return all(i -> iszero(coeff(K.pol, i)), 1:degree(K)-1)
end

@doc Markdown.doc"""
    iskummer_extension(L::SimpleNumField) -> Bool

Tests if $L/K$ is a Kummer extension, that is, if the defining polynomial is
of the form $x^n - b$ for some $b \in K$ and if $K$ contains the $n$-th roots
of unity.
"""
function iskummer_extension(K::SimpleNumField)
  if !isradical_extension(K)
    return false
  end

  k = base_field(K)
  Zk = maximal_order(k)
  _, o = torsion_units_gen_order(Zk)
  if o % degree(K) != 0
    return false
  end
  return true
end

function iskummer_extension(K::AnticNumberField)
  if degree(K) != 2
    return false
  end
  return isradical_extension(K)
end

function radical_extension(n::Int, a::FacElem;
                        cached::Bool = true, check::Bool = true)
  return radical_extension(n, evaluate(a), cached = cached, check = check)
end

@doc Markdown.doc"""
    radical_extension(n::Int, a::NumFieldElem; s = "_$",
                   check = true, cached = true) -> NumField, NumFieldElem

Given an element $a$ of a number field $K$ and an integer $n$, create the simple
extension of $K$ with the defining polynomial $x^n - a$.
"""
function radical_extension(n::Int, a::NumFieldElem; s::String = "_\$",
                        cached::Bool = true, check::Bool = true)
  k = parent(a)
  kx, x = PolynomialRing(k, cached = false)
  return number_field(x^n - a, s, check = check, cached = cached)
end

function radical_extension(n::Int, a::fmpq; s::String = "_\$",
                        cached::Bool = true, check::Bool = true)
  k = parent(a)
  kx, x = PolynomialRing(k, cached = false)
  return number_field(x^n - a, s, check = check, cached = cached)
end

################################################################################
#
#  Basis
#
################################################################################

@doc Markdown.doc"""
    basis(L::SimpleNumField) -> Vector{NumFieldElem}

Returns the canonical basis of a simple extension $L/K$, that is, the elements
$1,a,\dotsc,a^{d - 1}$, where $d$ is the degree of $K$ and $a$ the primitive
element.
"""
basis(::SimpleNumField)

################################################################################
#
#  Defining polynomial
#
################################################################################

export defining_polynomial

@doc Markdown.doc"""
    defining_polynomial(L::SimpleNumField) -> PolyElem

Given a simple number field $L/K$, constructed as $L = K[x]/(f)$, this function
returns $f$.
"""
defining_polynomial(::SimpleNumField)

defining_polynomial(K::NfRel) = K.pol

defining_polynomial(K::AnticNumberField) = K.pol

################################################################################
#
#  Discriminant
#
################################################################################

@doc Markdown.doc"""
    discriminant(L::SimpleNumField) -> NumFieldElem

The discriminant of the defining polynomial of $L$, *not* the discriminant of
the maximal order of $L$.
"""
function discriminant(K::SimpleNumField)
  return discriminant(defining_polynomial(K))
end

################################################################################
#
#  Absolute discriminant
#
################################################################################

@doc Markdown.doc"""
    absolute_discriminant(L::SimpleNumField, QQ) -> fmpq

The absolute discriminant of the defining polynomial of $L$, *not* the
discriminant of the maximal order of $L$. This is the norm of the discriminant
times the $d$-th power of the discriminant of the base field, where $d$ is the
degree of $L$.
"""
absolute_discriminant(::SimpleNumField)

function absolute_discriminant(K::AnticNumberField)
  return discriminant(K)
end

function absolute_discriminant(K::NfRel)
  d = norm(discriminant(K)) * absolute_discriminant(base_field(K))^degree(K)
  return d
end

function discriminant(K::FlintRationalField)
  return one(K)
end

################################################################################
#
#  Is subfield
#
################################################################################

@doc Markdown.doc"""
    issubfield(K::SimpleNumField, L::SimpleNumField) -> Bool, Map

Returns `true` and an injection from $K$ to $L$ if $K$ is a subfield of $L$.
Otherwise the function returns `false` and a morphism mapping everything to
$0$.
"""
issubfield(::SimpleNumField, ::SimpleNumField)

@doc Markdown.doc"""
    isisomorphic(K::SimpleNumField, L::SimpleNumField) -> Bool, Map

Returns `true` and an isomorphism from $K$ to $L$ if $K$ and $L$ are isomorphic.
Otherwise the function returns `false` and a morphism mapping everything to $0$.
"""
isisomorphic(::SimpleNumField, ::SimpleNumField)

################################################################################
#
#  Linear disjointness
#
################################################################################

# TODO (easy): Do this for Non-Simple number fields
@doc Markdown.doc"""
    islinear_disjoint(K::SimpleNumField, L::SimpleNumField) -> Bool

Given two number fields $K$ and $L$ with the same base field $k$, this function
returns whether $K$ and $L$ are linear disjoint over $k$.
"""
islinear_disjoint(K::SimpleNumField, L::SimpleNumField)


text:
let EV         = evs.EV,
    dEV        = evs.dEV,
    σ          = σv,
    wp         = prim.wp,
    Πz         = prim.Πz,
    β          = prim.β,
    fdEV       = similar(evs.dEV),
    θ1         = similar(θt),
    θ2         = similar(θt),
    roy        = 0.25,
    geoid      = 4.7,
    itype      = (geoid, roy,),
    nS         = length(prim.wp),
    i          = nS-1,
    idxd       = ShaleDrillingModel.dp1space(wp,i),
    idxs       = collect(ShaleDrillingModel.sprimes(wp,i)),
    horzn      = ShaleDrillingModel._horizon(wp,i)

    @views ubV =    tmpv.ubVfull[:,:,idxd]
    @views dubV =   tmpv.dubVfull[:,:,:,idxd]
    @views dubV_σ = tmpv.dubV_σ[:,:,idxd]
    @views q      = tmpv.q[:,:,idxd]

    @views EV0 = EV[:,:,i]
    @views EV1 = EV[:,:,idxs]
    @views dEV0 = dEV[:,:,:,i]
    @views dEV1 = dEV[:,:,:,idxs]
    @views fdEV0 = fdEV[:,:,:,i]

    for anticipate_e ∈ (true,false,)

        tmpv_vw = dcdp_tmpvars(ubV, dubV, dubV_σ, q, tmpv.lse, tmpv.tmp, tmpv.tmp_cart, tmpv.Πψtmp, tmpv.IminusTEVp)
        p = dcdp_primitives(flowfuncname, prim.β, prim.wp, prim.zspace, prim.Πz, prim.ψspace, anticipate_e)

        @testset "Check finite horizon gradient for anticipate_e = $anticipate_e" begin
            for k = 1:length(θt)
                h = peturb(θt[k])
                θ1 .= θt
                θ2 .= θt
                θ1[k] -= h
                θ2[k] += h
                hh = θ2[k] - θ1[k]

                fill!(EV, 0.0)
                fillflows!(ubV, flow, p, θ1, σ, i, itype...)
                ubV .+= β .* EV1
                ShaleDrillingModel.vfit!(EV0, tmpv_vw, p)
                fdEV0[:,:,k] .= -EV0

                fill!(EV, 0.0)
                fillflows!(ubV, flow, p, θ2, σ, i, itype...)
                ubV .+= β .* EV1
                ShaleDrillingModel.vfit!(EV0, tmpv_vw, p)
                fdEV0[:,:,k] .+= EV0
                fdEV0[:,:,k] ./= hh
            end

            fill!(EV, 0.0)
            fill!(dEV, 0.0)
            fillflows!(ubV, flow, p, θt, σ, i, itype...)
            fillflows_grad!(dubV, flowdθ, p, θt, σ, i, itype...)
            ubV .+= β .* EV1
            dubV .+= β .* dEV1
            ShaleDrillingModel.vfit!(EV0, dEV0, tmpv_vw, p)

            println("extrema(dEV0) = $(extrema(dEV0))")
            @views maxv, idx = findmax(abs.(fdEV0 .- dEV0))
            @views sub = CartesianIndices(fdEV0)[idx]
            println("worst value is $maxv at $sub for dθ")

            @test 0.0 < maxv < 1.0
            @test all(isfinite.(dEV0))
            @test all(isfinite.(fdEV0))
            @test fdEV0 ≈ dEV0
        end

        @testset "Check vfit/pfit for infinite horizon, anticipate_e = $anticipate_e" begin
            vfEV0 = zeros(size(EV0))

            fill!(EV, 0.0)
            fillflows!(ubV, flow, p, θt, σ, i, itype...)
            ubV .+= β .* EV1
            converged, iter, bnds = ShaleDrillingModel.solve_inf_vfit!(EV0, tmpv_vw, p; maxit=1000, vftol=1e-10)
            println("vfit done. converged = $converged after $iter iterations. error bounds are $bnds")
            vfEV0 .= EV0

            fill!(EV, 0.0)
            fillflows!(ubV, flow, p, θt, σ, i, itype...)
            ubV .+= β .* EV1
            converged, iter, bnds = ShaleDrillingModel.solve_inf_vfit_pfit!(EV0, tmpv_vw, p; vftol=1e-10, maxit0=20, maxit1=40)
            println("pfit done. converged = $converged after $iter iterations. error bounds are $bnds")

            @views maxv, idx = findmax(abs.(vfEV0 .- EV0))
            @views sub = CartesianIndices(EV0)[idx]
            println("worst value is $maxv at $sub for vfit vs pfit")
            @test EV0 ≈ vfEV0
        end

        @testset "Check gradient for infinite horizon, anticipate_e = $anticipate_e" begin

            for k = 1:length(θt)
                h = peturb(θt[k])
                θ1 .= θt
                θ2 .= θt
                θ1[k] -= h
                θ2[k] += h
                hh = θ2[k] - θ1[k]

                fill!(EV, 0.0)
                fillflows!(ubV, flow, p, θ1, σ, i, itype...)
                ubV .+= β .* EV1
                ShaleDrillingModel.solve_inf_vfit_pfit!(EV0, tmpv_vw, p; vftol=1e-10, maxit0=20, maxit1=40)
                fdEV0[:,:,k] .= -EV0

                fill!(EV, 0.0)
                fillflows!(ubV, flow, p, θ2, σ, i, itype...)
                ubV .+= β .* EV1
                ShaleDrillingModel.solve_inf_vfit_pfit!(EV0, tmpv_vw, p; vftol=1e-10, maxit0=20, maxit1=40)
                fdEV0[:,:,k] .+= EV0
                fdEV0[:,:,k] ./= hh
            end

            fill!(EV, 0.0)
            fill!(dEV, 0.0)
            fillflows!(ubV, flow, p, θt, σ, i, itype...)
            fillflows_grad!(dubV, flowdθ, p, θt, σ, i, itype...)
            ubV .+= β .* EV1
            dubV .+= β .* dEV1
            ShaleDrillingModel.solve_inf_vfit_pfit!(EV0, tmpv_vw, p; vftol=1e-10, maxit0=20, maxit1=40)
            ubV[:,:,1] .= β .* EV0
            ShaleDrillingModel.gradinf!(dEV0, tmpv_vw, p)   # note: destroys ubV & dubV

            println("extrema(dEV0) = $(extrema(dEV0))")
            @views maxv, idx = findmax(abs.(fdEV0 .- dEV0))
            @views sub = CartesianIndices(fdEV0)[idx]
            println("worst value is $maxv at $sub for dθ")

            @test 0.0 < maxv < 1.0
            @test all(isfinite.(dEV0))
            @test all(isfinite.(fdEV0))
            @test fdEV0 ≈ dEV0

        end
    end
end


text:
"""
    grdinfo(cmd0::String="", arg1=nothing; kwargs...)

Reads a 2-D grid file and reports metadata and various statistics for the (x,y,z) data in the grid file

Full option list at [`grdinfo`]($(GMTdoc)grdinfo.html)

Parameters
----------

- **C** | **numeric** :: [Type => Str | Number]

    Formats the report using tab-separated fields on a single line.
    ($(GMTdoc)grdinfo.html#c)
- **D** | **tiles** :: [Type => Number | Str]

    Divide a single grid’s domain (or the -R domain, if no grid given) into tiles of size
    dx times dy (set via -I).
    ($(GMTdoc)grdinfo.html#d)
- **F** :: [Type => Bool]

    Report grid domain and x/y-increments in world mapping format.
    ($(GMTdoc)grdinfo.html#f)
- **I** | **nearest** :: [Type => Number | Str]     ``Arg = [dx[/dy]|b|i|r]``

    Report the min/max of the region to the nearest multiple of dx and dy, and output
    this in the form -Rw/e/s/n
    ($(GMTdoc)grdinfo.html#i)
- **L** | **force_scan** :: [Type => Number | Str]

    Report stats after actually scanning the data.
    ($(GMTdoc)grdinfo.html#l)
- **M** | **minmax_pos** :: [Type => Bool]

    Find and report the location of min/max z-values.
    ($(GMTdoc)grdinfo.html#m)
- $(GMT.opt_R)
- **T** | **zmin_max** :: [Type => Number | Str]
    Determine min and max z-value.
    ($(GMTdoc)grdinfo.html#t)
- $(GMT.opt_V)
- $(GMT.opt_f)
"""
function grdinfo(cmd0::String="", arg1=nothing; kwargs...)

	length(kwargs) == 0 && !isa(arg1, GMTgrid) && return monolitic("grdinfo", cmd0, arg1)

	d = KW(kwargs)
	help_show_options(d)			# Check if user wants ONLY the HELP mode
	cmd, = parse_common_opts(d, "", [:R :V_params :f])
	cmd  = parse_these_opts(cmd, d, [[:C :numeric], [:D :tiles], [:F], [:I :nearest],
	                                 [:L :force_scan], [:M :minmax_pos], [:T :zmin_max]])
	common_grd(d, cmd0, cmd, "grdinfo ", arg1)		# Finish build cmd and run it
end

# ---------------------------------------------------------------------------------------------------
grdinfo(arg1, cmd0::String=""; kw...) = grdinfo(cmd0, arg1; kw...)


text:
function transform_to_steps!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
   # @inbounds - заметного ускорения здесь это не даёт!!!
   for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
            # макрос @. заменяет все "сквозные точки"
            # макрос @views заменяет многократное применение макроса @view
        end
    end
    return A
end

function swap!(A,B)
    for i in eachindex(A)
        A[i], B[i] = B[i], A[i]
    end
end

function swap_2!(A, i, iMax, k)
    A[i, k], A[iMax, k] = A[iMax, k], A[i, k]
end

function transform_to_steps_2!(a::Matrix)
    countSwaps = 1
    c = a
    eps = 0.000001
    for i = 1:size(a, 1)
        iMax = i
        for j = i:size(a, 1)
            if abs(c[j, i]) > abs(c[iMax, i])
                iMax = j
            end
        end
        if abs(c[iMax, i]) >= eps
            for k = 1:size(a, 2)
                swap_2!(c, i, iMax, k)
            end
            countSwaps = -countSwaps * (i != iMax ? 1 : -1)

            for j = i+1:size(a, 1)
                q = -c[j, i] / c[i, i]
                for k = size(a, 2):-1:i
                    c[j, k] += q*c[i, k]
                    if abs(c[j, k]) < eps
                        c[j, k] = 0
                    end
                end
            end
        end
    end
    if countSwaps > 0
        c * -1
    end
    c
end

function transform_to_steps_edit!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
    ans = []
    for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
        end
    end
    for i = 1:size(A,1)
        for j = i:size(A,2)
            if abs(A[i, j]) > epsilon
                push!(ans, j)
                break;
            end
        end
    end
    return ans
end

function rank_(B::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
    ans = []
    A = copy(B)
    for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
        end
    end
    for i = 1:size(A,1)
        for j = i:size(A,2)
            if abs(A[i, j]) > epsilon
                push!(ans, j)
                break;
            end
        end
    end
    return length(ans)
end

function det_(a)
    ans = 1
    epsilon = 1e-7
    s = 1
    c = copy(a)
    for k ∈ 1:size(c, 1)
         absval, Δk = findmax(abs, @view(c[k:end,k]))
         Δk > 1 && swap!(@view(c[k,k:end]), @view(c[k+Δk-1,k:end]))
         if Δk > 1
             s*=-1
         end
         for i ∈ k+1:size(c,1)
             t = c[i,k]/c[k,k]
             @. @views c[i,k:end] = c[i,k:end] - t * c[k,k:end]
         end
    end
    for i = 1:size(c, 1)
        ans*=c[i, i]
    end
    ans*s
end

function slau(A, b)
    countSwaps = 1
    a = [A b]
    c = copy(a)
    eps = 0.000001
    for i = 1:size(a, 1)
        iMax = i
        for j = i:size(a, 1)
            if abs(c[j, i]) > abs(c[iMax, i])
                iMax = j
            end
        end
        if abs(c[iMax, i]) >= eps
            for k = 1:size(a, 2)
                swap_2!(c, i, iMax, k)
            end
            countSwaps = -countSwaps * (i != iMax ? 1 : -1)

            for j = i+1:size(a, 1)
                q = -c[j, i] / c[i, i]
                for k = size(a, 2):-1:i
                    c[j, k] += q*c[i, k]
                    if abs(c[j, k]) < eps
                        c[j, k] = 0
                    end
                end
            end
        end
    end
    if countSwaps > 0
        c * -1
    end
    c[size(a, 1), size(a, 1)+1] /= c[size(a, 1), size(a, 1)]
    c[size(a, 1), size(a, 1)] = 1
    v = []
    for i = size(a, 1):-1:1
        t = c[i, size(a, 1)+1]
        for j = i+1:size(a, 1)
            t-=c[j]*c[i,j]
        end
        c[size(a, 1), i] = t
    end
    c[size(a,1),1:size(a,1)]
end

function inv_(a)
    c = copy(a)
    for i = 1:size(c, 1)
        for j = i+1:size(c, 1)
            c[i,j], c[j,i] = c[j,i], c[i,j]
        end
    end
    c
end
function transform_to_low_steps!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
   # @inbounds - заметного ускорения здесь это не даёт!!!
   for k ∈ size(A, 1):-1:1
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k-1:-1:1
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
            # макрос @. заменяет все "сквоные точки"
            # макрос @views заменяет многократное применение макроса @view
        end
    end
    return A
end

max_stars_repo_name: Trilonka/MireaAl

max_issues_repo_name: Trilonka/MireaAl

max_forks_repo_name: Trilonka/MireaAl


text:
function cross_valid_inner(train::Array{StatisticalMoments{Float64}}, fm::FeatureMap, options::OptionsML, kernelname::AbstractString, kernelparams::Array{Float64}, crossnum::Int64, lambdas::Array{Float64}, output_type::AbstractString, saveresults::Bool, dataname::AbstractString)
  # crossnum  the number of parts the training data is divided into to perform the cross validation
  # kernelparams is a grid of kernel parameters
  numlam = length(lambdas);
  if(kernelname =="")
    numkernelparams = 1;
  else
    numkernelparams = length(kernelparams);
  end
  lambdasScore = zeros(numlam,numkernelparams);
  pairs = unique([ [td.label[2], td.label[1] ] for td in train]); # pairs of (radius, volume)
  lpr = length(pairs);
  s = sample(1:lpr, lpr,replace=false); # Calculat a reshuffling of the the set 1, ..., lpr
  pairs = pairs[s];   # Reshuffle the data
    for k =1:crossnum # selecting the kth training and test sets.
#         println("testing ",k, "th cross validation")
        divisor  = convert(Int64,floor(lpr/crossnum));
        testselection = (1+divisor*(k-1)):divisor*k;
        trainpairs = pairs[testselection];
        #Get the testpair coresponding to the crossnum th section
        testtemp = filter(td -> ([td.label[2],td.label[1]] in trainpairs), train);
        #Use all remaining data as the training set
        traintemp= filter(td -> !([td.label[2],td.label[1]] in trainpairs), train);
        #Setting up ml model
        ml = setup_ml(traintemp, fm,  kernelname,options, output_type);
        for i =1:numlam
            #println("Testing lambdas = (",lambdas[i],",",lambdas[js],")");
            lambda = lambdas[i];
            for ks = 1: numkernelparams
              kernelparam =   kernelparams[ks];
              fit_L2(ml, options,  kernelname, kernelparam, lambda);
              Rsqd= test_linear(testtemp, dataname, fm,  ml,options,output_type);
              lambdasScore[i,ks]=lambdasScore[i,ks]+Rsqd;
            end
         end
  end
  #get indexes of best pair of lambdas
  inds = findn(lambdasScore.==lambdasScore[indmax(lambdasScore)]);
  lambdabest  = lambdas[inds[1][1]];
  klparambest =   kernelparams[inds[2][1]];
#     println("inds[1] = $(inds[1][1])  and inds[2] =$(inds[2][1])")
    if(saveresults)
      savename = string(get_model_name(fm,kernelname,output_type,options),"-",dataname,"-crossvalid");
      default_path = "../data/"
      println("best lambda score = $(round(lambdasScore[inds[1][1],inds[2][1]]/(Float64(crossnum)),3)) was with lambda = $(lambdabest) and klparameter = $(klparambest)");
      println("saving the best lambda and kernalparameter in $(default_path)$(savename).jld")
      save("$(default_path)$(savename).jld", "lambdabest",lambdabest, "klparambest", klparambest,"lambdasScorebest",lambdasScore[inds[1][1],inds[2][1]]/(Float64(crossnum)));
    end
  return  lambdabest[1], klparambest[1]
end


text:
using UnicodeFun

"""
    base_exp(v)

Convert the float `v` into a tuple of base and exponent in base-10.

# Examples

```julia-repl
julia> SolverTraces.base_exp(-3.5e2)
(-3.5000000000000004, 2)
```
"""
function base_exp(v::T) where {T<:AbstractFloat}
    v == zero(T) && return (zero(T),0)
    r = log10(abs(v))
    e = round(Int,r)
    b = T(10)^(r-e)
    if abs(b) < 1 && abs(abs(b)-1) > 1e-2
        b *= 10
        e -= 1
    end
    sign(v)*b,e
end

"""
    Tolerance(target, current, fmt, tol_fmt, lc, header)

Column displaying the progress of the algorithm towards a set
`target`. At each iteration, the `current` value has to be updated.
"""
mutable struct Tolerance{T<:AbstractFloat} <: TraceColumn
    target::T
    current::T
    fmt::FormatExpr
    tol_fmt::FormatExpr
    lc::LinearColorant{T}
    header::String
end

"""
     Tolerance(target[, header; print_target])

Construct a new [`Tolerance`](@ref) column with a set `target`.
"""
function Tolerance(target::T,header="Tolerance";print_target::Bool=true) where T
    tb,te = base_exp(target)
    tol_fmt = FormatExpr("{1:.2f}×10{2:3s}")
    target_str = if print_target
        "(" * (tb == one(T) ? "10$(to_superscript(te))" : strip(format(tol_fmt,tb,to_superscript(te)))) * ")"
    else
        ""
    end
    spacer = repeat(" ", max(0,length(header)-length(target_str)-10))
    fmt = FormatExpr("{1:s}{2:10s}$(crayon"reset")$(target_str)$spacer")
    lc = LinearColorant(one(T),zero(T),red_green_scale())
    Tolerance{T}(target,T(Inf),fmt,tol_fmt,lc,rpad(header,length(target_str)+10))
end

function(t::Tolerance{T})(i::Integer) where T
    if isinf(t.current) || isnan(t.current)
        t.lc(one(T)),string(t.current)
    else
        tb,te = base_exp(abs(t.current))
        (t.lc((log10(t.target)-log10(t.current))/log10(t.target)),
         format(t.tol_fmt,tb,to_superscript(te)))
    end
end

export Tolerance


text:
using ..common
using Compat
using ProtoBuf
import ProtoBuf.meta
import Base: hash, isequal, ==

type __enum_Status <: ProtoEnum
    SUCCESS::Int32
    ERROR::Int32
    ERROR_CHECKSUM::Int32
    ERROR_INVALID::Int32
    ERROR_EXISTS::Int32
    ERROR_ACCESS_TOKEN::Int32
    CHECKSUM_OK::Int32
    ERROR_UNSUPPORTED::Int32
    OOB_RESTART::Int32
    OOB_RESERVED1::Int32
    OOB_RESERVED2::Int32
    OOB_RESERVED3::Int32
    IN_PROGRESS::Int32
    __enum_Status() = new(0,1,2,3,4,5,6,7,8,9,10,11,12)
end #type __enum_Status
const Status = __enum_Status()

type __enum_ShortCircuitFdResponse <: ProtoEnum
    DO_NOT_USE_RECEIPT_VERIFICATION::Int32
    USE_RECEIPT_VERIFICATION::Int32
    __enum_ShortCircuitFdResponse() = new(0,1)
end #type __enum_ShortCircuitFdResponse
const ShortCircuitFdResponse = __enum_ShortCircuitFdResponse()

type __enum_DataTransferEncryptorMessageProto_DataTransferEncryptorStatus <: ProtoEnum
    SUCCESS::Int32
    ERROR_UNKNOWN_KEY::Int32
    ERROR::Int32
    __enum_DataTransferEncryptorMessageProto_DataTransferEncryptorStatus() = new(0,1,2)
end #type __enum_DataTransferEncryptorMessageProto_DataTransferEncryptorStatus
const DataTransferEncryptorMessageProto_DataTransferEncryptorStatus = __enum_DataTransferEncryptorMessageProto_DataTransferEncryptorStatus()

type DataTransferEncryptorMessageProto
    status::Int32
    payload::Array{UInt8,1}
    message::AbstractString
    cipherOption::Array{CipherOptionProto,1}
    DataTransferEncryptorMessageProto() = (o=new(); fillunset(o); o)
end #type DataTransferEncryptorMessageProto
const __req_DataTransferEncryptorMessageProto = Symbol[:status]
meta(t::Type{DataTransferEncryptorMessageProto}) = meta(t, __req_DataTransferEncryptorMessageProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::DataTransferEncryptorMessageProto) = ProtoBuf.protohash(v)
isequal(v1::DataTransferEncryptorMessageProto, v2::DataTransferEncryptorMessageProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::DataTransferEncryptorMessageProto, v2::DataTransferEncryptorMessageProto) = ProtoBuf.protoeq(v1, v2)

type DataTransferTraceInfoProto
    traceId::UInt64
    parentId::UInt64
    DataTransferTraceInfoProto() = (o=new(); fillunset(o); o)
end #type DataTransferTraceInfoProto
const __req_DataTransferTraceInfoProto = Symbol[:traceId,:parentId]
meta(t::Type{DataTransferTraceInfoProto}) = meta(t, __req_DataTransferTraceInfoProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::DataTransferTraceInfoProto) = ProtoBuf.protohash(v)
isequal(v1::DataTransferTraceInfoProto, v2::DataTransferTraceInfoProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::DataTransferTraceInfoProto, v2::DataTransferTraceInfoProto) = ProtoBuf.protoeq(v1, v2)

type BaseHeaderProto
    block::ExtendedBlockProto
    token::TokenProto
    traceInfo::DataTransferTraceInfoProto
    BaseHeaderProto() = (o=new(); fillunset(o); o)
end #type BaseHeaderProto
const __req_BaseHeaderProto = Symbol[:block]
meta(t::Type{BaseHeaderProto}) = meta(t, __req_BaseHeaderProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::BaseHeaderProto) = ProtoBuf.protohash(v)
isequal(v1::BaseHeaderProto, v2::BaseHeaderProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::BaseHeaderProto, v2::BaseHeaderProto) = ProtoBuf.protoeq(v1, v2)

type ClientOperationHeaderProto
    baseHeader::BaseHeaderProto
    clientName::AbstractString
    ClientOperationHeaderProto() = (o=new(); fillunset(o); o)
end #type ClientOperationHeaderProto
const __req_ClientOperationHeaderProto = Symbol[:baseHeader,:clientName]
meta(t::Type{ClientOperationHeaderProto}) = meta(t, __req_ClientOperationHeaderProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ClientOperationHeaderProto) = ProtoBuf.protohash(v)
isequal(v1::ClientOperationHeaderProto, v2::ClientOperationHeaderProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ClientOperationHeaderProto, v2::ClientOperationHeaderProto) = ProtoBuf.protoeq(v1, v2)

type CachingStrategyProto
    dropBehind::Bool
    readahead::Int64
    CachingStrategyProto() = (o=new(); fillunset(o); o)
end #type CachingStrategyProto
hash(v::CachingStrategyProto) = ProtoBuf.protohash(v)
isequal(v1::CachingStrategyProto, v2::CachingStrategyProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::CachingStrategyProto, v2::CachingStrategyProto) = ProtoBuf.protoeq(v1, v2)

type OpReadBlockProto
    header::ClientOperationHeaderProto
    offset::UInt64
    len::UInt64
    sendChecksums::Bool
    cachingStrategy::CachingStrategyProto
    OpReadBlockProto() = (o=new(); fillunset(o); o)
end #type OpReadBlockProto
const __req_OpReadBlockProto = Symbol[:header,:offset,:len]
const __val_OpReadBlockProto = @compat Dict(:sendChecksums => true)
meta(t::Type{OpReadBlockProto}) = meta(t, __req_OpReadBlockProto, ProtoBuf.DEF_FNUM, __val_OpReadBlockProto, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpReadBlockProto) = ProtoBuf.protohash(v)
isequal(v1::OpReadBlockProto, v2::OpReadBlockProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpReadBlockProto, v2::OpReadBlockProto) = ProtoBuf.protoeq(v1, v2)

type ChecksumProto
    _type::Int32
    bytesPerChecksum::UInt32
    ChecksumProto() = (o=new(); fillunset(o); o)
end #type ChecksumProto
const __req_ChecksumProto = Symbol[:_type,:bytesPerChecksum]
meta(t::Type{ChecksumProto}) = meta(t, __req_ChecksumProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ChecksumProto) = ProtoBuf.protohash(v)
isequal(v1::ChecksumProto, v2::ChecksumProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ChecksumProto, v2::ChecksumProto) = ProtoBuf.protoeq(v1, v2)

type __enum_OpWriteBlockProto_BlockConstructionStage <: ProtoEnum
    PIPELINE_SETUP_APPEND::Int32
    PIPELINE_SETUP_APPEND_RECOVERY::Int32
    DATA_STREAMING::Int32
    PIPELINE_SETUP_STREAMING_RECOVERY::Int32
    PIPELINE_CLOSE::Int32
    PIPELINE_CLOSE_RECOVERY::Int32
    PIPELINE_SETUP_CREATE::Int32
    TRANSFER_RBW::Int32
    TRANSFER_FINALIZED::Int32
    __enum_OpWriteBlockProto_BlockConstructionStage() = new(0,1,2,3,4,5,6,7,8)
end #type __enum_OpWriteBlockProto_BlockConstructionStage
const OpWriteBlockProto_BlockConstructionStage = __enum_OpWriteBlockProto_BlockConstructionStage()

type OpWriteBlockProto
    header::ClientOperationHeaderProto
    targets::Array{DatanodeInfoProto,1}
    source::DatanodeInfoProto
    stage::Int32
    pipelineSize::UInt32
    minBytesRcvd::UInt64
    maxBytesRcvd::UInt64
    latestGenerationStamp::UInt64
    requestedChecksum::ChecksumProto
    cachingStrategy::CachingStrategyProto
    storageType::Int32
    targetStorageTypes::Array{Int32,1}
    allowLazyPersist::Bool
    pinning::Bool
    targetPinnings::Array{Bool,1}
    OpWriteBlockProto() = (o=new(); fillunset(o); o)
end #type OpWriteBlockProto
const __req_OpWriteBlockProto = Symbol[:header,:stage,:pipelineSize,:minBytesRcvd,:maxBytesRcvd,:latestGenerationStamp,:requestedChecksum]
const __val_OpWriteBlockProto = @compat Dict(:storageType => StorageTypeProto.DISK, :allowLazyPersist => false, :pinning => false)
meta(t::Type{OpWriteBlockProto}) = meta(t, __req_OpWriteBlockProto, ProtoBuf.DEF_FNUM, __val_OpWriteBlockProto, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpWriteBlockProto) = ProtoBuf.protohash(v)
isequal(v1::OpWriteBlockProto, v2::OpWriteBlockProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpWriteBlockProto, v2::OpWriteBlockProto) = ProtoBuf.protoeq(v1, v2)

type OpTransferBlockProto
    header::ClientOperationHeaderProto
    targets::Array{DatanodeInfoProto,1}
    targetStorageTypes::Array{Int32,1}
    OpTransferBlockProto() = (o=new(); fillunset(o); o)
end #type OpTransferBlockProto
const __req_OpTransferBlockProto = Symbol[:header]
meta(t::Type{OpTransferBlockProto}) = meta(t, __req_OpTransferBlockProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpTransferBlockProto) = ProtoBuf.protohash(v)
isequal(v1::OpTransferBlockProto, v2::OpTransferBlockProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpTransferBlockProto, v2::OpTransferBlockProto) = ProtoBuf.protoeq(v1, v2)

type OpReplaceBlockProto
    header::BaseHeaderProto
    delHint::AbstractString
    source::DatanodeInfoProto
    storageType::Int32
    OpReplaceBlockProto() = (o=new(); fillunset(o); o)
end #type OpReplaceBlockProto
const __req_OpReplaceBlockProto = Symbol[:header,:delHint,:source]
const __val_OpReplaceBlockProto = @compat Dict(:storageType => StorageTypeProto.DISK)
meta(t::Type{OpReplaceBlockProto}) = meta(t, __req_OpReplaceBlockProto, ProtoBuf.DEF_FNUM, __val_OpReplaceBlockProto, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpReplaceBlockProto) = ProtoBuf.protohash(v)
isequal(v1::OpReplaceBlockProto, v2::OpReplaceBlockProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpReplaceBlockProto, v2::OpReplaceBlockProto) = ProtoBuf.protoeq(v1, v2)

type OpCopyBlockProto
    header::BaseHeaderProto
    OpCopyBlockProto() = (o=new(); fillunset(o); o)
end #type OpCopyBlockProto
const __req_OpCopyBlockProto = Symbol[:header]
meta(t::Type{OpCopyBlockProto}) = meta(t, __req_OpCopyBlockProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpCopyBlockProto) = ProtoBuf.protohash(v)
isequal(v1::OpCopyBlockProto, v2::OpCopyBlockProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpCopyBlockProto, v2::OpCopyBlockProto) = ProtoBuf.protoeq(v1, v2)

type OpBlockChecksumProto
    header::BaseHeaderProto
    OpBlockChecksumProto() = (o=new(); fillunset(o); o)
end #type OpBlockChecksumProto
const __req_OpBlockChecksumProto = Symbol[:header]
meta(t::Type{OpBlockChecksumProto}) = meta(t, __req_OpBlockChecksumProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpBlockChecksumProto) = ProtoBuf.protohash(v)
isequal(v1::OpBlockChecksumProto, v2::OpBlockChecksumProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpBlockChecksumProto, v2::OpBlockChecksumProto) = ProtoBuf.protoeq(v1, v2)

type ShortCircuitShmIdProto
    hi::Int64
    lo::Int64
    ShortCircuitShmIdProto() = (o=new(); fillunset(o); o)
end #type ShortCircuitShmIdProto
const __req_ShortCircuitShmIdProto = Symbol[:hi,:lo]
meta(t::Type{ShortCircuitShmIdProto}) = meta(t, __req_ShortCircuitShmIdProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ShortCircuitShmIdProto) = ProtoBuf.protohash(v)
isequal(v1::ShortCircuitShmIdProto, v2::ShortCircuitShmIdProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ShortCircuitShmIdProto, v2::ShortCircuitShmIdProto) = ProtoBuf.protoeq(v1, v2)

type ShortCircuitShmSlotProto
    shmId::ShortCircuitShmIdProto
    slotIdx::Int32
    ShortCircuitShmSlotProto() = (o=new(); fillunset(o); o)
end #type ShortCircuitShmSlotProto
const __req_ShortCircuitShmSlotProto = Symbol[:shmId,:slotIdx]
meta(t::Type{ShortCircuitShmSlotProto}) = meta(t, __req_ShortCircuitShmSlotProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ShortCircuitShmSlotProto) = ProtoBuf.protohash(v)
isequal(v1::ShortCircuitShmSlotProto, v2::ShortCircuitShmSlotProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ShortCircuitShmSlotProto, v2::ShortCircuitShmSlotProto) = ProtoBuf.protoeq(v1, v2)

type OpRequestShortCircuitAccessProto
    header::BaseHeaderProto
    maxVersion::UInt32
    slotId::ShortCircuitShmSlotProto
    supportsReceiptVerification::Bool
    OpRequestShortCircuitAccessProto() = (o=new(); fillunset(o); o)
end #type OpRequestShortCircuitAccessProto
const __req_OpRequestShortCircuitAccessProto = Symbol[:header,:maxVersion]
const __val_OpRequestShortCircuitAccessProto = @compat Dict(:supportsReceiptVerification => false)
meta(t::Type{OpRequestShortCircuitAccessProto}) = meta(t, __req_OpRequestShortCircuitAccessProto, ProtoBuf.DEF_FNUM, __val_OpRequestShortCircuitAccessProto, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpRequestShortCircuitAccessProto) = ProtoBuf.protohash(v)
isequal(v1::OpRequestShortCircuitAccessProto, v2::OpRequestShortCircuitAccessProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpRequestShortCircuitAccessProto, v2::OpRequestShortCircuitAccessProto) = ProtoBuf.protoeq(v1, v2)

type ReleaseShortCircuitAccessRequestProto
    slotId::ShortCircuitShmSlotProto
    traceInfo::DataTransferTraceInfoProto
    ReleaseShortCircuitAccessRequestProto() = (o=new(); fillunset(o); o)
end #type ReleaseShortCircuitAccessRequestProto
const __req_ReleaseShortCircuitAccessRequestProto = Symbol[:slotId]
meta(t::Type{ReleaseShortCircuitAccessRequestProto}) = meta(t, __req_ReleaseShortCircuitAccessRequestProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ReleaseShortCircuitAccessRequestProto) = ProtoBuf.protohash(v)
isequal(v1::ReleaseShortCircuitAccessRequestProto, v2::ReleaseShortCircuitAccessRequestProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ReleaseShortCircuitAccessRequestProto, v2::ReleaseShortCircuitAccessRequestProto) = ProtoBuf.protoeq(v1, v2)

type ReleaseShortCircuitAccessResponseProto
    status::Int32
    error::AbstractString
    ReleaseShortCircuitAccessResponseProto() = (o=new(); fillunset(o); o)
end #type ReleaseShortCircuitAccessResponseProto
const __req_ReleaseShortCircuitAccessResponseProto = Symbol[:status]
meta(t::Type{ReleaseShortCircuitAccessResponseProto}) = meta(t, __req_ReleaseShortCircuitAccessResponseProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ReleaseShortCircuitAccessResponseProto) = ProtoBuf.protohash(v)
isequal(v1::ReleaseShortCircuitAccessResponseProto, v2::ReleaseShortCircuitAccessResponseProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ReleaseShortCircuitAccessResponseProto, v2::ReleaseShortCircuitAccessResponseProto) = ProtoBuf.protoeq(v1, v2)

type ShortCircuitShmRequestProto
    clientName::AbstractString
    traceInfo::DataTransferTraceInfoProto
    ShortCircuitShmRequestProto() = (o=new(); fillunset(o); o)
end #type ShortCircuitShmRequestProto
const __req_ShortCircuitShmRequestProto = Symbol[:clientName]
meta(t::Type{ShortCircuitShmRequestProto}) = meta(t, __req_ShortCircuitShmRequestProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ShortCircuitShmRequestProto) = ProtoBuf.protohash(v)
isequal(v1::ShortCircuitShmRequestProto, v2::ShortCircuitShmRequestProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ShortCircuitShmRequestProto, v2::ShortCircuitShmRequestProto) = ProtoBuf.protoeq(v1, v2)

type ShortCircuitShmResponseProto
    status::Int32
    error::AbstractString
    id::ShortCircuitShmIdProto
    ShortCircuitShmResponseProto() = (o=new(); fillunset(o); o)
end #type ShortCircuitShmResponseProto
const __req_ShortCircuitShmResponseProto = Symbol[:status]
meta(t::Type{ShortCircuitShmResponseProto}) = meta(t, __req_ShortCircuitShmResponseProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ShortCircuitShmResponseProto) = ProtoBuf.protohash(v)
isequal(v1::ShortCircuitShmResponseProto, v2::ShortCircuitShmResponseProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ShortCircuitShmResponseProto, v2::ShortCircuitShmResponseProto) = ProtoBuf.protoeq(v1, v2)

type PacketHeaderProto
    offsetInBlock::Int64
    seqno::Int64
    lastPacketInBlock::Bool
    dataLen::Int32
    syncBlock::Bool
    PacketHeaderProto() = (o=new(); fillunset(o); o)
end #type PacketHeaderProto
const __req_PacketHeaderProto = Symbol[:offsetInBlock,:seqno,:lastPacketInBlock,:dataLen]
const __val_PacketHeaderProto = @compat Dict(:syncBlock => false)
const __wtype_PacketHeaderProto = @compat Dict(:offsetInBlock => :sfixed64, :seqno => :sfixed64, :dataLen => :sfixed32)
meta(t::Type{PacketHeaderProto}) = meta(t, __req_PacketHeaderProto, ProtoBuf.DEF_FNUM, __val_PacketHeaderProto, true, ProtoBuf.DEF_PACK, __wtype_PacketHeaderProto)
hash(v::PacketHeaderProto) = ProtoBuf.protohash(v)
isequal(v1::PacketHeaderProto, v2::PacketHeaderProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::PacketHeaderProto, v2::PacketHeaderProto) = ProtoBuf.protoeq(v1, v2)

type PipelineAckProto
    seqno::Int64
    reply::Array{Int32,1}
    downstreamAckTimeNanos::UInt64
    flag::Array{UInt32,1}
    PipelineAckProto() = (o=new(); fillunset(o); o)
end #type PipelineAckProto
const __req_PipelineAckProto = Symbol[:seqno]
const __val_PipelineAckProto = @compat Dict(:downstreamAckTimeNanos => 0)
const __pack_PipelineAckProto = Symbol[:flag]
const __wtype_PipelineAckProto = @compat Dict(:seqno => :sint64)
meta(t::Type{PipelineAckProto}) = meta(t, __req_PipelineAckProto, ProtoBuf.DEF_FNUM, __val_PipelineAckProto, true, __pack_PipelineAckProto, __wtype_PipelineAckProto)
hash(v::PipelineAckProto) = ProtoBuf.protohash(v)
isequal(v1::PipelineAckProto, v2::PipelineAckProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::PipelineAckProto, v2::PipelineAckProto) = ProtoBuf.protoeq(v1, v2)

type ReadOpChecksumInfoProto
    checksum::ChecksumProto
    chunkOffset::UInt64
    ReadOpChecksumInfoProto() = (o=new(); fillunset(o); o)
end #type ReadOpChecksumInfoProto
const __req_ReadOpChecksumInfoProto = Symbol[:checksum,:chunkOffset]
meta(t::Type{ReadOpChecksumInfoProto}) = meta(t, __req_ReadOpChecksumInfoProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ReadOpChecksumInfoProto) = ProtoBuf.protohash(v)
isequal(v1::ReadOpChecksumInfoProto, v2::ReadOpChecksumInfoProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ReadOpChecksumInfoProto, v2::ReadOpChecksumInfoProto) = ProtoBuf.protoeq(v1, v2)

type ClientReadStatusProto
    status::Int32
    ClientReadStatusProto() = (o=new(); fillunset(o); o)
end #type ClientReadStatusProto
const __req_ClientReadStatusProto = Symbol[:status]
meta(t::Type{ClientReadStatusProto}) = meta(t, __req_ClientReadStatusProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::ClientReadStatusProto) = ProtoBuf.protohash(v)
isequal(v1::ClientReadStatusProto, v2::ClientReadStatusProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::ClientReadStatusProto, v2::ClientReadStatusProto) = ProtoBuf.protoeq(v1, v2)

type DNTransferAckProto
    status::Int32
    DNTransferAckProto() = (o=new(); fillunset(o); o)
end #type DNTransferAckProto
const __req_DNTransferAckProto = Symbol[:status]
meta(t::Type{DNTransferAckProto}) = meta(t, __req_DNTransferAckProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::DNTransferAckProto) = ProtoBuf.protohash(v)
isequal(v1::DNTransferAckProto, v2::DNTransferAckProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::DNTransferAckProto, v2::DNTransferAckProto) = ProtoBuf.protoeq(v1, v2)

type OpBlockChecksumResponseProto
    bytesPerCrc::UInt32
    crcPerBlock::UInt64
    md5::Array{UInt8,1}
    crcType::Int32
    OpBlockChecksumResponseProto() = (o=new(); fillunset(o); o)
end #type OpBlockChecksumResponseProto
const __req_OpBlockChecksumResponseProto = Symbol[:bytesPerCrc,:crcPerBlock,:md5]
meta(t::Type{OpBlockChecksumResponseProto}) = meta(t, __req_OpBlockChecksumResponseProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::OpBlockChecksumResponseProto) = ProtoBuf.protohash(v)
isequal(v1::OpBlockChecksumResponseProto, v2::OpBlockChecksumResponseProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::OpBlockChecksumResponseProto, v2::OpBlockChecksumResponseProto) = ProtoBuf.protoeq(v1, v2)

type BlockOpResponseProto
    status::Int32
    firstBadLink::AbstractString
    checksumResponse::OpBlockChecksumResponseProto
    readOpChecksumInfo::ReadOpChecksumInfoProto
    message::AbstractString
    shortCircuitAccessVersion::UInt32
    BlockOpResponseProto() = (o=new(); fillunset(o); o)
end #type BlockOpResponseProto
const __req_BlockOpResponseProto = Symbol[:status]
meta(t::Type{BlockOpResponseProto}) = meta(t, __req_BlockOpResponseProto, ProtoBuf.DEF_FNUM, ProtoBuf.DEF_VAL, true, ProtoBuf.DEF_PACK, ProtoBuf.DEF_WTYPES)
hash(v::BlockOpResponseProto) = ProtoBuf.protohash(v)
isequal(v1::BlockOpResponseProto, v2::BlockOpResponseProto) = ProtoBuf.protoisequal(v1, v2)
==(v1::BlockOpResponseProto, v2::BlockOpResponseProto) = ProtoBuf.protoeq(v1, v2)

export Status, ShortCircuitFdResponse, DataTransferEncryptorMessageProto_DataTransferEncryptorStatus, DataTransferEncryptorMessageProto, BaseHeaderProto, DataTransferTraceInfoProto, ClientOperationHeaderProto, CachingStrategyProto, OpReadBlockProto, ChecksumProto, OpWriteBlockProto_BlockConstructionStage, OpWriteBlockProto, OpTransferBlockProto, OpReplaceBlockProto, OpCopyBlockProto, OpBlockChecksumProto, ShortCircuitShmIdProto, ShortCircuitShmSlotProto, OpRequestShortCircuitAccessProto, ReleaseShortCircuitAccessRequestProto, ReleaseShortCircuitAccessResponseProto, ShortCircuitShmRequestProto, ShortCircuitShmResponseProto, PacketHeaderProto, PipelineAckProto, ReadOpChecksumInfoProto, BlockOpResponseProto, ClientReadStatusProto, DNTransferAckProto, OpBlockChecksumResponseProto


text:
#!/usr/bin/env julia
# del2z <delta.z@aliyun.com>

using Flux

using Flux.Tracker

W = param(2) # 2.0 (tracked)
b = param(3) # 3.0 (tracked)

f(x) = W * x + b

grads = Tracker.gradient(() -> f(4), params(W, b))

grads[W] # 4.0
grads[b] # 1.0


text:
mutable struct WasmModule <: AbstractWasmModule
    wasm_module_ptr::Ptr{wasm_module_t}

    WasmModule(module_ptr::Ptr{wasm_module_t}) =
        finalizer(wasm_module_delete, new(module_ptr))
end
function WasmModule(store::WasmStore, wasm_byte_vec::WasmByteVec)
    wasm_module_ptr = wasm_module_new(store, wasm_byte_vec)
    wasm_module_ptr == C_NULL && error("Failed to create wasm module")

    WasmModule(wasm_module_ptr)
end

Base.unsafe_convert(::Type{Ptr{wasm_module_t}}, wasm_module::WasmModule) =
    wasm_module.wasm_module_ptr

mutable struct WasmMemory
    wasm_memory_ptr::Ptr{wasm_memory_t}
end
function WasmMemory(store::WasmStore, limits::Pair{UInt32,UInt32})
    limits = wasm_limits_t(limits...)
    memory_type = GC.@preserve limits wasm_memorytype_new(pointer_from_objref(limits))
    @assert memory_type != C_NULL "Failed to create memory type"

    wasm_memory_ptr = wasm_memory_new(store, memory_type)
    @assert wasm_memory_ptr != C_NULL "Failed to create memory"

    WasmMemory(wasm_memory_ptr)
end

map_to_extern(mem::WasmMemory) = wasm_memory_as_extern(mem)
Base.show(io::IO, wasm_memory::WasmMemory) = print(io, "WasmMemory()")
Base.unsafe_convert(::Type{Ptr{wasm_memory_t}}, wasm_memory::WasmMemory) = wasm_memory.wasm_memory_ptr

function WasmFunc(store::WasmStore, func::Function, return_type, input_types)
    params_vec =
        WasmPtrVec(collect(Ptr{wasm_valtype_t}, map(julia_type_to_valtype, input_types)))
    results_vec =
        return_type == Nothing ? WasmPtrVec(wasm_valtype_t) :
        WasmPtrVec([julia_type_to_valtype(return_type)])

    func_type = wasm_functype_new(params_vec, results_vec)
    @assert func_type != C_NULL "Failed to create functype"

    function jl_side_host(
        args::Ptr{wasm_val_vec_t},
        results::Ptr{wasm_val_vec_t},
    )::Ptr{wasm_trap_t}
        # TODO: support passing the arguments
        res = func()
        wasm_res = Ref(convert(wasm_val_t, res))
        data_ptr = unsafe_load(results).data
        wasm_val_copy(data_ptr, Base.pointer_from_objref(wasm_res))

        C_NULL
    end

    # Create a pointer to jl_side_host(args, results)
    func_ptr = Base.@cfunction(
        $jl_side_host,
        Ptr{wasm_trap_t},
        (Ptr{wasm_val_vec_t}, Ptr{wasm_val_vec_t})
    )

    host_func = wasm_func_new(store, func_type, func_ptr)
    wasm_functype_delete(func_type)

    # Keep a reference to func_ptr in the store, so that it not garbage collected
    add_extern_func!(store, func_ptr)

    host_func
end

struct WasmFuncRef
    wasm_func_ptr::Ptr{wasm_func_t}
end

Base.unsafe_convert(::Type{Ptr{wasm_func_t}}, wasm_func::WasmFuncRef) =
    wasm_func.wasm_func_ptr

function (wasm_func::WasmFuncRef)(args...)
    params_arity = wasm_func_param_arity(wasm_func)
    result_arity = wasm_func_result_arity(wasm_func)

    provided_params = length(args)
    if params_arity != provided_params
        error("Wrong number of argument to function, expected $params_arity, got $provided_params",)
    end

    converted_args = collect(wasm_val_t, map(arg -> convert(wasm_val_t, arg), args))
    params_vec = WasmVec(converted_args)

    default_val = wasm_val_t(tuple(zeros(UInt8, 16)...))
    results_vec = WasmVec([default_val for _ = 1:result_arity])

    wasm_func_call(wasm_func, params_vec, results_vec)

    collect(results_vec)
end

mutable struct WasmExtern
    wasm_extern_ptr::Ptr{wasm_extern_t}
end

Base.unsafe_convert(::Type{Ptr{wasm_extern_t}}, wasm_extern::WasmExtern) =
    wasm_extern.wasm_extern_ptr
function Base.show(io::IO, wasm_extern::WasmExtern)
    kind = wasm_extern_kind(wasm_extern) |> wasm_externkind_enum
    print(io, "WasmExtern($kind)")
end

function (wasm_extern::WasmExtern)(args...)
    extern_as_func = wasm_extern_as_func(wasm_extern)
    @assert extern_as_func != C_NULL "Can not use extern $wasm_extern as a function"

    WasmFuncRef(extern_as_func)(args...)
end

function name_vec_ptr_to_str(name_vec_ptr::Ptr{wasm_name_t})
    @assert name_vec_ptr != C_NULL "Failed to convert wasm_name"
    name_vec = Base.unsafe_load(name_vec_ptr)
    name = unsafe_string(name_vec.data, name_vec.size)
    wasm_name_delete(name_vec_ptr)

    name
end

function imports(wasm_module::WasmModule)
    wasm_imports_vec = WasmPtrVec(wasm_importtype_t)
    wasm_module_imports(wasm_module, wasm_imports_vec)
    WasmImports(wasm_module, wasm_imports_vec)
end


text:
@testset "rand" begin
    ϵ = Ellipsoid(PDMat([2 0; 0 2]), [0, 0])

    Random.seed!(8128)
    U = rand(ϵ, 1000000)
    Random.seed!()

    @test size(U) == (2, 1000000)
    @test maximum(U; dims=2) ≈ [1.0, 1.0] atol = 1e-3
    @test minimum(U; dims=2) ≈ [-1.0, -1.0] atol = 1e-3
    @test mean(U; dims=2) ≈ [0.0; 0.0] atol = 1e-3

    ϵ = Ellipsoid(PDMat([2 0; 0 2]), [0.5, 0.5])

    Random.seed!(8128)
    U = rand(ϵ, 1000000)
    Random.seed!()

    @test mean(U; dims=2) ≈ [0.5, 0.5] atol = 1e-2
end


text:
###################################################
## network/tools/virtualdata.jl
## generate "virtual" network data
###################################################

"""
    `noisyVirtualData`
    generates random rides in a network
    parameters:
    - `t` is the timings object representing timings expectation
    - `density` is the density of rides in the Graph (1.0 are all possible rides)
    - `frequency` is the average number of datapoints from one orig=>dest (>1)
    - optional parameter: `timeStd` is the standard deviation for a "one minute" ride (in seconds)

    rides must be longer than a 30s to be taken into account

    each timing is an exp variable, the number of rides is Geometric
"""
function noisyVirtualData(t::NetworkTimings, density::Float64=0.2, frequency::Float64=2.;
                          timeStd::Float64 = 20.)
    g = t.network.graph
    trips = NetworkTrip[]
    # subset rides
    tt = getPathTimes(t)
    if frequency <= 1.
        error("frequency must be >1")
    end
    geo = Geometric(1/frequency)
    for orig in 1:nv(g), dest in 1:nv(g)
        if rand() <= density && tt[orig,dest] >= 30.
            count = rand(geo) + 1.
            time  = max(20.,rand(Normal(tt[orig,dest],timeStd*sqrt(tt[orig,dest]/(count*60.)))))
            push!(trips, NetworkTrip((orig,orig,0.),(dest,dest,0.),time,count,false))
        end
    end
    return NetworkData(t.network,trips, maxSpeedTimes(t.network))
end

"""
    `simpleVirtualData`
    generates random rides in a network, one per selected OD, with a given expected
    multiplicative log error.
    parameters:
    - `t` is the timings object representing timings expectation
    - N is the number of OD that we will generate a ride on.
    - logVariance is the multiplicative variance of the data from the truth.

    rides must be longer than a 30s to be taken into account

"""
function simpleVirtualData(t::NetworkTimings, N::Int, logStd::Float64)
    nNodes = nv(t.network.graph)
    allRides = vec([(o,d) for o in 1:nNodes, d in 1:(nNodes -1)])
    shuffle!(allRides)
    selectedRides = allRides[1:N]
    for (i, (o,d)) in enumerate(selectedRides)
        if o == d
            selectedRides[i] = (o, nNodes)
        end
    end

    tt = getPathTimes(t)

    trips = NetworkTrip[]
    # subset rides

    for (o,d) in selectedRides
        realLogTime = log(tt[o,d])
        noisyLogTime = rand(Normal(realLogTime, logStd))
        push!(trips, NetworkTrip((o,o,0.),(d,d,0.),exp(noisyLogTime),1.,false))
    end
    return NetworkData(t.network,trips, maxSpeedTimes(t.network))
end

"""
    `perfectVirtualData`
    generates rides in a network
    parameters:
    - `t` is the timings object representing timings
    Rides are only created if travel time exceeds minTripTime
"""
function perfectVirtualData(t::NetworkTimings, minTripTime::Float64=30.)
    g = t.network.graph
    trips = NetworkTrip[]
    tt = getPathTimes(t)
    for orig in 1:nv(g), dest in 1:nv(g)
        if tt[orig, dest] > minTripTime
            push!(trips, NetworkTrip((orig, orig, 0.), (dest, dest, 0.), tt[orig,dest], 1., false))
        end
    end
    return NetworkData(t.network, trips, maxSpeedTimes(t.network))
end


text:
using Test
using RenderingGeometry

componentwise_ops = (:+, :-)
scalar_ops = (:*, :/)
unary_ops = (:-, abs)

function test_unary_operators(type, name)
    @testset "Unary operators for $name" begin
        @testset "with operator: $op" for op ∈ unary_ops
            @eval begin
                dataset = generate_set($type)
                operated_dataset = [$op(pair) for pair ∈ dataset]
                for (pair₁, pair₂) ∈ zip(dataset, operated_dataset)
                    for (index₁, index₂) ∈ zip(pair₁, pair₂)
                        @test $op(index₁) == index₂
                    end
                end
            end
        end
        increment_progress()
    end
end

function test_scalar_ops_with_single_type(type, op)
    if type<:IntegerPair
        test_scalar_ops_with_types(type, Int, op)
    elseif type<:FloatPair
        test_scalar_ops_with_types(type, Float32, op)
    else
        test_scalar_ops_with_types(type, Float64, op)
    end
end

function test_scalar_ops_with_types(type₁, type₂, op)
    @eval begin
        dataset = generate_set($type₁)
        dataset_type₂ = Array{$type₂, 1}([1, 25, 76, 20000])
        pair_dataset = [(pair, scalar) for pair ∈ dataset, scalar ∈ dataset_type₂]
        operated_dataset = [$op(pair, scalar) for (pair, scalar) ∈ pair_dataset]
        if $op == :/
            @test_throws AssertionError $op($type₁(2, 3), type₂(0))
        end
        for ((pair₁, scalar), pair₂) ∈ zip(pair_dataset, operated_dataset)
            for (index₁, index₂) ∈ zip(pair₁, pair₂)
                if $op == :*
                    @test index₂ ≈ $op(scalar, index₁)
                end
                @test $op(index₁, scalar) ≈ index₂
            end
        end
    end
end

function test_scalar_operators(type, name)
    @testset "Scalar operators for $name" begin
        @testset "with operator (same types): $op" for op ∈ scalar_ops
            test_scalar_ops_with_single_type(type, op)

        end

        @testset "with operator(different types): $op" for op ∈ scalar_ops
            for other_type in (Int, Float32, Float64)
                test_scalar_ops_with_types(type, other_type, op)

            end
        end
        increment_progress()
    end
end

test_componentwise_ops_with_single_type(type, op) = test_componentwise_ops_with_types(type, type, op)

function test_componentwise_ops_with_types(type₁, type₂, op)
    @eval begin
        dataset = generate_set($type₁)
        if $type₁<:$type₂
            pair_dataset = [(pair₁, pair₂) for pair₁ ∈ dataset, pair₂ ∈ dataset]
        else
            dataset_type₂ = generate_set($type₂)
            pair_dataset = [(pair₁, pair₂) for pair₁ ∈ dataset, pair₂ ∈ dataset_type₂]
        end
        operated_dataset = [$op(pair₁, pair₂) for (pair₁, pair₂) ∈ pair_dataset]
        for ((pair₁, pair₂), pair₃) ∈ zip(pair_dataset, operated_dataset)
            for (index₁, index₂, index₃) ∈ zip(pair₁, pair₂, pair₃)
                if $type₁<:$type₂
                    @test $op(index₁, index₂) == index₃
                else
                    @test typeof(index₃) == promote_type(typeof(index₁), typeof(index₂))
                    @test isapprox($op(index₁, index₂), index₃; atol=1)
                end
            end
        end
    end
end

function test_componentwise_operators(type, name)
    @testset "Componentwise operators for $name" begin
        @testset "with operator (same types): $op" for op ∈ componentwise_ops
            test_componentwise_ops_with_single_type(type, op)
        end

        @testset "with operator(different types): $op" for op ∈ componentwise_ops
            for other_type ∈ (type<:VectorPair ? pair_vector_types : pair_point_types)
                test_componentwise_ops_with_types(type, other_type, op)
            end
        end

        @testset "with equality operator:" begin
            @eval begin
                zero_pair = $type()
                @test zero_pair == $type(0, 0)
                with_values = $type(0, 4)
                @test with_values == $type(0, 4)
                @test zero_pair != with_values
            end
        end
        increment_progress()
    end
end

function test_all_pair_operators()
    @testset "Pair operator tests" begin
        log_test_batch_begin("Pair operator")
        @testset "Pair operator tests for $name" for (type, name) ∈ pair_type_iterable
            test_unary_operators(type, name)
            test_componentwise_operators(type, name)
            test_scalar_operators(type, name)
        end
        log_test_batch_end()
    end
end