text:
# Issue #14

using JuliaInterpreter: JuliaInterpreter, pc_expr, evaluate_call!, finish_and_return!, @lookup, enter_call_expr
# Execute a frame using Julia's regular compiled-code dispatch for any :call expressions
runframe(frame::JuliaStackFrame, pc=frame.pc[]) = Some{Any}(finish_and_return!(Compiled(), frame, pc))

stack = @make_stack map(x->2x, 1:10)
state = dummy_state(stack)
execute_command(state, state.stack[1], Val{:finish}(), "finish")
@test isempty(state.stack)
@test state.overall_result == 2 .* [1:10...]

# Issue #12
function complicated_keyword_stuff(args...; kw...)
    args[1] == args[1]
    (args..., kw...)
end
stack = @make_stack complicated_keyword_stuff(1)
state = dummy_state(stack)
execute_command(state, state.stack[1], Val{:n}(), "n")
execute_command(state, state.stack[1], Val{:finish}(), "finish")
@test isempty(state.stack)

@test runframe(JuliaInterpreter.enter_call(complicated_keyword_stuff, 1, 2)) ==
      runframe(@make_stack(complicated_keyword_stuff(1, 2))[1])
@test runframe(JuliaInterpreter.enter_call(complicated_keyword_stuff, 1, 2; x=7, y=33)) ==
      runframe(@make_stack(complicated_keyword_stuff(1, 2; x=7, y=33))[1])

# Issue #22
f22() = string(:(a+b))
@test step_through(enter_call_expr(:($f22()))) == "a + b"
f22() = string(QuoteNode(:a))
@test step_through(enter_call_expr(:($f22()))) == ":a"


text:
using JuMP, JuDGE, Test

include("solvers/setup_gurobi.jl")
#include("solvers/setup_cplex.jl")
#include("solvers/setup_coin.jl")
#include("solvers/setup_glpk.jl")

function newsvendor(;depth=1,cost=5.0,price=8.0,demands=[50.0,60.0,70.0],CVaR=RiskNeutral(),visualise=false)
   mytree = narytree(depth,length(demands))

   demand=Dict{AbstractTree,Float64}()
   nodes=collect(mytree,order=:breadth)
   for i in eachindex(nodes)
      if i==1
         demand[nodes[i]]=0
      else
         demand[nodes[i]]=demands[(i-2)%length(demands)+1]
      end
   end

   function sub_problems(node)
      model = Model(JuDGE_SP_Solver)
      @expansion(model, 0<=papers_ordered<=1000, lag=1, duration=1)
      @capitalcosts(model, papers_ordered*cost)
      @variable(model, sales>=0, Int)
      @constraint(model, maxsale1, sales <= papers_ordered)
      @constraint(model, maxsale2, sales <= demand[node])
      @objective(model, Min, -price*sales)
      return model
   end

   judy = JuDGEModel(mytree, ConditionallyUniformProbabilities, sub_problems, JuDGE_MP_Solver, risk=CVaR, check=false)
   JuDGE.solve(judy)

   println("Objective: "*string(objective_value(judy.master_problem)))

   println("Re-solved Objective: " * string(resolve_subproblems(judy)))

   if visualise
      solution=JuDGE.solution_to_dictionary(judy)
      solution[:demand]=demand
      JuDGE.visualize_tree(mytree,solution)
   end

   deteq = DetEqModel(mytree, ConditionallyUniformProbabilities, sub_problems, JuDGE_DE_Solver, risk=CVaR, check=false)
   JuDGE.solve(deteq)
   println("Deterministic Equivalent Objective: " * string(objective_value(deteq.problem)))

   JuDGE.print_expansions(deteq)
   JuDGE.write_solution_to_file(deteq,joinpath(@__DIR__,"deteq_solution.csv"))
   objective_value(judy.master_problem)
end

@test newsvendor(cost=5.0,price=8.0,demands=[10,80,100]) ≈ -53.333 atol = 1e-3
@test newsvendor(cost=5.0,price=16.0,demands=[10,80,100]) ≈ -513.333 atol = 1e-3
@test newsvendor(cost=5.0,price=8.0,demands=[10,80,100],CVaR=Risk(0.5,0.5)) ≈ -30.0 atol = 1e-3
@test newsvendor(cost=5.0,price=16.0,demands=[10,80,100],CVaR=Risk(0.5,0.5)) ≈ -320.0 atol = 1e-3
@test newsvendor(depth=2,cost=5.0,price=8.0,demands=[10,20,30],CVaR=Risk(0.15,0.5)) ≈ -61.011 atol = 1e-3
@test newsvendor(depth=3,cost=5.0,price=8.0,demands=[10,20,30],CVaR=Risk(0.15,0.05),visualise=true) ≈ -90.526 atol = 1e-3

ext: jl


text:
# Autogenerated wrapper script for libudfread_jll for x86_64-apple-darwin
export libudfread

JLLWrappers.@generate_wrapper_header("libudfread")
JLLWrappers.@declare_library_product(libudfread, "@rpath/libudfread.0.dylib")
function __init__()
    JLLWrappers.@generate_init_header()
    JLLWrappers.@init_library_product(
        libudfread,
        "lib/libudfread.0.dylib",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
"""
Error codes defined by DTC Initium.
"""
const errorcodes = Dict(-3 =>"Illegal Command ID",
                        -5 =>"Parameter Is Missing",
                        -6 =>"Value Too Low",
                        -7 =>"Value Too High",
                        -10 =>"Upper Val < Low Val",
                        -11 =>"Bad Name For Parameter",
                        -12=>"Need Integer Number",
                        -13=>"Need Float Point",
                        -14=>"Illegal CRS Number",
                        -18=>"Bad Sensor Port Number",
                        -19=>"Bad Upper Port Number",
                        -20=>"Upper Port Number < Lower Number",
                        -21=>"Bad Scanner Number",
                        -25=>"Bad Upper Scanner Number",
                        -27=>"Bad Logical Range",
                        -32=>"Too Many Parameters",
                        -39=>"Non-Vol. Mem. Error",
                        -53=>"No Module This CRS",
                        -68=>"Port Not Defined in Scan Table",
                        -69=>"Port Not Defined",
                        -75=>"SDU Table Not Defined",
                        -80=>"DATA Not Acquired",
                        -82=>"Data Acquisition Aborted",
                        -230=>"Module Not Ready")

struct DTCInitiumError <: Exception
    "Code of the error"
    val::Int
    "Error message"
    msg::String
end

"""
`DTCInitiumError(val::Integer)`

Creates an exception with an error code and corresponding message.

Not all errors are documented. In this case the message is "Unknown Error".

"""
DTCInitiumError(msg::String) = DTCInitiumError(0, msg)
function DTCInitiumError(val::Integer)
    if haskey(errorcodes, val)
        DTCInitiumError(val, errorcodes[val])
    else
        DTCInitiumError(val, "Unknown Error")
    end
end


text:
using BKMaxflow
using Test

# Julia API
weights, neighbors = create_graph(JuliaImpl{Float64,Int}, 4)

add_edge!(weights, neighbors, 1, 2, 1., 1.)
add_edge!(weights, neighbors, 1, 3, 2., 2.)
add_edge!(weights, neighbors, 2, 3, 3., 4.)
add_edge!(weights, neighbors, 2, 4, 5., 5.)
add_edge!(weights, neighbors, 3, 4, 6., 6.)

w = reshape(weights, 2, :)
flow, label = boykov_kolmogorov(1, 4, neighbors, w)
@test flow == 3
@test label[2] == label[3] == BK_T

# C API
# low-level
err = Ref{bk_error}(C_NULL)

g = bk_create_graph_double(2, 1, err[])

bk_add_node_double(g, 1, err[])
bk_add_node_double(g, 1, err[])

bk_add_tweights_double(g, 0, 1., 5., err[])
bk_add_tweights_double(g, 1, 2., 6., err[])

bk_add_edge_double(g, 0, 1, 3., 4., err[])

@test bk_maxflow_double(g, false, err[]) == 3

@test bk_what_segment_double(g, 0, err[]) == 1  # sink
@test bk_what_segment_double(g, 1, err[]) == 1  # sink

bk_delete_graph_double(g)

# high-level
g = create_graph(CImpl{Cdouble}, 2, 1)

add_node(CImpl{Cdouble}, g, 1)
add_node(CImpl{Cdouble}, g, 1)

add_tweights(CImpl, g, 0, 1., 5.)
add_tweights(CImpl, g, 1, 2., 6.)

add_edge(CImpl, g, 0, 1, 3., 4.)

@test maxflow(CImpl{Cdouble}, g) == 3

@test what_segment(CImpl{Cdouble}, g, 0) == 1  # sink
@test what_segment(CImpl{Cdouble}, g, 1) == 1  # sink

delete_graph(CImpl{Cdouble}, g)

max_stars_repo_name: Gnimuc/BKmaxflow

max_issues_repo_name: Gnimuc/BKmaxflow

max_forks_repo_name: Gnimuc/BKmaxflow


text:
#!/usr/bin/env julia
using ArgParse

s = ArgParseSettings(
        description = "Calculate classical self-organization model."
)

@add_arg_table s begin
    "--Nparticles"
        arg_type = Int
        required = true
    "--mass"
        arg_type = Float64
        required = true
    "--indices"
        required = true
    "--deltas"
        required = true
    "--etas"
        required = true
    "--kappas"
        required = true
    "--U0s"
        required = true
    "--T"
        required = true
    "--seed"
        arg_type = Int
        required = true
    "--o"
        required = true
end

parameters = parse_args(s)

# Output
const odir = parameters["o"]
@assert isdir(odir)

# Integration
const seed = parameters["seed"]
const T = float(eval(parse(parameters["T"])))
@assert T[1]<T[end]

# Particles
const Nparticles = parameters["Nparticles"]
const particlemass = parameters["mass"]

# Modes
const indices = float(eval(parse(parameters["indices"])))
const Nmodes = length(indices)

const deltas = float(eval(parse(parameters["deltas"])))
const etas = float(eval(parse(parameters["etas"])))
const kappas = float(eval(parse(parameters["kappas"])))
const U0s = float(eval(parse(parameters["U0s"])))

@assert length(deltas)==Nmodes
@assert length(etas)==Nmodes
@assert length(kappas)==Nmodes
@assert length(U0s)==Nmodes

using multimode

const particles = multimode.classical.Particles(Nparticles, particlemass)
const modes = multimode.classical.CavityMode[]

for i=1:Nmodes
    push!(modes, multimode.classical.CavityMode(
                    float(indices[i]*pi),
                    float(deltas[i]),
                    float(etas[i])/sqrt(Nparticles),
                    float(kappas[i]),
                    float(U0s[i])))
end

const system = multimode.classical.MultimodeSystem(particles, modes)

keyparameters = Dict(
    "N"=>parameters["Nparticles"],
    # "m"=>parameters["mass"],
    "index"=>parameters["indices"],
    "eta"=>parameters["etas"],
    # "delta"=>parameters["deltas"],
    # "kappa"=>parameters["kappas"],
    # "U0"=>parameters["U0s"],
    # "T"=>T[end],
    "seed"=>parameters["seed"],
    )

function dict2filename(d::Dict; extension=".dat")
    pairs = ["$(key)=$(item)" for (key, item) in d]
    return join(sort(pairs), ";")*extension
end
outname = "timeevolution_"*dict2filename(keyparameters)
outpath = joinpath(odir, outname)

# # Modes
# const index1 = 4
# const k1 = float(index1*pi)
# const delta1 = -10.
# const eta1 = 50/sqrt(Nparticles)
# const kappa1 = 10.0
# const U0_1 = 0.0/sqrt(Nparticles)
# const mode1 = multimode.classical.CavityMode(k1, delta1, eta1, kappa1, U0_1)

# const index2 = 6
# const k2 = float(index2*pi)
# const delta2 = -10.
# #const eta2 = 36.07135/sqrt(Nparticles)
# const eta2 = 34/sqrt(Nparticles)
# const kappa2 = 10.0
# const U0_2 = 0.0/sqrt(Nparticles)
# const mode2 = multimode.classical.CavityMode(k2, delta2, eta2, kappa2, U0_2)

# const system = multimode.classical.MultimodeSystem(particles, [mode1, mode2])
# const targetpath = "data/timeevolution_modes_$(index1)_$(index2).dat"

# Initial state
state0 = multimode.classical.ClassicalState(Nparticles, length(system.modes))
x0 = multimode.classical.x(state0)
srand(seed)
for j=1:Nparticles
    x0[j] = rand()
end

# Time evolution
tout, states = multimode.classical.timeevolution(T, system, state0)

# Write results to file
f = open(outpath, "w")
write(f, "# System: ")
write(f, string(system))
write(f, "\n# t state")
for i=1:length(T)
    write(f, "\n$(T[i]);")
    write(f, string(states[i]))
end
close(f)


text:
module GeneRegulatoryNetwork
export pcregress

using LinearAlgebra, Statistics, Arpack

function pcregress(X)
    n=size(X,2)
    A=1.0 .-Matrix(I,n,n)
    for k in 1:n
        y=X[:,k]
        𝒳=X[:,1:end.≠k]
        _,ϕ=Arpack.eigs(𝒳'𝒳,nev=3,which=:LM)
        s=𝒳*ϕ
        s ./=(norm.(s[:,i] for i=1:size(s,2)).^2)'
        b=sum(y.*s,dims=1)
        𝒷=ϕ*b'
        A[k,A[k,:].==1.0]=𝒷
    end
    return A
  end

end


text:
using Test
using NCDatasets
import NCDatasets: nc_type, check, libnetcdf, nc_inq_user_type, NC_ENUM, ncType, jlType

#=
# example from https://www.unidata.ucar.edu/software/netcdf/workshops/2011/groups-types/EnumCDL.html

ds = NCDataset("enum.nc");
name = "primary_cloud"

ncid = ds.ncid
varid = NCDatasets.nc_inq_varid(ncid,name)

ndims_ = NCDatasets.nc_inq_varndims(ncid,varid)

ndimsp = Ref(Cint(0))
cname = zeros(UInt8,NCDatasets.NC_MAX_NAME+1)
dimids = zeros(Cint,ndims_)
nattsp = Ref(Cint(0))
xtypep = Ref(NCDatasets.nc_type(0))

NCDatasets.check(ccall((:nc_inq_var,NCDatasets.libnetcdf),Cint,(Cint,Cint,Ptr{UInt8},Ptr{NCDatasets.nc_type},Ptr{Cint},Ptr{Cint},Ptr{Cint}),ncid,varid,cname,xtypep,ndimsp,dimids,nattsp))

xtype = xtypep[]

#nc_inq_enum(int ncid, nc_type xtype, char *name, nc_type *base_nc_typep,
#                 size_t *base_sizep, size_t *num_membersp);

#nc_inq_enum(int ncid, nc_type xtype, char *name, nc_type *base_nc_typep,
#                 size_t *base_sizep, size_t *num_membersp);

name,base_nc_type,base_size,num_members = nc_inq_enum(ncid,xtype)

idx = 0
T = NCDatasets.jlType[base_nc_type]

for idx = 0:num_members-1
    member_name,value = nc_inq_enum_member(ncid,xtype,T,idx)
    @show member_name,value
end
=#

# create a file with enum type

fname = tempname()
#fname = "enum4.nc"
#rm(fname)
ds = NCDataset(fname,"c");
ncid = ds.ncid

T = Int8

base_typeid = ncType[T]

type_name = "cloud_t"
typeid = NCDatasets.nc_def_enum(ncid,base_typeid,type_name)

members = [
    "Clear" => 0, "Cumulonimbus" => 1, "Stratus" => 2,
    "Stratocumulus" => 3, "Cumulus" => 4, "Altostratus" => 5, "Nimbostratus" => 6,
    "Altocumulus" => 7, "Missing" => 127]

members_dict = Dict(members)

for (member_name,member_value) in members
    NCDatasets.nc_insert_enum(ncid,typeid,member_name,member_value,T)
end

# read enum type

name2,size2,base_nc_type2,nfields2,class2 = NCDatasets.nc_inq_user_type(ncid,typeid)

@test name2 == type_name
@test base_nc_type2 == base_typeid
@test nfields2 == length(members)
@test class2 == NC_ENUM

name2,base_nc_type2,base_size2,num_members2 = NCDatasets.nc_inq_enum(ncid,typeid)

@test name2 == type_name
@test base_nc_type2 == T
@test base_size2 == sizeof(T)
@test num_members2 == length(members)

for idx = 0:num_members2-1
    member_name,value = NCDatasets.nc_inq_enum_member(ncid,typeid,idx,T)
    @test members_dict[member_name] == value
    identifier = NCDatasets.nc_inq_enum_ident(ncid,typeid,value)
    @test identifier == member_name
end

close(ds)
#run(`ncdump -h $fname`)

# TODO:
# create variable and attribute with enum type


text:
#NOTE: \perp does not work, reserved as infix operator, use \bigbot instead
"""
    module TwoLevelSystems

Defines two-level dispersion, and utilties for Jacobians.
"""
module TwoLevelSystems

export TwoLevelSystem

using Formatting
using LinearAlgebra
using SparseArrays
using ...VectorFields

import ...INDEX_OFFSET
import ..AbstractDispersion
import ..susceptability

"""
    TwoLevelSystem(; [ωa=0, D0=0, γperp=1e8]) -> tls

The content is mutable, and can be modified, as in `tls.D0=.1`

Note `\\perp` is reserved, use `\\bigbot` instead
"""
mutable struct TwoLevelSystem <: AbstractDispersion
    ωₐ::Float64
    D₀::Float64
    γ⟘::Float64
    h::Vector{Float64}
    hp1⁻¹::Vector{Float64}
    chi::Vector{ComplexF64}
    chis::Matrix{ComplexF64}
    dχdψr::Array{ComplexF64,3}
    dχdψi::Array{ComplexF64,3}
    dχdω::Array{ComplexF64,3}
    dχdϕ::Array{ComplexF64,3}

    TwoLevelSystem(ωₐ, D₀, γ⟘) = new(ωₐ, D₀, γ⟘)
end

function TwoLevelSystem(;ωₐ=0, D₀=0, γ⟘=1e8, kwargs...)
    ωa = get(kwargs,:ωa,ωₐ)
    omega_a = get(kwargs,:omega_a,ωa)
    omega = get(kwargs,:omega,omega_a)
    ω = get(kwargs,:ω,omega)

    γ_perp = get(kwargs,:γ_perp,γ⟘)
    g = get(kwargs,:g,γ_perp)
    gamma = get(kwargs,:gamma,g)
    γperp = get(kwargs,:γperp,gamma)
    γp = get(kwargs,:γp,γperp)
    gamma_perp = get(kwargs,:gamma_perp,γp)
    γ = get(kwargs,:γ,gamma_perp)

    D0 = get(kwargs,:D0,D₀)
    D = get(kwargs,:D,D0)
    return TwoLevelSystem(ω, D, γ)
end

"""
    susceptability(::TwoLevelSystem,ω) -> χ::ComplexF64

    susceptability(::TwoLevelSystem, ω, ωs, ψs)

The second computes in-place (stored in TwoLevelSystem).
`ψs` is an ElectricField"
"""
susceptability(tls::TwoLevelSystem,ω) = χ(tls,ω)
susceptability(tls::TwoLevelSystem,args...) = χ!(tls,args...)

Base.conj(tls::TwoLevelSystem) = TwoLevelSystem(tls.ωa,tls.D0,-tls.γp)

function Base.getproperty(tls::TwoLevelSystem,sym::Symbol)
    if Base.sym_in(sym,(:γ,:γ_perp,:gamma_perp,:γp,:γperp,:gamma,:g))
        return getfield(tls,:γ⟘)
    elseif Base.sym_in(sym,(:ω,:omega_a,:ωa,:omega))
        return getfield(tls,:ωₐ)
    elseif Base.sym_in(sym,(:D,:D0))
        return getfield(tls,:D₀)
    else
        return getfield(tls,sym)
    end
end

function Base.propertynames(::TwoLevelSystem, private=false)
    if private
        return fieldnames(TwoLevelSystem)
    else
        return (:γ⟘,:ωₐ,:D₀)
    end
end

function Base.setproperty!(tls::TwoLevelSystem, sym::Symbol, val::Real)
    if Base.sym_in(sym,(:γ,:γ_perp,:gamma_perp,:γp,:γperp,:gamma,:g))
        return setfield!(tls,:γ⟘,float(val))
    elseif Base.sym_in(sym,(:ω,:omega_a,:ωa,:omega))
        return setfield!(tls,:ωₐ,float(val))
    elseif Base.sym_in(sym,(:D,:D0))
        return setfield!(tls,:D₀,float(val))
    else
        return setfield!(tls,sym,float(val))
    end
end

################################################################################
# Pretty Printing

import ...PRINTED_COLOR_NUMBER
import ...PRINTED_COLOR_DARK

function Base.show(io::IO, tls::TwoLevelSystem)
    printstyled(io, "Two Level System",color=PRINTED_COLOR_DARK)
    print(io," (ωₐ=")
    printstyled(io, fmt("3.2f",tls.ωₐ),color=PRINTED_COLOR_NUMBER)
    print(io,", D₀=")
    printstyled(io, fmt("2.3f",tls.D₀),color=PRINTED_COLOR_NUMBER)
    print(io,", γ⟘=")
    printstyled(io, fmt("1.1f",tls.γ⟘),color=PRINTED_COLOR_NUMBER)
    print(io,")")
end

################################ AUXILLIARIES ##################################

γ(tls::TwoLevelSystem,ω) = tls.γ⟘*one(ω)/(ω-complex(tls.ωₐ,-tls.γ⟘)*one(ω))

Γ(tls::TwoLevelSystem,ω) = abs2(γ(tls,ω))

H!(tls::TwoLevelSystem) = nothing

@inline function H!(tls::TwoLevelSystem, ωs::Vector, ψs::ScalarField)
    N, M = size(ψs)
    isdefined(tls,:h) ? nothing : tls.h = zeros(Float64,N)
    isdefined(tls,:hp1⁻¹) ? nothing : tls.hp1⁻¹ = similar(tls.h)
    isdefined(tls,:chi) ? nothing : tls.chi = Vector{ComplexF64}(undef,N)
    isdefined(tls,:chis) ? nothing : tls.chis = Matrix{ComplexF64}(undef,N,M)
    isdefined(tls,:dχdψr) ? nothing : tls.dχdψr = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdψi) ? nothing : tls.dχdψi = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdω) ? nothing : tls.dχdω = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdϕ) ? nothing : tls.dχdϕ = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.h)==(N,1) ? nothing : tls.h = Vector{Float64}(undef,N)
    size(tls.hp1⁻¹)==(N,1) ? nothing : tls.hp1⁻¹ = Vector{Float64}(undef,N)
    size(tls.chi)==(N,1) ? nothing : tls.chi = Vector{ComplexF64}(undef,N)
    size(tls.chis)==(N,M) ? nothing : tls.chis = Matrix{ComplexF64}(undef,N,M)
    size(tls.dχdψr)==(N,M) ? nothing : tls.dχdψr = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdψi)==(N,M) ? nothing : tls.dχdψi = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdω)==(N,M) ? nothing : tls.dχdω = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdϕ)==(N,M) ? nothing : tls.dχdϕ = Array{ComplexF64,3}(undef,N,M,M)
    @inbounds @simd for i ∈ 1:N tls.h[i] = 0 end
    @inbounds for μ ∈ eachindex(ωs)
        G = Γ(tls,ωs[μ])
        @inbounds for i ∈ 1:N tls.h[i] += G*abs2(ψs[i,μ]) end
    end
    @fastmath @inbounds @simd for i ∈ 1:N tls.hp1⁻¹[i] = 1/(1+tls.h[i]) end
    return nothing
end

@inline function H!(tls::TwoLevelSystem, ωs::Vector, ψs::ElectricField)
    N,M = size(ψs)
    isdefined(tls,:h) ? nothing : tls.h = zeros(Float64,N)
    isdefined(tls,:hp1⁻¹) ? nothing : tls.hp1⁻¹ = similar(tls.h)
    isdefined(tls,:chi) ? nothing : tls.chi = Vector{ComplexF64}(undef,N)
    isdefined(tls,:chis) ? nothing : tls.chis = Matrix{ComplexF64}(undef,N,M)
    isdefined(tls,:dχdψr) ? nothing : tls.dχdψr = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdψi) ? nothing : tls.dχdψi = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdω) ? nothing : tls.dχdω = Array{ComplexF64,3}(undef,N,M,M)
    isdefined(tls,:dχdϕ) ? nothing : tls.dχdϕ = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.h)==(N,1) ? nothing : tls.h = Vector{Float64}(undef,N)
    size(tls.hp1⁻¹)==(N,1) ? nothing : tls.hp1⁻¹ = Vector{Float64}(undef,N)
    size(tls.chi)==(N,1) ? nothing : tls.chi = Vector{ComplexF64}(undef,N)
    size(tls.chis)==(N,M) ? nothing : tls.chis = Matrix{ComplexF64}(undef,N,M)
    size(tls.dχdψr)==(N,M) ? nothing : tls.dχdψr = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdψi)==(N,M) ? nothing : tls.dχdψi = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdω)==(N,M) ? nothing : tls.dχdω = Array{ComplexF64,3}(undef,N,M,M)
    size(tls.dχdϕ)==(N,M) ? nothing : tls.dχdϕ = Array{ComplexF64,3}(undef,N,M,M)
    ψx,ψy,ψz = ψs.x,ψs.y,ψs.z
    @inbounds @simd for i ∈ 1:N tls.h[i] = 0 end
    @inbounds for μ ∈ eachindex(ωs)
        G = Γ(tls,ωs[μ])
        @inbounds for i ∈ 1:N
            j = mod1(i,N÷3)
            tls.h[i] += G*(abs2(ψx[j,μ])+abs2(ψy[j,μ])+abs2(ψz[j,μ]))
        end
    end
    @fastmath @inbounds @simd for i ∈ 1:N tls.hp1⁻¹[i] = 1/(1+tls.h[i]) end
    return nothing
end

χ(tls::TwoLevelSystem,ω) = tls.D₀*γ(tls,ω)

@inline function χ!(tls::TwoLevelSystem, ω, ωs::Vector, ψs::ScalarField)
    index = size(ψs,1)÷2+INDEX_OFFSET
    if index ≥ size(ψs,1)
        @warn "INDEX_OFFSET=$INDEX_OFFSET inconsistent with number of sites $(size(ψs,1))"
        index = 1
    end
    H!(tls,ωs,ψs)
    Dg = tls.D₀*γ(tls,ω)
    gp² = tls.γp^2
    @fastmath @inbounds @simd for i ∈ eachindex(tls.chi) tls.chi[i] = Dg*tls.hp1⁻¹[i] end
    @inbounds for μ ∈ eachindex(ωs)
        Dg = tls.D₀*γ(tls,ωs[μ])
        @fastmath @inbounds @simd for i ∈ eachindex(tls.chi) tls.chis[i,μ] = Dg*tls.hp1⁻¹[i] end
    end
    @inbounds for ν ∈ eachindex(ωs) # derivative wrt ν
        Gν = Γ(tls,ωs[ν])
        ων = ωs[ν]
        @inbounds for μ ∈ eachindex(ωs) # field μ
            Dgμ = tls.D₀*γ(tls,ωs[μ])
            Dgμ² = tls.D₀*γ(tls,ωs[μ])^2
            Dgμ2Gν = Dgμ*2Gν
            Dgμ2Gν² = Dgμ2Gν*Gν
            @inbounds @simd ivdep for i ∈ eachindex(tls.chi)
                k = i#mod1(i,length(tls.chi))
                hp1⁻² = tls.hp1⁻¹[i]^2
                tls.dχdψr[i,μ,ν], tls.dχdψi[i,μ,ν] = -Dgμ2Gν*hp1⁻².*reim(ψs[i,ν])
                tls.dχdω[i,μ,ν] = -(μ==ν)*Dgμ²*tls.hp1⁻¹[i]/tls.γp +
                    Dgμ2Gν²*hp1⁻²*(ων-tls.ωa)*abs2(ψs[k,ν])/gp²
                tls.dχdϕ[i,μ,ν] = -Dgμ2Gν*hp1⁻²*abs2(ψs[k,ν])/real(ψs[index,ν])
            end
        end
    end
    return nothing
end
@inline function χ!(tls::TwoLevelSystem,ω,ωs::Vector,ψs::ElectricField)
    index = size(ψs,1)÷2+INDEX_OFFSET
    H!(tls,ωs,ψs)
    Dg = tls.D₀*γ(tls,ω)
    gp² = tls.γp^2
    @fastmath @inbounds @simd for i ∈ eachindex(tls.chi) tls.chi[i] = Dg*tls.hp1⁻¹[i] end
    @inbounds for μ ∈ eachindex(ωs)
        Dg = tls.D₀*γ(tls,ωs[μ])
        @fastmath @inbounds @simd for i ∈ eachindex(tls.chi) tls.chis[i,μ] = Dg*tls.hp1⁻¹[i] end
    end
    ψx, ψy, ψz = ψs.x, ψs.y, ψs.z
    @inbounds for ν ∈ eachindex(ωs) # derivative wrt ν
        Gν = Γ(tls,ωs[ν])
        ων = ωs[ν]
        @inbounds for μ ∈ eachindex(ωs) # field μ
            Dgμ = tls.D₀*γ(tls,ωs[μ])
            Dgμ² = tls.D₀*γ(tls,ωs[μ])^2
            Dgμ2Gν = Dgμ*2Gν
            Dgμ2Gν² = Dgμ2Gν*Gν
            @inbounds @simd ivdep for i ∈ eachindex(tls.chi)
                k = mod1(i,length(tls.chi)÷3)
                hp1⁻² = tls.hp1⁻¹[i]^2
                tls.dχdψr[i,μ,ν], tls.dχdψi[i,μ,ν] = -Dgμ2Gν*hp1⁻².*reim(ψs[i,ν])
                tls.dχdω[i,μ,ν] = -(μ==ν)*Dgμ²*tls.hp1⁻¹[i]/tls.γp +
                    Dgμ2Gν²*hp1⁻²*(ων-tls.ωa)*(abs2(ψx[k,ν])+abs2(ψy[k,ν])+abs2(ψz[k,ν]))/gp²
                tls.dχdϕ[i,μ,ν] = -Dgμ2Gν*hp1⁻²*(abs2(ψx[k,ν])+abs2(ψy[k,ν])+abs2(ψz[k,ν]))/real(ψs[index,ν])
            end
        end
    end
    return nothing
end

end # module


text:
X = [1 1; 2 2; 1 -1]
y = [1; 1; 2]

model = Model(X, y, :multinomial)
@test predict(model, X) == y
@test values(model, X) == [3.0 6.0 -2.0]'


text:
#===============================================================================
# Hamiltonian Monte Carlo for the Poisson change-point problem with a fixed
# number of change points, k, and a step-like rate function.
#
# In this implementation, step locations and heights are transformed to
# unconstrained spaces, but the locations are not done so in an ordered
# manner -- their ordering must be checked after each leapfrog step.
#
# Because the locations and heights (and their derivatives) operate on different
# scales, the HMC move is split into two parts.
#
# Usage:
# To be honest, this implementation didn't perform as well as I initially
# thought it would -- Green's Gibbs-based implementation converges more quickly
# as far as I can tell. I believe the reason for this (which I only realised
# quite far along) is that the energy/likelihood function is riddled with
# discontinuities -- one for each datum -- and is almost step like in
# appearance. This implies that the gradient -- which is what provides HMC with
# its ability to avoid random walks -- doesn't provide information about the
# overall behaviour of the function, and is of relatively little use. It is for
# this reason that I've set the default number of leapfrog steps to one for the
# location move. The actual operational issues are a bit more subtle: the
# gradient usually informs the momentum variables, which compensate for the
# expected change in energy; in this case, these two changes disagree with each
# other across every discontinuity, and thus meaningful changes to the momentum
# often harm the performance of the simulation.
#
# In any case, the algorithm is still correct (as far as I can tell), and does
# work. A nice way to visualise its output is as follows (e.g., when k = 2):
#
#     using CSV,DataFrames,StatPlots
#     df = CSV.read("samples.txt", header=["s0","s1","s2","s3","h0","h1","h2"])
#     cs = (:s1,:s2) # or c = (:h0,:h1,:h2).
#     ps = vcat([density(hdf[c]) for c in cs], [scatter(hdf[c]) for c in cs])
#     plot(ps...)
===============================================================================#

module HCP

export hcp, readvec

import Base: show, string

include("structs.jl") # must appear
include("ht.jl")      # in this
include("loc.jl")     # order.

#= Main =======================================================================#

"""
    hcp(filename; <keyword arguments>)

Performs HMC for the Poisson-process change-point problem.

# Arguments
  - `data::Vector{Float64}`: an array of non-negative event times.
  - `k::Int=3`: the number of change points.
  - `R::Float64=40907.0`: an upper limit on the event times present in the data.
  - `alpha::Float64=1.0`, `beta::Float64=200.0`: the rates of the Beta prior on
        step heights.
  - `ll::Float64=.006`, `lh::Int=.1`: leapfrog step sizes for location and
        height moves respectively.
  - `Ll::Float64=1`, `Lh::Int=100`: leapfrog step counts for location and
        height moves respectively.
  - `samples::Int=10_000`, `burnin::Int=samples/10`: the number of samples to
        draw. Unless otherwise specified, a warm-up or 'burn-in' stage will be
        run prior to the sampling.
  - `frequency::Int=1`: the frequency at which samples are written to file,
        e.g., a frequency of 2 implies that every second sample is discarded.
"""
function hcp(data::Vector{Float64};
             k::Int=1, R::Float64=40907.,
             alpha::Float64=1., beta::Float64=200.,
             ll::Float64=.006, lh::Float64=.1,
             Ll::Int=1, Lh::Int=100,
             samples::Int=10_000, burnin::Int=div(samples,10),
             frequency::Int=1)

    Y = sort(data)
    C = Constants(k,R,alpha,beta,ll,lh,Ll,Lh)
    S = State(Y,k,R)

    X = Auxiliary(k)
    loc_init(Y,S,X)
    loc_unconstrained(C,X)
    ht_init(Y,S,X)
    ht_unconstrained(X)

    function hmc(name::AbstractString, I::Int)
        (I <= 0) && return

        U = Counts()
        open(name*".txt", "w") do f
            for i = 1:I
                loc_move(Y,C,S,X,U)
                ht_move(Y,C,S,X,U)
                (i % frequency == 0) && write(f, string(S)*"\n")
            end
        end
        println(name*":")
        println("  locs: $(U.laccepts)/$(U.lprops) " *
            @sprintf("(%.2f%%)", 100*U.laccepts/U.lprops))
        println("  hts:  $(U.haccepts)/$(U.hprops) " *
            @sprintf("(%.2f%%)", 100*U.haccepts/U.hprops))
    end

    hmc("burnin", burnin)
    hmc("samples", samples)

end

#= I/O ========================================================================#

"""
    readvec(path)

Returns a `Vector{Float64}` of the numbers contained in the given file.

# Arguments
  - `path::Union{AbstractString,IO}`: the name or `IO` instance of the file
        containing the change-point data. These should be non-negative numbers
        formatted in a single row, column, or a matrix with no missing elements.
        Columns should be comma-separated.
"""
function readvec(path::Union{AbstractString,IO})
    vec(readdlm(path, ',', Float64))
end

end # module.

max_stars_repo_name: kevdur/rjcp

max_issues_repo_name: kevdur/rjcp

max_forks_repo_name: kevdur/rjcp


text:
# --- Transducible processes

"""
    reducingfunction(xf, step; simd)

Apply transducer `xf` to the reducing function `step` to create a new
reducing function.

!!! compat "Transducers.jl 0.3"

    New in version 0.3.

!!! warning

    Be careful using `reducingfunction` with stateful transducers like
    [`Scan`](@ref) with mutable `init` (e.g., `Scan(push!, [])`).  See
    more in Examples below.

# Arguments
- `xf::Transducer`: A transducer.
- `step`: A callable which accepts 1 and 2 arguments.  If it only
  accepts 2 arguments, wrap it with [`Completing`](@ref) to "add"
  1-argument form (i.e., [`complete`](@ref) protocol).

# Keyword Arguments
- `simd`: `false`, `true`, or `:ivdep`. See [`maybe_usesimd`](@ref).

# Examples
```jldoctest
julia> using Transducers

julia> rf = reducingfunction(Map(x -> x + 1), *);

julia> rf(10, 2) === 10 * (2 + 1)
true
```

## Warning: Be careful when using `reducingfunction` with stateful transducers

Stateful `Transducer`s themselves in Transducers.jl are not inherently
broken with `reducingfunction`.  However, it can produce incorrect
results when combined with mutable states:

```jldoctest reducingfunction; setup = :(using Transducers)
julia> scan_state = [];

julia> rf_bad = reducingfunction(Scan(push!, scan_state) |> Cat(), string);

julia> transduce(rf_bad, "", 1:3)
"112123"
```

The first run works.  However, observe that the vector `scan_state` is
not empty anymore:

```jldoctest reducingfunction
julia> scan_state
3-element Array{Any,1}:
 1
 2
 3
```

Thus, the second run produces an incorrect result:

```jldoctest reducingfunction
julia> transduce(rf_bad, "", 1:3)
"123112312123123"
```

One way to solve this issue is to use [`CopyInit`](@ref) or [`OnInit`](@ref).

```jldoctest reducingfunction
julia> scan_state = CopyInit([])
CopyInit(Any[])

julia> rf_good = reducingfunction(Scan(push!, scan_state) |> Cat(), string);

julia> transduce(rf_good, "", 1:3)
"112123"

julia> scan_state
CopyInit(Any[])

julia> transduce(rf_good, "", 1:3)
"112123"
```
"""
@inline reducingfunction(xf::Transducer, step; simd::SIMDFlag = Val(false)) =
    maybe_usesimd(Reduction(xf, step), simd)

# Use `_asmonoid` automatically only when `init` is not specified:
@inline _reducingfunction(xf, step; init = DefaultInit, simd::SIMDFlag = Val(false), _...) =
    maybe_usesimd(Reduction(xf, init === DefaultInit ? _asmonoid(step) : step), simd)

"""
    __foldl__(rf, init, reducible::T)

Left fold a `reducible` with reducing function `rf` and initial value
`init`.  This is primary an API for overloading when the reducible
"container" or "context" (e.g., I/O stream) of type `T` can provide a
better reduction mechanism than the default iterator-based one.

For a simple iterable type `MyType`, a valid implementation is:

```julia
function __foldl__(rf, val, itr::MyType)
    for x in itr
        val = @next(rf, val, x)
    end
    return complete(rf, val)
end
```

although in this case default ` __foldl__` can handle `MyType` and
thus there is no need for defining it.  In general, defining
`__foldl__` is useful only when there is a better way to go over items
in `reducible` than `Base.iterate`.

See also: [`@next`](@ref).
"""
__foldl__

const FOLDL_RECURSION_LIMIT = Val(10)
# const FOLDL_RECURSION_LIMIT = nothing
_dec(::Nothing) = nothing
_dec(::Val{n}) where n = Val(n - 1)

function __foldl__(rf, init::T, coll) where {T}
    ret = iterate(coll)
    ret === nothing && return complete(rf, init)
    x, state = ret
    val = @next(rf, init, x)

    # Doing "manual Union splitting" (?).  This somehow helps the
    # compiler to generate faster code even though the code inside the
    # `if` branches are identical.
    # * https://github.com/JuliaFolds/Transducers.jl/pull/188
    # * https://github.com/JuliaLang/julia/pull/34293#discussion_r363550608
    if val isa T
        return _foldl_iter(rf, val, coll, state, FOLDL_RECURSION_LIMIT)
    else
        return _foldl_iter(rf, val, coll, state, FOLDL_RECURSION_LIMIT)
    end
end

@inline function _foldl_iter(rf, val::T, iter, state, counter) where T
    while true
        ret = iterate(iter, state)
        ret === nothing && break
        x, state = ret
        y = @next(rf, val, x)
        counter === Val(0) || y isa T ||
            return _foldl_iter(rf, y, iter, state, _dec(counter))
        val = y
    end
    return complete(rf, val)
end

@inline __foldl__(rf, init, coll::Tuple) =
    complete(rf, @return_if_reduced foldlargs(rf, init, coll...))

# TODO: use IndexStyle
@inline function __foldl__(rf, init, arr::Union{AbstractArray, Broadcasted})
    isempty(arr) && return complete(rf, init)
    idxs = eachindex(arr)
    val = @next(rf, init, @inbounds arr[idxs[firstindex(idxs)]])
    @simd_if rf for k in firstindex(idxs) + 1:lastindex(idxs)
        i = @inbounds idxs[k]
        val = @next(rf, val, @inbounds arr[i])
    end
    return complete(rf, val)
end

@inline _getvalues(i) = ()
@inline _getvalues(i, a, rest...) = ((@inbounds a[i]), _getvalues(i, rest...)...)

# TODO: merge this with array implementation
@static if VERSION >= v"1.1-"
    @inline function __foldl__(
            rf, init,
            zs::Iterators.Zip{<:Tuple{Vararg{AbstractArray}}})
        isempty(zs) && return complete(rf, init)
        idxs = eachindex(zs.is...)
        val = @next(rf, init, _getvalues(firstindex(idxs), zs.is...))
        @simd_if rf for i in firstindex(idxs) + 1:lastindex(idxs)
            val = @next(rf, val, _getvalues(i, zs.is...))
        end
        return complete(rf, val)
    end
end

## Convert zip-of-products (ZoP) to product-of-zips (PoZ).

# Arguments for `product` and alike:
unproduct(x::Iterators.ProductIterator) = x.iterators
unproduct(x::CartesianIndices) = x.indices
unproduct(x::AbstractArray) = axes(x)

# After ZoP-to-PoZ transformation, we need to re-shape the value in
# the form that would be fed to the reducing function if it were
# folding `ZoP`:
@inline function _make_zop_getvalues(iterators)
    # Avoid including `iterators` themselves in the closure if
    # possible:
    iterinfo = map(iterators) do itr
        if itr isa CartesianIndices
            Val(CartesianIndices)
        elseif itr isa AbstractArray
            itr
        else
            nothing
        end
    end
    return function (xs)
        map(_unzip((iterinfo, _unzip(xs)))) do (it, x)
            if it === Val(CartesianIndices)
                return CartesianIndex(x)
            elseif it isa AbstractArray
                return @inbounds it[x...]
            else
                return x
            end
        end
    end
end

@static if VERSION >= v"1.1-"
    @inline __foldl__(
        rf,
        init,
        zs::Iterators.Zip{<:Tuple{
            Vararg{Union{AbstractArray,Iterators.ProductIterator}},
        }},
    ) = __foldl__(
        Reduction(Map(_make_zop_getvalues(zs.is)), rf),
        init,
        Iterators.product(map(Base.splat(zip), _unzip(map(unproduct, zs.is)))...),
    )
end

@inline function __foldl__(
        rf, init,
        prod::Iterators.ProductIterator{<:Tuple{Any,Any,Vararg{Any}}})
    val = _foldl_product(rf, init, (), prod.iterators...)
    val isa Reduced && return val
    return complete(rf, val)
end

@noinline _foldl_product(rf, val, ::Any) = error("Unreachable")
@inline _foldl_product(rf, val, ::Tuple) = next(rf, val, ())

@inline function _foldl_product(rf, val, outer, iterators...)
    inner_iterators, outer_iterator = _poptail(iterators)
    for input in outer_iterator
        val_ = _foldl_product(rf, val, (input, outer...), inner_iterators...)
        val_ isa Reduced && return val_
        val = val_
    end
    return val
end

@inline function _foldl_product(rf, val, outer, iterator)
    # TODO: Handle the case inner iterators are tuples.  In such case,
    # inner-most non-tuple iterators should use @simd_if.
    @simd_if rf for input in iterator
        val = @next(rf, val, (input, outer...))
    end
    return val
end

function __simple_foldl__(rf, val, itr)
    for x in itr
        val = @next(rf, val, x)
    end
    return complete(rf, val)
end

"""
    simple_transduce(xform, step, init, coll)

Simplified version of [`transduce`](@ref).  For simple transducers Julia
may be able to emit a good code.  This function exists only for
performance tuning.
"""
function simple_transduce(xform, f, init, coll)
    rf = Reduction(xform, f)
    return __simple_foldl__(rf, start(rf, init), coll)
end

"""
    foldl_nocomplete(rf, init, coll)

Call [`__foldl__`](@ref) without calling [`complete`](@ref).
"""
@inline foldl_nocomplete(rf, init, coll) = __foldl__(skipcomplete(rf), init, coll)

"""
    foldl(step, xf::Transducer, reducible; init, simd) :: T
    foldl(step, ed::Eduction; init, simd) :: T
    transduce(xf, step, init, reducible; simd) :: Union{T, Reduced{T}}

Compose transducer `xf` with reducing step function `step` and reduce
`itr` using it.

!!! note
    `transduce` differs from `foldl` as `Reduced{T}` is returned if
    the transducer `xf` or `step` aborts the reduction and `step` is
    _not_ automatically wrapped by [`Completing`](@ref).

This API is modeled after $(_cljref("transduce")).

For parallel versions, see [`reduce`](@ref) and [`dreduce`](@ref).

See also: [Empty result handling](@ref).

# Arguments
- `xf::Transducer`: A transducer.
- `step`: A callable which accepts 1 and 2 arguments.  If it only
  accepts 2 arguments, wrap it with [`Completing`](@ref) to "add"
  1-argument form (i.e., [`complete`](@ref) protocol).
- `reducible`: A reducible object (array, dictionary, any iterator, etc.).
- `init`: An initial value fed to the first argument to reducing step
  function `step`.  This argument can be omitted for well know binary
  operations like `+` or `*`.  Supported binary operations are listed
  in InitialValues.jl documentation.  When `Init` (not the result of
  `Init`, such as `Init(*)`) is given, it is automatically "instantiated"
  as `Init(step)` (where `step` is appropriately unwrapped if `step` is
  a `Completing`).  See [Empty result handling](@ref) in the manual
  for more information.
- `simd`: If `true` or `:ivdep`, enable SIMD using `Base.@simd`.  If
  `:ivdep`, use `@simd ivdep for ... end` variant.  Read Julia manual
  of `Base.@simd` to understand when it is appropriate to use this
  option.  For example, `simd = :ivdep` _must not_ be used with stateful
  transducer like [`Scan`](@ref).  This option has no effect if
  `false` (default).

# Examples
```jldoctest
julia> using Transducers

julia> foldl(Filter(isodd), 1:4, init=0.0) do state, input
           @show state, input
           state + input
       end
(state, input) = (0.0, 1)
(state, input) = (1.0, 3)
4.0
```
"""
foldl

"""
    transduce(xf, step, init, reducible) :: Union{T, Reduced{T}}

See [`foldl`](@ref).
"""
transduce

"""
    mapfoldl(xf::Transducer, step, reducible; init, simd)

!!! warning

    `mapfoldl` exists primary for backward compatibility.  It is
    recommended to use `foldl`.

Like [`foldl`](@ref) but `step` is _not_ automatically wrapped by
[`Completing`](@ref).

# Examples
```jldoctest
julia> using Transducers

julia> function step_demo(state, input)
           @show state, input
           state + input
       end;

julia> function step_demo(state)
           println("Finishing with state = ", state)
           state
       end;

julia> mapfoldl(Filter(isodd), step_demo, 1:4, init=0.0)
(state, input) = (0.0, 1)
(state, input) = (1.0, 3)
Finishing with state = 4.0
4.0
```
"""
mapfoldl

function transduce(xform::Transducer, f, init, coll; kwargs...)
    rf = _reducingfunction(xform, f; init = init, kwargs...)
    return transduce(rf, init, coll; kwargs...)
end

_unreduced__foldl__(rf, step, coll) = unreduced(__foldl__(rf, step, coll))

# TODO: should it be an internal?
@inline function transduce(rf1::AbstractReduction, init, coll;
                           simd::SIMDFlag = Val(false))
    # Inlining `transduce` and `__foldl__` were essential for the
    # `restack` below to work.
    rf0, foldable = retransform(rf1, asfoldable(coll))
    rf = maybe_usesimd(rf0, simd)
    state = start(rf, init)
    result = __foldl__(rf, state, foldable)
    if unreduced(result) isa DefaultInitOf
        throw(EmptyResultError(rf0))
        # Should I check if `init` is a `DefaultInit`?
    end
    # At this point, `return result` is the semantically correct thing
    # to do.  What follows are some convoluted instructions to
    # convince the compiler that this function is type-stable (in some
    # cases).  Note that return type would be inference-dependent only
    # if `init` is a `OptInit` type.  In the default case where `init
    # isa DefaultInitOf`, the real code pass is the `throw` above.

    # Unpacking as `ur_result` and re-packing it later somehow helps
    # the compiler to correctly eliminate a possibility in a `Union`.
    ur_result = unreduced(result)
    if ur_result isa InferableInit
        # Using `rf0` instead of `rf` helps the compiler.  Note that
        # this means that we are relying on that enabling SIMD does
        # not change the return type.
        realtype = _nonidtype(Core.Compiler.return_type(
            _unreduced__foldl__,
            typeof((rf0, state, foldable)),
        ))
        if realtype isa Type
            realvalue = convert(realtype, ur_result)
            if result isa Reduced
                return Reduced(realvalue)
            else
                return realvalue
            end
        end
    end
    if result isa Reduced
        return Reduced(ur_result)
    else
        return ur_result
    end
end

Base.mapfoldl(f::F, step::OP, itr::Foldable; kw...) where {F, OP} =
    foldl(step, Map(f), itr; kw...)

function Base.mapfoldl(xform::Transducer, step, itr;
                       simd::SIMDFlag = Val(false),
                       init = DefaultInit)
    unreduced(transduce(xform, step, init, itr; simd=simd))
end

# disambiguation
function Base.mapfoldl(xform::Transducer, step, itr::Foldable;
                       simd::SIMDFlag = Val(false),
                       init = DefaultInit)
    unreduced(transduce(xform, step, init, itr; simd=simd))
end

struct Eduction{F, C} <: Foldable
    rf::F
    coll::C
end

Eduction(xform::Transducer, coll) =
    Eduction(Reduction(xform, Completing(push!!)), coll)

Eduction(xform::Transducer, ed::Eduction) =
    Eduction(Transducer(ed) |> xform, ed.coll)

Transducer(ed::Eduction) = Transducer(ed.rf)

transduce(xform::Transducer, f, init, ed::Eduction) =
    transduce(Transducer(ed) |> xform, f, init, ed.coll)

Base.IteratorSize(::Type{<:Eduction}) = Base.SizeUnknown()

function Base.iterate(ts::Eduction, state = nothing)
    if state === nothing
        cret = iterate(ts.coll)
        cret === nothing && return nothing
        input, cstate = cret

        result = start(ts.rf, Union{}[])

        cdone = false
        rdone = false
        @goto step
        # Assuming the initial `result` can never be `Reduced`.
    else
        cstate, cdone, result, rdone = state
    end
    if !cdone && !rdone
        while isempty(unwrap_all(result))
            cret = iterate(ts.coll, cstate)
            if cret === nothing
                cdone = true
                result = complete(ts.rf, unreduced(result))
                # Stateful transducers may have flushed something.
                # Let's not `return nothing` yet.
                break
            end
            input, cstate = cret

            @label step
            result = next(ts.rf, result, input)
            if isreduced(result)
                rdone = true
                result = unreduced(result)
                break
            end
        end
    end
    buffer = unwrap_all(result)
    isempty(buffer) && return nothing
    y = popfirst!(buffer)
    next_state = (cstate, cdone, result, rdone)
    return (y, next_state)
end

"""
    eduction(xf::Transducer, coll)

Create a iterable and reducible object.

* [Iterable](https://docs.julialang.org/en/v1/manual/interfaces/#man-interface-iteration-1).
* Reducible; i.e., it can be handled by [`transduce`](@ref) efficiently.

This API is modeled after $(_cljref("eduction")).

# Examples
```jldoctest
julia> using Transducers

julia> for x in eduction(Filter(isodd) |> Take(3), 1:1000)
           @show x
       end
x = 1
x = 3
x = 5
```
"""
eduction(xform, coll) = Eduction(xform, coll)
# Note on `simd` keyword argument: `eduction` ATM does not support
# `simd` argument which could be done in principle.  However, how
# `foldl` and `foreach` with `Eduction` treat `simd` argument must be
# tweaked if that happens.
#
# Note on API:
# Exporting `Eduction` could also work.  But `Base` has, e.g.,
# `skipmissing` so maybe this is better for more uniform API.

"""
    setinput(ed::Eduction, coll)

Set input collection of eduction `ed` to `coll`.

!!! compat "Transducers.jl 0.3"

    Previously, `setinput` combined with `eduction` was a recommended
    way to use transducers in a type stable manner.  As of v0.3, all
    the `foldl`-like functions and `eduction` are type stable for many
    cases.  This workaround is no more necessary.

# Examples
```jldoctest
julia> using Transducers

julia> ed = eduction(Map(x -> 2x), Float64[]);

julia> xs = ones(2, 3);

julia> foldl(+, setinput(ed, xs))
12.0
```
"""
setinput(ed::Eduction, coll) =
    _setinput(eltype(ed.coll), eltype(coll), ed, coll)

_setinput(::Type{T}, ::Type{T}, ed, coll) where T = @set ed.coll = coll
_setinput(::Type, ::Type, ed, coll) = eduction(Transducer(ed), coll)

"""
    append!(xf::Transducer, dest, src) -> dest

This API is modeled after $(_cljref("into")).

!!! warning
    The performance of `append!(dest, src::Eduction)` is poor.
    Use `append!!` instead if two-argument form is preferred.

# Examples
```jldoctest
julia> using Transducers

julia> append!(Drop(2), [-1, -2], 1:5)
5-element Array{Int64,1}:
 -1
 -2
  3
  4
  5
```
"""
Base.append!(xf::Transducer, to, from) =
    unreduced(transduce(xf, Completing(push!), to, from))

"""
    BangBang.append!!(xf::Transducer, dest, src) -> dest′
    BangBang.append!!(dest, src::Eduction) -> dest′

Mutate-or-widen version of [`append!`](@ref).

!!! compat "Transducers.jl 0.4.4"

    New in version 0.4.4.

!!! compat "Transducers.jl 0.4.37"

    Performance optimization for `append!!(dest, src::Eduction)`
    requires version 0.4.37.

# Examples
```jldoctest
julia> using Transducers, BangBang

julia> append!!(Drop(2) |> Map(x -> x + 0.0), [-1, -2], 1:5)
5-element Array{Float64,1}:
 -1.0
 -2.0
  3.0
  4.0
  5.0
```
"""
BangBang.append!!
BangBang.append!!(xf::Transducer, to, from) =
    unreduced(transduce(xf |> Map(SingletonVector), Completing(append!!), to, from))

function BangBang.__appendto!!__(to, foldable::Foldable)
    xf, from = extract_transducer(foldable)
    return append!!(xf, to, from)
end

"""
    collect(xf::Transducer, itr) :: Vector
    collect(ed::Eduction) :: Vector

Process an iterable `itr` using a transducer `xf` and collect the result
into a `Vector`.

For parallel versions, see [`tcollect`](@ref) and [`dcollect`](@ref).

!!! compat "Transducers.jl 0.4.8"

    `collect` now accepts eductions.

# Examples
```jldoctest
julia> using Transducers

julia> collect(Interpose(missing), 1:3)
5-element Array{Union{Missing, Int64},1}:
 1
  missing
 2
  missing
 3
```
"""
function Base.collect(xf::Transducer, coll)
    result = finish!(append!!(xf, collector(), coll))
    if result isa Vector{Union{}}
        et = @default_finaltype(xf, coll)
        return et[]
    end
    return result
end
# Base.collect(xf, coll) = append!([], xf, coll)

Base.collect(ed::Eduction) = collect(extract_transducer(ed)...)

"""
    copy(xf::Transducer, T, foldable) :: Union{T, Empty{T}}
    copy(xf::Transducer, foldable::T) :: Union{T, Empty{T}}
    copy([T,] eduction::Eduction) :: Union{T, Empty{T}}

Process `foldable` with a transducer `xf` and then create a container of type `T`
filled with the result.  Return
[`BangBang.Empty{T}`](https://juliafolds.github.io/BangBang.jl/dev/#BangBang.NoBang.Empty)
if the transducer does not produce anything.  (This is because there is no
consistent interface to create an empty container given its type and not all
containers support creating an empty container.)

For parallel versions, see [`tcopy`](@ref) and [`dcopy`](@ref).

!!! compat "Transducers.jl 0.4.4"

    New in version 0.4.4.

!!! compat "Transducers.jl 0.4.8"

    `copy` now accepts eductions.

# Examples
```jldoctest
julia> using Transducers
       using BangBang: Empty

julia> copy(Map(x -> x => x^2), Dict, 2:2)
Dict{Int64,Int64} with 1 entry:
  2 => 4

julia> @assert copy(Filter(_ -> false), Set, 1:1) === Empty(Set)

julia> using TypedTables

julia> @assert copy(Map(x -> (a=x, b=x^2)), Table, 1:1) == Table(a=[1], b=[1])

julia> using StructArrays

julia> @assert copy(Map(x -> (a=x, b=x^2)), StructVector, 1:1) == StructVector(a=[1], b=[1])

julia> using DataFrames

julia> @assert copy(
           Map(x -> (A = x.a + 1, B = x.b + 1)),
           DataFrame(a = [1], b = [2]),
       ) == DataFrame(A = [2], B = [3])
```
"""
Base.copy(xf::Transducer, ::Type{T}, foldable) where {T} = append!!(xf, Empty(T), foldable)
Base.copy(xf::Transducer, foldable) = copy(xf, _materializer(foldable), foldable)

function Base.copy(::Type{T}, ed::Eduction) where {T}
    xf, foldable = extract_transducer(ed)
    return copy(xf, T, foldable)
end

function Base.copy(ed::Eduction)
    xf, foldable = extract_transducer(ed)
    return copy(xf, foldable)
end

"""
    map!(xf::Transducer, dest, src; simd)

Feed `src` to transducer `xf`, storing the result in `dest`.
Collections `dest` and `src` must have the same shape.  Transducer
`xf` may contain filtering transducers.  If some entries `src` are
skipped, the corresponding entries in `dest` will be unchanged.
Transducer `xf` must not contain any expansive transducers such as
[`MapCat`](@ref).

See also [`copy!`](@ref).

# Examples
```jldoctest
julia> using Transducers

julia> xs = collect(1:5)
       ys = zero(xs)
       map!(Filter(isodd), ys, xs)
5-element Array{Int64,1}:
 1
 0
 3
 0
 5

julia> ans === ys
true
```
"""
function Base.map!(xf::Transducer, dest::AbstractArray, src::AbstractArray;
                   simd::SIMDFlag = Val(false))
    _map!(_prepare_map(xf, dest, src, simd)...)
    return dest
end

_map!(rf, coll, dest) = transduce(restack(rf), nothing, coll)

# The idea behind `restack` (previously called `darkritual`):
# Deep-copy `AbstractReduction` so that compiler can treat the all
# reducing function tree nodes as local variables (???).  Aslo, it
# tells compiler that `dest` is a local variable so that it won't
# fetch `dest` via `getproperty` in each iteration.  (This is too much
# magic...  My reasoning of how it works could be completely wrong.
# But at least it should not change the semantics of the function.)
# Probably related: https://github.com/JuliaLang/julia/pull/18632

function _prepare_map(xf, dest, src, simd)
    isexpansive(xf) && error("map! only supports non-expanding transducer")
    # TODO: support Dict
    indices = eachindex(dest, src)

    rf = reducingfunction(
        ZipSource(GetIndex{true}(src) |> xf) |> SetIndex{true}(dest),
        (::Vararg) -> nothing,
        simd = simd)

    return rf, indices, dest
end

"""
    copy!(xf::Transducer, dest, src)

Feed `src` to transducer `xf`, storing the result in `dest`.
Collections `dest` and `src` may have the same shape.  Source `src`
must be iterable.  Destination `dest` must implement `empty!` and
`push!`.

See also [`map!`](@ref).

# Examples
```jldoctest
julia> using Transducers

julia> copy!(PartitionBy(x -> x ÷ 3) |> Map(sum), Int[], 1:10)
4-element Array{Int64,1}:
  3
 12
 21
 19
```
"""
Base.copy!(xf::Transducer, dest, src) = append!(xf, empty!(dest), src)

function Base.foldl(step, xform::Transducer, itr;
                    kw...)
    mapfoldl(xform, Completing(step), itr; kw...)
end

@inline function Base.foldl(step, foldable::Foldable; init = DefaultInit, kwargs...)
    xf, coll = extract_transducer(foldable)
    return unreduced(transduce(xf, Completing(step), init, coll; kwargs...))
end

"""
    foreach(eff, xf::Transducer, reducible; simd)
    foreach(eff, ed::Eduction; simd)

Feed the results of `xf` processing items in `reducible` into a unary
function `eff`.  This is useful when the primary computation at the
bottom is the side-effect.  It is also equivalent to `foreach(eff,
eduction(xf, coll))`.  Note that

```julia
foreach(eduction(xf, coll)) do x
    ...
end
```

can be more efficient than

```julia
for x in eduction(xf, coll)
    ...
end
```

as the former does not have to translate the transducer protocol to
the iterator protocol.

`foreach` supports all constructs in the native `for` loop as well as
the enhancements [^julia_issue_22891] to `break` with a value (`break
D(x)` below) and append the `else` clause (`E(x)` below).

[^julia_issue_22891]: See also: [break with value + loop else clauses
    (JuliaLang/julia#22891)](https://github.com/JuliaLang/julia/issues/22891)

This native `for` loop

```julia
ans = for x in xs
    A(x)
    B(x) && break
    C(x) && break D(x)
else
    E(x)
end
```

can be written as

```julia
ans = foreach(Map(identity), xs) do x
    A(x)
    B(x) && return reduced()
    C(x) && return reduced(D(x))
    x  # required for passing `x` to `E(x)` below
end |> ifunreduced() do x
    E(x)
end
```

See: [`mapfoldl`](@ref), [`reduced`](@ref), [`ifunreduced`](@ref).

!!! compat "Transducers.jl 0.3"

    `foreach` is changed to return what the `do` block (`eff`
    function) returns as-is in version 0.3.  This was required for
    supporting "for-else" (`|> ifunreduced`).  Previously, it only
    supported break-with-value and always applied `unreduced` before
    it returns.

# Examples
```jldoctest
julia> using Transducers

julia> foreach(eduction(Filter(isodd), 1:4)) do input
           @show input
       end
input = 1
input = 3
3

julia> foreach(Filter(!ismissing), [1, missing, 2, 3]) do input
           @show input
           if iseven(input)
               return reduced()
           end
       end
input = 1
input = 2
Reduced(nothing)
```

It is often useful to append [`|> unreduced`](@ref unreduced) to
unwrap `Reduced` in the final result (note that `|>` here is the
standard function application, not the transducer composition).

```jldoctest; setup = :(using Transducers)
julia> foreach(Filter(!ismissing), [1, missing, 2, 3]) do input
           reduced("got \$input")
       end |> unreduced
"got 1"
```

Combination of break-with-value and for-else is useful for triggering
action after (e.g.) some kind of membership testing failed:

```jldoctest; setup = :(using Transducers)
julia> has2(xs) = foreach(Filter(!ismissing), xs) do input
           input == 2 && reduced(true)
       end |> ifunreduced() do input
           @show input
           false
       end;

julia> has2([1, missing, 2, 3])
true

julia> has2([1, missing])
input = false
false
```

However, note the output `input = false` in the last example.  This is
because how `&&` works in Julia

```jldoctest
julia> false && "otherwise"
false
```

Thus, pure membership testing functions like `has2` above can be
written in a more concise manner:

```jldoctest; setup = :(using Transducers)
julia> simpler_has2(xs) = foreach(Filter(!ismissing), xs) do input
           input == 2 && reduced(true)
       end |> unreduced;

julia> simpler_has2([1, missing, 2, 3])
true

julia> simpler_has2([1, missing])
false
```
"""
Base.foreach(eff, xform::Transducer, coll; kwargs...) =
    transduce(xform, SideEffect(eff), nothing, coll; kwargs...)
function Base.foreach(eff, reducible::Reducible; kwargs...)
    xf, coll = extract_transducer(reducible)
    return transduce(xf, SideEffect(eff), nothing, coll; kwargs...)
end

"""
    ifunreduced(f, [x])

Equivalent to [`unreduced(x)`](@ref unreduced) if `x` is a
[`Reduced`](@ref); otherwise run `f(x)`.  Return a curried version if
`x` is not provided.

See: [`foreach`](@ref).

# Examples
```jldoctest
julia> using Transducers

julia> 1 |> ifunreduced() do x
           println("called with x = ", x)
       end
called with x = 1

julia> reduced(1) |> ifunreduced() do x
           println("called with x = ", x)
       end
1
```

Notice that nothing is printed in the last example.

# Implementation

```julia
ifunreduced(f) = x -> ifunreduced(f, x)
ifunreduced(f, x::Reduced) = unreduced(x)
ifunreduced(f, x) = f(x)
```
"""
ifunreduced(f) = x -> ifunreduced(f, x)
ifunreduced(f, x::Reduced) = unreduced(x)
ifunreduced(f, x) = f(x)

"""
    Channel(xf::Transducer, itr; kwargs...)
    Channel(ed::Eduction; kwargs...)

Pipe items from an iterable `itr` processed by the transducer `xf`
through a channel.  `Channel(xf, itr)` and `Channel(eduction(xf,
itr))` are equivalent.  Note that `itr` itself can be a `Channel`.

Keyword arguments are passed to `Channel(function; kwargs...)`.

# Examples
```jldoctest
julia> using Transducers

julia> ch1 = Channel(Filter(isodd), 1:5);

julia> ch2 = Channel(Map(x -> 2x - 1), ch1);

julia> ed = eduction(Map(x -> 1:x), ch2);

julia> ch3 = Channel(Cat(), ed);

julia> foreach(PartitionBy(isequal(1)), ch3) do input
           @show input
       end;
input = [1, 1]
input = [2, 3, 4, 5]
input = [1]
input = [2, 3, 4, 5, 6, 7, 8, 9]
```
"""
Base.Channel(xform::Transducer, itr; kwargs...) =
    Channel(; kwargs...) do chan
        foreach(x -> put!(chan, x), xform, itr)
        return
    end

Base.Channel(ed::Eduction; kwargs...) =
    Channel(Transducer(ed), ed.coll; kwargs...)

Base.Channel{T}(xform::Transducer, itr, size=0; kwargs...) where {T} =
    _Channel(T, size; kwargs...) do chan
        foreach(x -> put!(chan, x), xform, itr)
        return
    end

Base.Channel{T}(ed::Eduction, size=0; kwargs...) where {T} =
    Channel{T}(Transducer(ed), ed.coll, size; kwargs...)

"""
    AdHocFoldable(foldl, [collection = nothing])

Provide a different way to fold `collection` without creating a
wrapper type.

# Arguments
- `foldl::Function`: a function that implements [`__foldl__`](@ref).
- `collection`: a collection passed to the last argument of
  `foldl`.

# Examples
```jldoctest
julia> using Transducers
       using Transducers: @next, complete
       using ArgCheck

julia> function uppertriangle(A::AbstractMatrix)
           @argcheck !Base.has_offset_axes(A)
           return AdHocFoldable(A) do rf, acc, A
               for j in 1:size(A, 2), i in 1:min(j, size(A, 1))
                   acc = @next(rf, acc, @inbounds A[i, j])
               end
               return complete(rf, acc)
           end
       end;

julia> A = reshape(1:6, (3, 2))
3×2 reshape(::UnitRange{Int64}, 3, 2) with eltype Int64:
 1  4
 2  5
 3  6

julia> collect(Map(identity), uppertriangle(A))
3-element Array{Int64,1}:
 1
 4
 5

julia> function circularwindows(xs::AbstractVector, h::Integer)
           @argcheck !Base.has_offset_axes(xs)
           @argcheck h >= 0
           @argcheck 2 * h + 1 <= length(xs)
           return AdHocFoldable(xs) do rf, acc, xs
               buffer = similar(xs, 2 * h + 1)
               @inbounds for i in 1:h
                   buffer[1:h - i + 1] .= @view xs[end - h + i:end]
                   buffer[h - i + 2:end] .= @view xs[1:h + i]
                   acc = @next(rf, acc, buffer)
               end
               for i in h + 1:length(xs) - h
                   acc = @next(rf, acc, @inbounds @view xs[i - h:i + h])
               end
               @inbounds for i in 1:h
                   buffer[1:end - i] .= @view xs[end - 2 * h + i:end]
                   buffer[end - i + 1:end] .= @view xs[1:i]
                   acc = @next(rf, acc, buffer)
               end
               return complete(rf, acc)
           end
       end;

julia> collect(Map(collect), circularwindows(1:9, 2))
9-element Array{Array{Int64,1},1}:
 [8, 9, 1, 2, 3]
 [9, 1, 2, 3, 4]
 [1, 2, 3, 4, 5]
 [2, 3, 4, 5, 6]
 [3, 4, 5, 6, 7]
 [4, 5, 6, 7, 8]
 [5, 6, 7, 8, 9]
 [6, 7, 8, 9, 1]
 [7, 8, 9, 1, 2]

julia> expressions(str::AbstractString; kwargs...) =
           AdHocFoldable(str) do rf, val, str
               pos = 1
               while true
                   expr, pos = Meta.parse(str, pos;
                                          raise = false,
                                          depwarn = false,
                                          kwargs...)
                   expr === nothing && break
                   val = @next(rf, val, expr)
               end
               return complete(rf, val)
           end;

julia> collect(Map(identity), expressions(\"\"\"
       x = 1
       y = 2
       \"\"\"))
2-element Array{Expr,1}:
 :(x = 1)
 :(y = 2)

julia> counting = AdHocFoldable() do rf, acc, _
           i = 0
           while true
               i += 1
               acc = @next(rf, acc, i)
           end
       end;

julia> foreach(counting) do i
           @show i;
           i == 3 && return reduced()
       end;
i = 1
i = 2
i = 3
```
"""
struct AdHocFoldable{F, C} <: Foldable
    f::F
    coll::C
end

AdHocFoldable(f) = AdHocFoldable(f, nothing)

@inline __foldl__(rf, init, foldable::AdHocFoldable) =
    foldable.f(rf, init, foldable.coll)


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: connectparticipant
using AWS.Compat
using AWS.UUIDs

"""
    complete_attachment_upload(attachment_ids, client_token, x-_amz-_bearer)
    complete_attachment_upload(attachment_ids, client_token, x-_amz-_bearer, params::Dict{String,<:Any})

Allows you to confirm that the attachment has been uploaded using the pre-signed URL
provided in StartAttachmentUpload API.  The Amazon Connect Participant Service APIs do not
use Signature Version 4 authentication.

# Arguments
- `attachment_ids`: A list of unique identifiers for the attachments.
- `client_token`: A unique, case-sensitive identifier that you provide to ensure the
  idempotency of the request.
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

"""
function complete_attachment_upload(
    AttachmentIds,
    ClientToken,
    X_Amz_Bearer;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/complete-attachment-upload",
        Dict{String,Any}(
            "AttachmentIds" => AttachmentIds,
            "ClientToken" => ClientToken,
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function complete_attachment_upload(
    AttachmentIds,
    ClientToken,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/complete-attachment-upload",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AttachmentIds" => AttachmentIds,
                    "ClientToken" => ClientToken,
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_participant_connection(type, x-_amz-_bearer)
    create_participant_connection(type, x-_amz-_bearer, params::Dict{String,<:Any})

Creates the participant's connection. Note that ParticipantToken is used for invoking this
API instead of ConnectionToken. The participant token is valid for the lifetime of the
participant – until they are part of a contact. The response URL for WEBSOCKET Type has a
connect expiry timeout of 100s. Clients must manually connect to the returned websocket URL
and subscribe to the desired topic.  For chat, you need to publish the following on the
established websocket connection:
{\"topic\":\"aws/subscribe\",\"content\":{\"topics\":[\"aws/chat\"]}}  Upon websocket URL
expiry, as specified in the response ConnectionExpiry parameter, clients need to call this
API again to obtain a new websocket URL and perform the same steps as before.  Message
streaming support: This API can also be used together with the StartContactStreaming API to
create a participant connection for chat contacts that are not using a websocket. For more
information about message streaming, Enable real-time chat message streaming in the Amazon
Connect Administrator Guide.  Feature specifications: For information about feature
specifications, such as the allowed number of open websocket connections per participant,
see Feature specifications in the Amazon Connect Administrator Guide.   The Amazon Connect
Participant Service APIs do not use Signature Version 4 authentication.

# Arguments
- `type`: Type of connection information required.
- `x-_amz-_bearer`: This is a header parameter. The ParticipantToken as obtained from
  StartChatContact API response.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ConnectParticipant"`: Amazon Connect Participant is used to mark the participant as
  connected for message streaming.
"""
function create_participant_connection(
    Type, X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config()
)
    return connectparticipant(
        "POST",
        "/participant/connection",
        Dict{String,Any}(
            "Type" => Type, "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_participant_connection(
    Type,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/connection",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "Type" => Type,
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    disconnect_participant(x-_amz-_bearer)
    disconnect_participant(x-_amz-_bearer, params::Dict{String,<:Any})

Disconnects a participant. Note that ConnectionToken is used for invoking this API instead
of ParticipantToken. The Amazon Connect Participant Service APIs do not use Signature
Version 4 authentication.

# Arguments
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ClientToken"`: A unique, case-sensitive identifier that you provide to ensure the
  idempotency of the request.
"""
function disconnect_participant(
    X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config()
)
    return connectparticipant(
        "POST",
        "/participant/disconnect",
        Dict{String,Any}(
            "ClientToken" => string(uuid4()),
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function disconnect_participant(
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/disconnect",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "ClientToken" => string(uuid4()),
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_attachment(attachment_id, x-_amz-_bearer)
    get_attachment(attachment_id, x-_amz-_bearer, params::Dict{String,<:Any})

Provides a pre-signed URL for download of a completed attachment. This is an asynchronous
API for use with active contacts. The Amazon Connect Participant Service APIs do not use
Signature Version 4 authentication.

# Arguments
- `attachment_id`: A unique identifier for the attachment.
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

"""
function get_attachment(
    AttachmentId, X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config()
)
    return connectparticipant(
        "POST",
        "/participant/attachment",
        Dict{String,Any}(
            "AttachmentId" => AttachmentId,
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_attachment(
    AttachmentId,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/attachment",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AttachmentId" => AttachmentId,
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_transcript(x-_amz-_bearer)
    get_transcript(x-_amz-_bearer, params::Dict{String,<:Any})

Retrieves a transcript of the session, including details about any attachments. Note that
ConnectionToken is used for invoking this API instead of ParticipantToken. The Amazon
Connect Participant Service APIs do not use Signature Version 4 authentication.

# Arguments
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ContactId"`: The contactId from the current contact chain for which transcript is
  needed.
- `"MaxResults"`: The maximum number of results to return in the page. Default: 10.
- `"NextToken"`: The pagination token. Use the value returned previously in the next
  subsequent request to retrieve the next set of results.
- `"ScanDirection"`: The direction from StartPosition from which to retrieve message.
  Default: BACKWARD when no StartPosition is provided, FORWARD with StartPosition.
- `"SortOrder"`: The sort order for the records. Default: DESCENDING.
- `"StartPosition"`: A filtering option for where to start.
"""
function get_transcript(X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config())
    return connectparticipant(
        "POST",
        "/participant/transcript",
        Dict{String,Any}("headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_transcript(
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/transcript",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer)
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    send_event(content_type, x-_amz-_bearer)
    send_event(content_type, x-_amz-_bearer, params::Dict{String,<:Any})

Sends an event. Note that ConnectionToken is used for invoking this API instead of
ParticipantToken. The Amazon Connect Participant Service APIs do not use Signature Version
4 authentication.

# Arguments
- `content_type`: The content type of the request. Supported types are:
  application/vnd.amazonaws.connect.event.typing
  application/vnd.amazonaws.connect.event.connection.acknowledged
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ClientToken"`: A unique, case-sensitive identifier that you provide to ensure the
  idempotency of the request.
- `"Content"`: The content of the event to be sent (for example, message text). This is not
  yet supported.
"""
function send_event(
    ContentType, X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config()
)
    return connectparticipant(
        "POST",
        "/participant/event",
        Dict{String,Any}(
            "ContentType" => ContentType,
            "ClientToken" => string(uuid4()),
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function send_event(
    ContentType,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/event",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "ContentType" => ContentType,
                    "ClientToken" => string(uuid4()),
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    send_message(content, content_type, x-_amz-_bearer)
    send_message(content, content_type, x-_amz-_bearer, params::Dict{String,<:Any})

Sends a message. Note that ConnectionToken is used for invoking this API instead of
ParticipantToken. The Amazon Connect Participant Service APIs do not use Signature Version
4 authentication.

# Arguments
- `content`: The content of the message.
- `content_type`: The type of the content. Supported types are text/plain.
- `x-_amz-_bearer`: The authentication token associated with the connection.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ClientToken"`: A unique, case-sensitive identifier that you provide to ensure the
  idempotency of the request.
"""
function send_message(
    Content, ContentType, X_Amz_Bearer; aws_config::AbstractAWSConfig=global_aws_config()
)
    return connectparticipant(
        "POST",
        "/participant/message",
        Dict{String,Any}(
            "Content" => Content,
            "ContentType" => ContentType,
            "ClientToken" => string(uuid4()),
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function send_message(
    Content,
    ContentType,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/message",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "Content" => Content,
                    "ContentType" => ContentType,
                    "ClientToken" => string(uuid4()),
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    start_attachment_upload(attachment_name, attachment_size_in_bytes, client_token, content_type, x-_amz-_bearer)
    start_attachment_upload(attachment_name, attachment_size_in_bytes, client_token, content_type, x-_amz-_bearer, params::Dict{String,<:Any})

Provides a pre-signed Amazon S3 URL in response for uploading the file directly to S3. The
Amazon Connect Participant Service APIs do not use Signature Version 4 authentication.

# Arguments
- `attachment_name`: A case-sensitive name of the attachment being uploaded.
- `attachment_size_in_bytes`: The size of the attachment in bytes.
- `client_token`: A unique case sensitive identifier to support idempotency of request.
- `content_type`: Describes the MIME file type of the attachment. For a list of supported
  file types, see Feature specifications in the Amazon Connect Administrator Guide.
- `x-_amz-_bearer`: The authentication token associated with the participant's connection.

"""
function start_attachment_upload(
    AttachmentName,
    AttachmentSizeInBytes,
    ClientToken,
    ContentType,
    X_Amz_Bearer;
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/start-attachment-upload",
        Dict{String,Any}(
            "AttachmentName" => AttachmentName,
            "AttachmentSizeInBytes" => AttachmentSizeInBytes,
            "ClientToken" => ClientToken,
            "ContentType" => ContentType,
            "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function start_attachment_upload(
    AttachmentName,
    AttachmentSizeInBytes,
    ClientToken,
    ContentType,
    X_Amz_Bearer,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return connectparticipant(
        "POST",
        "/participant/start-attachment-upload",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "AttachmentName" => AttachmentName,
                    "AttachmentSizeInBytes" => AttachmentSizeInBytes,
                    "ClientToken" => ClientToken,
                    "ContentType" => ContentType,
                    "headers" => Dict{String,Any}("X-Amz-Bearer" => X_Amz_Bearer),
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end


text:
using OrdinaryDiffEq, DiffEqDevTools, DiffEqBase, Base.Test, ODEInterfaceDiffEq

const linear_bigα = parse(BigFloat,"1.01")
f = (t,u) -> (linear_bigα*u)
analytic = (t,u0) -> u0*exp(linear_bigα*t)
prob_ode_bigfloatlinear = ODETestProblem(f,parse(BigFloat,"0.5"),analytic,(0.0,10.0))

f = (t,u,du) -> begin
  for i in 1:length(u)
    du[i] = linear_bigα*u[i]
  end
end
probbig = ODETestProblem(f,map(BigFloat,rand(4,2)).*ones(4,2)/2,analytic,(0.0,10.0))

linear = (t,u) -> (1.01*u)
analytic_linear = (t,u0) -> u0*exp(1.01*t)
probnum = ODETestProblem(linear,1/2,analytic_linear,(0.0,10.0))

probnumbig = prob_ode_bigfloatlinear
#prob    = prob_ode_large2Dlinear

f_2dlinear = (t,u,du) -> begin
  for i in 1:length(u)
    du[i] = 1.01*u[i]
  end
end
analytic_2dlinear = (t,u0) -> u0*exp.(1.01*t)
prob = ODETestProblem(f_2dlinear,rand(4,2),analytic_2dlinear,(0.0,10.0))

dts = 1.//2.^(7:-1:4)
testTol = .2
bools = Vector{Bool}(0)

## DP5()
sim = test_convergence(dts,probnum,DP5())
@test abs(sim.𝒪est[:l2]-5) < testTol
sim = test_convergence(dts,prob,DP5())
@test abs(sim.𝒪est[:l2]-5) < testTol

tabalg = ExplicitRK()
sol1 =solve(probnum,DP5(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(prob,DP5(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(prob,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 3e-10)

sol1 =solve(probnum,DP5(),dt=1/2^6,beta2=0.04)
sol2 =solve(probnum,tabalg,dt=1/2^6,beta2=0.04)

# Should be identical
sol1 =solve(probnum,DP5())
sol2 =solve(probnum,tabalg,beta2=0.04,beta1=0.17)
sol3 =solve(probnum,dopri5())

@test sol1.t ≈ sol2.t ≈ sol3.t

sol1 =solve(prob,DP5(),dt=1/8)
sol2 =solve(prob,tabalg,beta2=0.04,beta1=0.17,dt=1/8)
sol3 =solve(prob,dopri5(),dt=1/8)

@test sol1.t ≈ sol2.t ≈ sol3.t

### BS3()
sim = test_convergence(dts,probnum,BS3())
@test abs(sim.𝒪est[:l2]-3) < testTol
sim = test_convergence(dts,prob,BS3())
@test abs(sim.𝒪est[:l2]-3) < testTol

tabalg = ExplicitRK(tableau=constructBogakiShampine3())
sol1 =solve(probnum,BS3(),dt=1/2^1,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,tabalg,dt=1/2^1,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(prob,BS3(),dt=1/2^1,adaptive=false,save_timeseries=false)
sol2 =solve(prob,tabalg,dt=1/2^1,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,BS3(),dt=1/2^6)

@test length(sol1) == length(sol2)

### BS5()
dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,BS5())
@test abs(sim.𝒪est[:l2]-5) < testTol
sim = test_convergence(dts,probbig,BS5())
@test abs(sim.𝒪est[:l2]-5) < testTol

tabalg = ExplicitRK(tableau=constructBogakiShampine5())
sol1 =solve(probnum,BS5(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(prob,BS5(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(prob,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,BS5(),dt=1/2^6)

@test length(sol1) <= length(sol2) # Dual error estimators is more strict

### Tsit5()

dts = 1.//2.^(7:-1:3)
sim = test_convergence(dts,probnum,Tsit5())
@test abs(sim.𝒪est[:l2]-5) < testTol+.1
sim = test_convergence(dts,prob,Tsit5())
@test abs(sim.𝒪est[:l2]-5) < testTol+.1

tabalg = ExplicitRK(tableau=constructTsitouras5())
sol1 =solve(probnum,Tsit5(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(prob,Tsit5(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(prob,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,Tsit5(),dt=1/2^6)

@test length(sol1) == length(sol2)

### Vern6()

dts = 1.//2.^(8:-1:5)
sim = test_convergence(dts,probnumbig,Vern6())
@test abs(sim.𝒪est[:l2]-6) < testTol
sim = test_convergence(dts,probbig,Vern6())
@test abs(sim.𝒪est[:l2]-6) < testTol

tabalg = ExplicitRK(tableau=constructVernerEfficient6(BigFloat))
sol1 =solve(probnumbig,Vern6(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnumbig,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(probbig,Vern6(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(probbig,tabalg,dt=1/2^6)
sol2 =solve(probbig,Vern6(),dt=1/2^6)

@test length(sol1) == length(sol2)

### Vern7()

dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,Vern7())
@test abs(sim.𝒪est[:l2]-7) < testTol
sim = test_convergence(dts,probbig,Vern7())
@test abs(sim.𝒪est[:l2]-7) < testTol

tabalg = ExplicitRK(tableau=constructVerner7(BigFloat))
sol1 =solve(probnumbig,Vern7(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnumbig,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(probbig,Vern7(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(probbig,tabalg,dt=1/2^6)
sol2 =solve(probbig,Vern7(),dt=1/2^6)

@test length(sol1) == length(sol2)

### TanYam7()

dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,TanYam7())
@test abs(sim.𝒪est[:l2]-7) < testTol
sim = test_convergence(dts,probbig,TanYam7())
@test abs(sim.𝒪est[:l2]-7) < testTol

tabalg = ExplicitRK(tableau=constructTanakaYamashitaEfficient7(Float64))
sol1 =solve(probnum,TanYam7(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

tabalg = ExplicitRK(tableau=constructTanakaYamashitaEfficient7(BigFloat))
sol1 =solve(probbig,TanYam7(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,TanYam7(),dt=1/2^6)

@test length(sol1) == length(sol2)

### Vern8()

dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,Vern8())
@test abs(sim.𝒪est[:l2]-8) < testTol
sim = test_convergence(dts,probbig,Vern8())
@test abs(sim.𝒪est[:l2]-8) < testTol

tabalg = ExplicitRK(tableau=constructVerner8(BigFloat))
sol1 =solve(probnumbig,Vern8(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnumbig,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(probbig,Vern8(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,Vern8(),dt=1/2^6)

@test length(sol1) == length(sol2)

### DP8()

dts = 1.//2.^(3:-1:1)
sim = test_convergence(dts,probnumbig,DP8())
@test abs(sim.𝒪est[:l2]-8) < testTol
sim = test_convergence(dts,probbig,DP8())
@test abs(sim.𝒪est[:l2]-8) < testTol

sol1 =solve(probnum,DP8(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnum,DP8(),dt=1/2^6)

# Should be identical
sol1 =solve(probnum,DP8())
sol2 =solve(probnum,dop853())

@test sol1.t ≈ sol2.t

# Should be identical
sol1 =solve(probbig,DP8(),dt=1/2^6)
sol2 =solve(probbig,dop853(),dt=1/2^6)

@test sol1.t ≈ sol2.t

### TsitPap8()

dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,TsitPap8())
@test abs(sim.𝒪est[:l2]-8) < testTol
sim = test_convergence(dts,probbig,TsitPap8())
@test abs(sim.𝒪est[:l2]-8) < testTol

tabalg = ExplicitRK(tableau=constructTsitourasPapakostas8(BigFloat))
sol1 =solve(probnumbig,TsitPap8(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnumbig,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test sol1.u[end] - sol2.u[end] < 1e-10

sol1 =solve(probbig,TsitPap8(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(sol1.u[end] - sol2.u[end] .< 1e-10)

sol1 =solve(prob,tabalg,dt=1/2^6)
sol2 =solve(prob,TsitPap8(),dt=1/2^6)

@test length(sol1) == length(sol2)

### Vern9()

dts = 1.//2.^(6:-1:3)
sim = test_convergence(dts,probnumbig,Vern9())
@test abs(sim.𝒪est[:l2]-9) < testTol
sim = test_convergence(dts,probbig,Vern9())
@test abs(sim.𝒪est[:l2]-9) < testTol

tabalg = ExplicitRK(tableau=constructVernerEfficient9(BigFloat))
sol1 =solve(probnumbig,Vern9(),dt=1/2^6,adaptive=false,save_timeseries=false)
sol2 =solve(probnumbig,tabalg,dt=1/2^6,adaptive=false,save_timeseries=false)

@test abs(sol1.u[end] - sol2.u[end]) < 1e-15

sol1 =solve(probbig,Vern9(),dt=1/2^3,adaptive=false,save_timeseries=false)
sol2 =solve(probbig,tabalg,dt=1/2^3,adaptive=false,save_timeseries=false)

@test minimum(abs(sol1.u[end] - sol2.u[end]) .< 1e-15)

sol1 =solve(probbig,tabalg,dt=1/2^6)
sol2 =solve(probbig,Vern9(),dt=1/2^6)

@test length(sol1) == length(sol2)


text:
"""
Base abstract class for black-box optimization algorithms.
"""
abstract type Optimizer end

"""
Optimizers derived from `SteppingOptimizer` implement classical iterative optimization scheme
`step!()` → `step!()` → …
"""
abstract type SteppingOptimizer <: Optimizer end
evaluator(so::SteppingOptimizer) = so.evaluator

"""
    step!(opt::SteppingOptimizer)

Do one iteration of the method.
"""
function step! end # FIXME avoid defining 0-arg function

"""
Base abstract class for optimizers that perform
`ask()` → ..eval fitness.. → `tell!()`
sequence at each step.
"""
abstract type AskTellOptimizer <: Optimizer end

"""
    ask(ato::AskTellOptimizer)

Ask for a new candidate solution to be generated, and a list of individuals
it should be ranked with.

The individuals are supplied as an array of tuples
with the individual and its index.

See also `tell!()`
"""
function ask end # FIXME avoid defining 0-arg function

# Our design is inspired by the object-oriented, ask-and-tell "optimizer API
# format" as proposed in:
#
#  Collette, Y., N. Hansen, G. Pujol, D. Salazar Aponte and
#  R. Le Riche (2010). On Object-Oriented Programming of Optimizers -
#  Examples in Scilab. In P. Breitkopf and R. F. Coelho, eds.:
#  Multidisciplinary Design Optimization in Computational Mechanics, Wiley,
#  pp. 527-565.
#  https://www.lri.fr/~hansen/collette2010Chap14.pdf
#
# but since Julia is not OO this is more reflected in certain patterns of how
# to specify and call optimizers. The basic ask-and-tell pattern is:
#
#   while !optimizer.stop
#     x = ask(optimizer)
#     y = f(x)
#     optimizer = tell(optimizer, x, y)
#   end
#
# after which the best solutions can be found by:
#
#   yopt, xopt = best(optimizer)
#
# We have extended this paradigm with the use of an archive that saves
# information on what we have learnt about the search space as well as the
# best solutions found. For most multi-objective optimization problems there
# is no single optimum. Instead there are many pareto optimal solutions.
# An archive collects information about the pareto optimal set or some
# approximation of it. Different archival strategies can be implemented.

"""
    tell!(ato::AskTellOptimizer, rankedCandidates)

Tell the optimizer about the ranking of candidates.
Returns the number of `rankedCandidates` that were inserted into the population,
because of the improved fitness.

See also `ask()`.
"""
function tell! end # FIXME avoid defining 0-arg function

"""
Base class for population-based optimization methods.
"""
abstract type PopulationOptimizer <: AskTellOptimizer end

population(popopt::PopulationOptimizer) = popopt.population
popsize(popopt::PopulationOptimizer) = popsize(population(popopt))

function setup!(o::SteppingOptimizer)
    # Do nothing, override if you need to setup prior to the optimization loop
end

function setup!(o::AskTellOptimizer, evaluator::Evaluator)
    # Do nothing, override if you need to setup prior to the optimization loop
end

function shutdown!(o::Optimizer)
    # Do nothing, override if you need to setup prior to the optimization loop
end

shutdown!(o::SteppingOptimizer) =
  shutdown!(evaluator(o)) # shutdown the evaluator

# The standard name function converts the type of the optimizer to a string
# and strips off trailing "Opt".
function name(o::Optimizer)
    s = string(typeof(o))
    if s[end-2:end] == "Opt"
        return s[1:end-3]
    else
        return s
    end
end

"""
    trace_state(io::IO, optimizer::Optimizer, mode::Symbol)

Trace the current optimization state to a given IO stream.
Called by `OptRunController` `trace_progress()`.

Override it for your optimizer to generate method-specific diagnostic traces.
"""
function trace_state(io::IO, optimizer::Optimizer, mode::Symbol) end


text:
@testset "AWAP" begin
    using SimpleSDMDataSources: rastername, zipurl, zipname, zippath

    raster_file = joinpath(ENV["ECODATASOURCES_PATH"], "AWAP/vprp/vprph09/20010101.grid")
    @test rasterpath(AWAP, VapourPressure{H09}, Date(2001, 1)) == raster_file
    @test rastername(AWAP, VapourPressure{H09}, Date(2001, 1)) == "20010101.grid"

    @test zipurl(AWAP, VapourPressure{H09}, Date(2001, 1)) ==
        "http://www.bom.gov.au/web03/ncc/www/awap/vprp/vprph09/daily/grid/0.05/history/nat/2001010120010101.grid.Z"
    @test zippath(AWAP, VapourPressure{H09}, Date(2001, 1)) ==
        joinpath(ENV["ECODATASOURCES_PATH"], "AWAP/vprp/vprph09/20010101.grid.Z")
    @test zipname(AWAP, VapourPressure{H09}, Date(2001, 1)) == "20010101.grid.Z"

    dates = DateTime(2001, 01, 01), DateTime(2001, 01, 02)
    download_raster(AWAP, VapourPressure{H09}; dates=dates)

    @test isfile(raster_file)
end


text:
using DrWatson
@quickactivate
using ArgParse
using GenerativeAD
using GenerativeAD.Models: anomaly_score, generalized_anomaly_score_gpu
using BSON
using StatsBase: fit!, predict, sample

using Flux
using MLDataPattern

s = ArgParseSettings()
@add_arg_table! s begin
	"max_seed"
		default = 1
		arg_type = Int
		help = "max_seed"
	"category"
		default = "wood"
		arg_type = String
		help = "category"
	"contamination"
		arg_type = Float64
		help = "contamination rate of training data"
		default = 0.0
end
parsed_args = parse_args(ARGS, s)
@unpack category, max_seed, contamination = parsed_args

modelname = "Conv-SkipGANomaly"

function sample_params()
	argnames = (:num_filters, :extra_layers, :lr, :batch_size,
				:iters, :check_every, :patience, :lambda, :init_seed, )
	par_vec = (
			   2 .^ (3:6),
			   [0:1 ...],
			   10f0 .^ (-4:-3),
			   2 .^ (3:5),
			   [10000],
			   [30],
			   [10],
			   [0.9],
			   1:Int(1e8),
			   )
	w = (weights= sample([1,10:10:90...],3),)
	return merge(NamedTuple{argnames}(map(x->sample(x,1)[1], par_vec)), w)
end

"""
	function fit(data, parameters)

parameters => type named tuple with keys
	num_filters   - number of kernels/masks in convolutional layers
	extra_layers  - number of additional conv layers in discriminator
	lr            - learning rate for optimiser
	iters         - number of optimisation steps (iterations) during training
	batch_size    - batch/minibatch size

Note:
	data = load_data("MNIST")
	(x_train, y_train), (x_val, y_val), (x_test, y_test) = data
"""
function fit(data, parameters)
	# define models (Generator, Discriminator)
	model, _ = GenerativeAD.Models.SkipGANomaly_constructor(parameters)

	try
		global info, fit_t, _, _, _ = @timed fit!(model |>gpu , data, parameters)
	catch e
		println("Error caught.")
		return (fit_t = NaN, model = nothing, history = nothing, n_parameters = NaN), []
	end

	training_info = (
		fit_t = fit_t,
		model = (info[2] |> cpu),
		history = info[1], # losses through time
		npars = info[3], # number of parameters
		iters = info[4] # optim iterations of model
		)

	return training_info, [(x -> generalized_anomaly_score_gpu(model|>cpu, x, R=r, L=l, lambda=lam),
		merge(parameters, (R=r, L=l, test_lambda=lam,)))
		for r in ["mae", "mse"] for l in ["mae", "mse"] for lam = 0.1:0.1:0.9 ]
end

#_________________________________________________________________________________________________

if abspath(PROGRAM_FILE) == @__FILE__
	try_counter = 0
	max_tries = 10*max_seed
	cont_string = (contamination == 0.0) ? "" : "_contamination-$contamination"
	while try_counter < max_tries
		parameters = sample_params()

		for seed in 1:max_seed
			savepath = datadir("experiments/images_mvtec$cont_string/$(modelname)/$(category)/ac=1/seed=$(seed)")
			mkpath(savepath)

			data = GenerativeAD.load_data("MVTec-AD", seed=seed, category=category,
					contamination=contamination, img_size=128)

			# computing additional parameters
			in_ch = size(data[1][1],3)
			isize = maximum([size(data[1][1],1), size(data[1][1],2)])

			isize = (isize % 32 != 0) ? isize + 32 - isize % 32 : isize
			# update parameter
			parameters = merge(parameters, (isize=isize, in_ch = in_ch, out_ch = 1))
			# here, check if a model with the same parameters was already tested
			if GenerativeAD.check_params(savepath, parameters)

				data = GenerativeAD.Models.preprocess_images(data, parameters, denominator=32)
				#(X_train,_), (X_val, y_val), (X_test, y_test) = data
				training_info, results = fit(data, parameters)
				# saving model separately
				if training_info.model != nothing
					tagsave(joinpath(savepath, savename("model", parameters, "bson", digits=5)),
						Dict("model"=>training_info.model),
						safe = true)
					training_info = merge(training_info, (model = nothing,))
				end
				# here define what additional info should be saved together with parameters, scores, labels and predict times
				save_entries = merge(training_info, (modelname = modelname, seed = seed,
					category = category, dataset = "MVTec-AD_$category",
					contamination=contamination))

					# now loop over all anomaly score funs
				for result in results
					GenerativeAD.experiment(result..., data, savepath; save_entries...)
				end
				global try_counter = max_tries + 1
			else
				@info "Model already present, sampling new hyperparameters..."
				global try_counter += 1
			end
		end
	end
	(try_counter == max_tries) ? (@info "Reached $(max_tries) tries, giving up.") : nothing
end