text:
function POMDPs.transition(mdp::PedCarMDP, s::PedCarMDPState, a::PedCarMDPAction)
    ego_dist = mdp._ego_transition_dict[(s.ego, LonAccelDirection(a.acc, 4))]
    if s.car == mdp.off_grid
        act_car = LonAccelDirection(0., 0)
    else
        set_car_model!(mdp, s, a)
        a_lon = round(mdp.car_models[s.route].a.a_lon)
        if a_lon == 0.0; a_lon = 0.0; end # in case it rounds to -0.0
        act_car = LonAccelDirection(a_lon, mdp.car_models[s.route].a.direction)
        # println("car takes ", a_lon)
        # println("model said ", mdp.car_models[s.route].a.a_lon)
    end
    car_dist = mdp._car_transition_dict[(s.car, s.route, act_car)]
    ped_dist = mdp._ped_transition_dict[(s.ped, ConstantSpeedDawdling(0., 0))]
    # merge the distribution
    n_next = length(ego_dist.vals)*length(car_dist.vals)*length(ped_dist.vals)
    states = Vector{PedCarMDPState}(undef, n_next)
    probs = Vector{Float64}(undef, n_next)
    l = 1
    for (ego, pego) in weighted_iterator(ego_dist)
        for (car_route, pcar) in weighted_iterator(car_dist)
            car, route = car_route
            for (ped, pped) in weighted_iterator(ped_dist)
                collision = crash(mdp, ego, ped, car)
                # collision = false
                # collision = mdp._collision_checker[(ego, ped, car)]
                states[l] = PedCarMDPState(collision, ego, ped, car, route)
                probs[l] = pego*pcar*pped
                l += 1
            end
        end
    end
    normalize!(probs, 1)
    return SparseCat(states, probs)::SparseCat{Vector{PedCarMDPState}, Vector{Float64}}
end

function init_transition!(mdp::PedCarMDP)
    mdp._ego_transition_dict = ego_transition(mdp)
    mdp._car_transition_dict = car_transition(mdp)
    mdp._ped_transition_dict = ped_transition(mdp)
end

using ProgressMeter
function init_collision_checker!(mdp::PedCarMDP)
    @showprogress for i=1:n_states(mdp)
        s = ind2state(mdp, i)
        mdp._collision_checker[(s.ego, s.ped, s.car)] = s.crash
    end
end

function ego_transition(mdp::PedCarMDP)
    env = mdp.env
    pos_res = mdp.pos_res
    vel_res = mdp.vel_res
    ego_states = get_ego_states(env, pos_res, vel_res)
    ego_transition_dict = EgoTransitionDict()
    for ego in ego_states
        for a in actions(mdp)
            act = LonAccelDirection(a.acc, 4)
            ego_p = propagate(ego, act, env.roadway, mdp.ΔT)
            ego_states, ego_probs = ego_interpolation(mdp, mdp._car_grid, ego_p)
            ego_transition_dict[(ego, act)] = SparseCat(ego_states, ego_probs)
        end
    end
    return ego_transition_dict
end

function car_transition(mdp::PedCarMDP)
    env = mdp.env
    pos_res = mdp.pos_res
    vel_res = mdp.vel_res
    car_action_space = mdp.car_action_space
    routes = get_car_routes(env)
    car_transition_dict = CarTransitionDict()
    for route in routes
        car_route = CarRoute(route[1], route[end])
        car_states = get_car_states(env, route, pos_res, vel_res)
        for car in car_states
            for a in car_action_space
                car_states = Vector{Tuple{VehicleState, CarRoute}}(undef, 0)
                car_probs = Vector{Float64}(undef, 0)
                k = 1
                lane = get_lane(env.roadway, car)
                dir = get_direction(lane, [env.roadway[l] for l in route])
                acts_car = SVector(LonAccelDirection(a - mdp.a_noise, dir), LonAccelDirection(a, dir), LonAccelDirection(a + mdp.a_noise, dir))
                for act in acts_car
                    car_p = propagate(car, act, env.roadway, mdp.ΔT)
                    itp_car_ps, itp_car_weights = car_interpolation(mdp, mdp._car_grid, car_p)
                    for (j, car_pss) in enumerate(itp_car_ps)
                        index_itp_state = findall(x -> x==(car_pss, car_route), car_states)
                        if !(itp_car_weights[j] ≈ 0.)
                            if isempty(index_itp_state)
                                push!(car_states, (car_pss, car_route))
                                push!(car_probs, itp_car_weights[j]*1/length(acts_car))
                            else
                                car_probs[index_itp_state] .+= itp_car_weights[j]*1/length(acts_car)
                            end
                        end
                    end
                end
                normalize!(car_probs, 1)
                car_transition_dict[(car, car_route, LonAccelDirection(a, dir))] = SparseCat(car_states, car_probs)
            end
        end
        car_transition_dict[(mdp.off_grid, car_route, LonAccelDirection(0., 0))] = car_reset(mdp)
    end # end route
    car_transition_dict[(mdp.off_grid, OFF_ROUTE, LonAccelDirection(0., 0))] = car_reset(mdp)
    return car_transition_dict
end

function ped_transition(mdp::PedCarMDP)
    env = mdp.env
    pos_res = mdp.pos_res
    vel_res = mdp.vel_ped_res ## not the same vel res
    ped_action_space = mdp.ped_action_space
    ped_transition_dict = PedTransitionDict()
    ped_states = get_ped_states(env, pos_res, vel_res)
    for ped in ped_states
        for a in ped_action_space
            acts_ped = SVector(ConstantSpeedDawdling(0., 0.), ConstantSpeedDawdling(1., 0.), ConstantSpeedDawdling(2., 0.))
            ped_states = Vector{VehicleState}(undef, 0)
            ped_probs = Vector{Float64}(undef, 0)
            k = 1
            p_a = 1/length(acts_ped) # uniform
            for act in acts_ped
                ped_p = propagate(ped, act, env.roadway, mdp.ΔT)
                itp_ped_ps, itp_ped_weights = ped_interpolation(mdp, mdp._ped_grid, ped_p)
                for (j, ped_pss) in enumerate(itp_ped_ps)
                    index_itp_state = findall(x -> x==ped_pss, ped_states)
                    if !(itp_ped_weights[j] ≈ 0.)
                        if isempty(index_itp_state)
                            push!(ped_states, ped_pss)
                            push!(ped_probs, itp_ped_weights[j]*p_a)
                        else
                            ped_probs[index_itp_state] .+= itp_ped_weights[j]*p_a
                        end
                    end
                end
            end
            normalize!(ped_probs, 1)
            ped_transition_dict[(ped, ConstantSpeedDawdling(0., 0.))] = SparseCat(ped_states, ped_probs)
        end
    end
    ped_transition_dict[(mdp.off_grid, ConstantSpeedDawdling(0., 0))] = ped_reset(mdp)
    return ped_transition_dict
end

function ego_interpolation(mdp::PedCarMDP, grids::Dict{LaneTag, RectangleGrid{2}}, ego::VehicleState)
    N_itp = 4
    itp_car_ps, itp_car_weights = interpolate_state(mdp, ego)
    # states_ = Vector{VehicleState}(N_itp)
    # probs_ = Vector{Float64}(N_itp)
    # fill!(states_, itp_car_ps[1])
    # fill!(probs_, itp_car_weights[1]/(N_itp - length(itp_car_ps)+1))
    # for i=2:length(itp_car_ps)
    #     states_[i] = itp_car_ps[i]
    #     probs_[i] = itp_car_weights[i]
    # end
    # normalize!(probs_, 1)
    return itp_car_ps, itp_car_weights
end

function car_interpolation(mdp::PedCarMDP, grids::Dict{LaneTag, RectangleGrid{2}}, car::VehicleState)
    N_itp = 4
    if car.posF.s >= get_end(get_lane(mdp.env.roadway, car)) && isempty(get_lane(mdp.env.roadway, car).exits)
        states = SVector{N_itp}(fill(mdp.off_grid, N_itp))
        probs = SVector{N_itp}(fill(1.0/N_itp, N_itp))
        return [mdp.off_grid], [1.0]
        # return SparseCat(states, probs)::SparseCat{SVector{N_itp, VehicleState}, SVector{N_itp, Float64}}
    else
        # interpolate car_p in discrete space
        itp_car_ps, itp_car_weights = interpolate_state(mdp, car)
        @assert length(itp_car_ps) <= N_itp
        states_ = Vector{VehicleState}(undef, N_itp)
        fill!(states_, mdp.off_grid)
        probs_ = zeros(N_itp)
        # can have a varying number of states (between 1 and 4)
        for i=1:length(itp_car_ps)
            states_[i] = itp_car_ps[i]
            probs_[i] = itp_car_weights[i]
        end
        normalize!(probs_, 1)
        return itp_car_ps, itp_car_weights
        # states = SVector(states_...)
        # probs = SVector(probs_...)
        # return SparseCat(states, probs)::SparseCat{SVector{N_itp, VehicleState}, SVector{N_itp, Float64}}
    end
end

function ped_interpolation(mdp::PedCarMDP, grids::Dict{LaneTag, RectangleGrid{3}}, ped::VehicleState)
    N_itp = 4
    if ped.posG == mdp.off_grid.posG || (ped.posF.s >= get_end(get_lane(mdp.env.roadway, ped)) && isapprox(ped.posF.ϕ, 0.)) ||
               (isapprox(ped.posF.s, 0., atol=0.01) && isapprox(ped.posF.ϕ, float(pi)))
        states = SVector{N_itp}(fill(mdp.off_grid, N_itp))
        probs = SVector{N_itp}(fill(1.0/N_itp, N_itp))
        return [mdp.off_grid], [1.0]
        # return SparseCat(states, probs)::SparseCat{SVector{N_itp, VehicleState}, SVector{N_itp, Float64}}
    else
        # interpolate ped_p in discrete space
        itp_ped_ps, itp_ped_weights = interpolate_pedestrian(mdp, ped)
        @assert length(itp_ped_ps) <= N_itp
        states_ = Vector{VehicleState}(undef, N_itp)
        fill!(states_, mdp.off_grid)
        probs_ = zeros(N_itp)
        # can have a varying number of states (between 1 and 4)
        for i=1:length(itp_ped_ps)
            states_[i] = itp_ped_ps[i]
            probs_[i] = itp_ped_weights[i]
        end
        normalize!(probs_, 1)
        return itp_ped_ps, itp_ped_weights
        # states = SVector(states_...)
        # probs = SVector(probs_...)
        # return SparseCat(states, probs)::SparseCat{SVector{N_itp, VehicleState}, SVector{N_itp, Float64}}
    end
end

# distribution if the car is off the grid
function car_reset(mdp::PedCarMDP, min_speed::Float64 = 6.0)
    # list of car starting states
    routes = get_car_routes(mdp.env)
    v_space = min_speed:mdp.vel_res:mdp.env.params.speed_limit
    N_states = 0
    for route in routes
        N_states += length(v_space)
    end
    car_states = Vector{VehicleState}(undef, N_states)
    start_routes = Vector{SVector{2, LaneTag}}(undef, N_states)
    i = 1
    for route in routes
        lane = route[1]
        for v in v_space
            car_states[i] = VehicleState(Frenet(mdp.env.roadway[lane], 0.), mdp.env.roadway, v)
            start_routes[i] = SVector{2, LaneTag}(route[1], route[end])
            i += 1
        end
    end
    car_states[end] = mdp.off_grid #TODO FIX, it is overwriting the last state
    start_routes[end] = OFF_ROUTE
    car_probs = ones(N_states)
    car_probs[end] = 1-mdp.car_birth
    car_probs[1:end-1] .= mdp.car_birth/(length(car_states)-1)
    normalize!(car_probs, 1)
    return SparseCat(collect(zip(car_states, start_routes)), car_probs)
end

function ped_reset(mdp::PedCarMDP)
    # list of pedestrian starting states
    n_headings = 2
    lanes = get_ped_lanes(mdp.env)
    v_space = 1.0:mdp.vel_ped_res:2.0
    N_states = length(lanes)*length(v_space)*n_headings
    ped_states = Vector{VehicleState}(undef, N_states)
    i = 1
    for lane in lanes
        dlane = get_discretized_lane(lane, mdp.env.roadway, mdp.pos_res)
        for v in v_space
            ped_states[i] = VehicleState(Frenet(mdp.env.roadway[lane], 0., 0., 0.), mdp.env.roadway, v)
            i += 1
            ped_states[i] = VehicleState(Frenet(mdp.env.roadway[lane], dlane[end], 0., float(pi)), mdp.env.roadway, v)
            i += 1
        end
    end
    ped_states[end] = mdp.off_grid
    ped_probs = ones(N_states)
    ped_probs[end] = 1-mdp.ped_birth
    ped_probs[1:end-1] .= mdp.ped_birth/(length(ped_states)-1)
    normalize!(ped_probs, 1)
    return SparseCat(ped_states, ped_probs)
end

function initial_ego_state(mdp::PedCarMDP)
    lanes = AutomotivePOMDPs.get_ego_route(mdp.env)
    posF = Frenet(mdp.env.roadway[lanes[1]], mdp.ego_start)
    v0 = 0.
    return VehicleState(posF, mdp.env.roadway, v0)
end

function initial_car_state_distribution(mdp::PedCarMDP)
    routes = get_car_routes(mdp.env)
    init_car_routes = []
    init_car_states = VehicleState[]
    for route in routes
        car_states = get_car_states(mdp.env, route, mdp.pos_res, mdp.vel_res)
        for cs in car_states
            push!(init_car_states, cs)
            push!(init_car_routes, route)
        end
    end
    push!(init_car_states, mdp.off_grid)
    push!(init_car_routes, SVector{2, LaneTag}(LaneTag(0,0), LaneTag(0,0)))
    return init_car_states, init_car_routes
end

function initial_car_state(mdp::PedCarMDP, rng::AbstractRNG)
    init_car_states, init_car_routes = initial_car_state_distribution(mdp)
    return rand(rng, init_car_states), rand(rng, init_car_routes)
end

function initial_ped_state_distribution(mdp::PedCarMDP)
    init_ped_states = get_ped_states(mdp.env, mdp.pos_res, mdp.vel_ped_res)
    push!(init_ped_states, mdp.off_grid)
    # uniform (maybe add more weights to the states when pedestrians are not there?)
    probs = ones(length(init_ped_states))
    normalize!(probs, 1)
    return SparseCat(init_ped_states, probs)
end

function initial_ped_state(mdp::PedCarMDP, rng::AbstractRNG)
    init_dist = initial_ped_state_distribution(mdp)
    return rand(rng, init_dist)
end

function POMDPs.initialstate(mdp::PedCarMDP, rng::AbstractRNG)
    return rand(rng, initialstate_distribution(mdp))
end

function POMDPs.initialstate_distribution(mdp::PedCarMDP)
    ego = initial_ego_state(mdp)
    init_car_states, init_car_routes = initial_car_state_distribution(mdp)
    init_ped_dist = initial_ped_state_distribution(mdp)
    init_states = Vector{PedCarMDPState}(undef, 0)
    for i=1:length(init_car_states)
        for j=1:length(init_ped_dist.vals)
            ped = init_ped_dist.vals[j]
            car = init_car_states[i]
            route = init_car_routes[i]
            collision = crash(mdp, ego, car, ped)
            push!(init_states, PedCarMDPState(collision, ego, ped, car, SVector{2, LaneTag}(route[1], route[end])))
        end
    end
    # uniform
    probs = ones(length(init_states))
    normalize!(probs, 1)
    return SparseCat(init_states, probs)
end


text:
"""
For fileformat

MTF forecasting file

Example:
RANK;Dep;Des;2017;2018;2019;2020;2021;2022;2023;2024
Base;ALANDISLANDS;BELARUS;0.002739726;0.000362248;0.000781042;0.000861343;0.000887779;0.00103794;0.001057792;0.000948328
Base;ALANDISLANDS;DENMARK;0.008219178;0.007356467;0.01588544;0.015864169;0.01588544;0.01588544;0.01588544;0.015864169
Base;ALANDISLANDS;ESTONIA;0.010958904;0.003430455;0.004855271;0.004871514;0.004855271;0.004855271;0.004855271;0.004871514
Base;ALANDISLANDS;FINLAND;2.536986301;2.813270647;2.762885585;2.822041934;2.882595408;2.943180367;3.004312251;3.065653134
"""
module For

export read

using CSV
# using DataFrames

# const fileformat = Dict(1=>String, 2=>String, 3=>String, 4=>String,
# 5=>String, 6=>String, 7=>Int64, 8=>Int64, 9=>Int64, 10=>String)

# const header_format = ["DATEACTIVE", "AIRPORT", "RWY", "TIMEOPEN",
# "TIMECLOSED", "YN", "NUM1", "NUM2", "NUM3", "T"]

function read(file)
    df = CSV.read(file, delim=';', header=1, copycols=true)
    return df
end

end # Module

ext: jl


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

mutable struct BGPCommunity <: SwaggerModel
    serviceSupportedRegion::Any # spec type: Union{ Nothing, String } # spec name: serviceSupportedRegion
    communityName::Any # spec type: Union{ Nothing, String } # spec name: communityName
    communityValue::Any # spec type: Union{ Nothing, String } # spec name: communityValue
    communityPrefixes::Any # spec type: Union{ Nothing, Vector{String} } # spec name: communityPrefixes
    isAuthorizedToUse::Any # spec type: Union{ Nothing, Bool } # spec name: isAuthorizedToUse
    serviceGroup::Any # spec type: Union{ Nothing, String } # spec name: serviceGroup

    function BGPCommunity(;serviceSupportedRegion=nothing, communityName=nothing, communityValue=nothing, communityPrefixes=nothing, isAuthorizedToUse=nothing, serviceGroup=nothing)
        o = new()
        validate_property(BGPCommunity, Symbol("serviceSupportedRegion"), serviceSupportedRegion)
        setfield!(o, Symbol("serviceSupportedRegion"), serviceSupportedRegion)
        validate_property(BGPCommunity, Symbol("communityName"), communityName)
        setfield!(o, Symbol("communityName"), communityName)
        validate_property(BGPCommunity, Symbol("communityValue"), communityValue)
        setfield!(o, Symbol("communityValue"), communityValue)
        validate_property(BGPCommunity, Symbol("communityPrefixes"), communityPrefixes)
        setfield!(o, Symbol("communityPrefixes"), communityPrefixes)
        validate_property(BGPCommunity, Symbol("isAuthorizedToUse"), isAuthorizedToUse)
        setfield!(o, Symbol("isAuthorizedToUse"), isAuthorizedToUse)
        validate_property(BGPCommunity, Symbol("serviceGroup"), serviceGroup)
        setfield!(o, Symbol("serviceGroup"), serviceGroup)
        o
    end
end # type BGPCommunity

const _property_map_BGPCommunity = Dict{Symbol,Symbol}(Symbol("serviceSupportedRegion")=>Symbol("serviceSupportedRegion"), Symbol("communityName")=>Symbol("communityName"), Symbol("communityValue")=>Symbol("communityValue"), Symbol("communityPrefixes")=>Symbol("communityPrefixes"), Symbol("isAuthorizedToUse")=>Symbol("isAuthorizedToUse"), Symbol("serviceGroup")=>Symbol("serviceGroup"))
const _property_types_BGPCommunity = Dict{Symbol,String}(Symbol("serviceSupportedRegion")=>"String", Symbol("communityName")=>"String", Symbol("communityValue")=>"String", Symbol("communityPrefixes")=>"Vector{String}", Symbol("isAuthorizedToUse")=>"Bool", Symbol("serviceGroup")=>"String")
Base.propertynames(::Type{ BGPCommunity }) = collect(keys(_property_map_BGPCommunity))
Swagger.property_type(::Type{ BGPCommunity }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_BGPCommunity[name]))}
Swagger.field_name(::Type{ BGPCommunity }, property_name::Symbol) =  _property_map_BGPCommunity[property_name]

function check_required(o::BGPCommunity)
    true
end

function validate_property(::Type{ BGPCommunity }, name::Symbol, val)
end


text:
import Random

function get_mnist()
    mnist = chainer.datasets.mnist
    train, test = mnist.get_mnist()
end

function get_model()
    basemodel = @pydef mutable struct MNIST <: chainer.Chain
        function __init__(self, n_hidden = 100)
            pybuiltin(:super)(MNIST, self).__init__()
            @pywith self.init_scope() begin
                self.l1 = L.Linear(nothing, n_hidden)
                self.l2 = L.Linear(nothing, n_hidden)
                self.l3 = L.Linear(nothing, 10)
            end
        end

        function __call__(self, x)
            h = F.relu(self.l1(x))
            h = F.relu(self.l2(h))
            h = self.l3(h)
            return h
        end
    end

    L.Classifier(basemodel())
end

"""
    train()
Train MNIST model.

```julia
julia> using Gomah
# begin train
julia> train()
[...]
#After training, you can test trained model.
julia> predict()
accuracy for test set = xx.yy [%]
```
"""
function train()
    batchsize = 128
    epochs = 10
    train_set, _ = get_mnist()
    N = pybuiltin(:len)(train_set)
    train_set, val_set = chainer.datasets.split_dataset(
        train_set,
        convert(Int, 0.9 * N),
        Random.shuffle(collect(0:N-1))
    )
    train_iter = SerialIterator(train_set, batchsize)
    test_iter = SerialIterator(
        val_set,
        batchsize,
        repeat = false,
        shuffle = false
    )
    model = get_model()
    optimizer = optimizers.MomentumSGD()
    optimizer.setup(model)
    updater = training.updaters.StandardUpdater(train_iter, optimizer)
    trainer = training.Trainer(updater, (epochs, "epoch"), out = "result")
    trainer.extend(extensions.LogReport())
    trainer.extend(extensions.Evaluator(test_iter, model))
    trainer.extend(extensions.PrintReport([
        "epoch",
        "main/loss",
        "main/accuracy",
        "validation/main/loss",
        "validation/main/accuracy",
        "elapsed_time"
    ]))
    trainer.extend(
        extensions.snapshot_object(model, filename = "bestmodel.npz"),
        trigger = training.triggers.MinValueTrigger("validation/main/loss")
    )
    trainer.run()
end

"""
    predict()
Test pre-trained model of MNIST

After training, you can check accuracy for test set.
See train()
"""
function predict()
    model = get_model()
    _, test_set = get_mnist()
    chainer.serializers.load_npz("result/bestmodel.npz", model)

    counter = 0

    @pywith chainer.using_config("train", false) begin
        @pywith chainer.function.no_backprop_mode() begin
            for t in test_set
                img, label = t
                y = model.predictor(np.expand_dims(img, axis = 0))
                predict = np.argmax(np.squeeze(y.array))
                if predict == label
                    counter += 1
                end
            end
        end
    end

    acc = counter / pybuiltin(:len)(test_set)
    println("accuracy for test set = $(100*acc) [%]",)
end


text:
using PkgBenchmark

mkconfig(; kwargs...) = BenchmarkConfig(; env = Dict("JULIA_NUM_THREADS" => "1"), kwargs...)

group_target = benchmarkpkg(dirname(@__DIR__), mkconfig();
                            resultfile = joinpath(@__DIR__, "result-target.json"))

group_baseline = benchmarkpkg(dirname(@__DIR__), mkconfig(; id = "master");
                              resultfile = joinpath(@__DIR__, "result-baseline.json"))

judgement = judge(group_target, group_baseline)


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: appconfig
using AWS.Compat
using AWS.UUIDs

"""
    create_application(name)
    create_application(name, params::Dict{String,<:Any})

An application in AppConfig is a logical unit of code that provides capabilities for your
customers. For example, an application can be a microservice that runs on Amazon EC2
instances, a mobile application installed by your users, a serverless application using
Amazon API Gateway and AWS Lambda, or any system you run on behalf of others.

# Arguments
- `name`: A name for the application.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the application.
- `"Tags"`: Metadata to assign to the application. Tags help organize and categorize your
  AppConfig resources. Each tag consists of a key and an optional value, both of which you
  define.
"""
create_application(Name; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications", Dict{String, Any}("Name"=>Name); aws_config=aws_config)
create_application(Name, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Name"=>Name), params)); aws_config=aws_config)

"""
    create_configuration_profile(application_id, location_uri, name)
    create_configuration_profile(application_id, location_uri, name, params::Dict{String,<:Any})

Information that enables AppConfig to access the configuration source. Valid configuration
sources include Systems Manager (SSM) documents, SSM Parameter Store parameters, and Amazon
S3 objects. A configuration profile includes the following information.   The Uri location
of the configuration data.   The AWS Identity and Access Management (IAM) role that
provides access to the configuration data.   A validator for the configuration data.
Available validators include either a JSON Schema or an AWS Lambda function.   For more
information, see Create a Configuration and a Configuration Profile in the AWS AppConfig
User Guide.

# Arguments
- `application_id`: The application ID.
- `location_uri`: A URI to locate the configuration. You can specify a Systems Manager
  (SSM) document, an SSM Parameter Store parameter, or an Amazon S3 object. For an SSM
  document, specify either the document name in the format
  ssm-document://<Document_name> or the Amazon Resource Name (ARN). For a parameter,
  specify either the parameter name in the format ssm-parameter://<Parameter_name> or
  the ARN. For an Amazon S3 object, specify the URI in the following format:
  s3://<bucket>/<objectKey> . Here is an example:
  s3://my-bucket/my-app/us-east-1/my-config.json
- `name`: A name for the configuration profile.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the configuration profile.
- `"RetrievalRoleArn"`: The ARN of an IAM role with permission to access the configuration
  at the specified LocationUri.
- `"Tags"`: Metadata to assign to the configuration profile. Tags help organize and
  categorize your AppConfig resources. Each tag consists of a key and an optional value, both
  of which you define.
- `"Validators"`: A list of methods for validating the configuration.
"""
create_configuration_profile(ApplicationId, LocationUri, Name; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles", Dict{String, Any}("LocationUri"=>LocationUri, "Name"=>Name); aws_config=aws_config)
create_configuration_profile(ApplicationId, LocationUri, Name, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("LocationUri"=>LocationUri, "Name"=>Name), params)); aws_config=aws_config)

"""
    create_deployment_strategy(deployment_duration_in_minutes, growth_factor, name, replicate_to)
    create_deployment_strategy(deployment_duration_in_minutes, growth_factor, name, replicate_to, params::Dict{String,<:Any})

A deployment strategy defines important criteria for rolling out your configuration to the
designated targets. A deployment strategy includes: the overall duration required, a
percentage of targets to receive the deployment during each interval, an algorithm that
defines how percentage grows, and bake time.

# Arguments
- `deployment_duration_in_minutes`: Total amount of time for a deployment to last.
- `growth_factor`: The percentage of targets to receive a deployed configuration during
  each interval.
- `name`: A name for the deployment strategy.
- `replicate_to`: Save the deployment strategy to a Systems Manager (SSM) document.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the deployment strategy.
- `"FinalBakeTimeInMinutes"`: The amount of time AppConfig monitors for alarms before
  considering the deployment to be complete and no longer eligible for automatic roll back.
- `"GrowthType"`: The algorithm used to define how percentage grows over time. AWS
  AppConfig supports the following growth types:  Linear: For this type, AppConfig processes
  the deployment by dividing the total number of targets by the value specified for Step
  percentage. For example, a linear deployment that uses a Step percentage of 10 deploys the
  configuration to 10 percent of the hosts. After those deployments are complete, the system
  deploys the configuration to the next 10 percent. This continues until 100% of the targets
  have successfully received the configuration.  Exponential: For this type, AppConfig
  processes the deployment exponentially using the following formula: G*(2^N). In this
  formula, G is the growth factor specified by the user and N is the number of steps until
  the configuration is deployed to all targets. For example, if you specify a growth factor
  of 2, then the system rolls out the configuration as follows:  2*(2^0)   2*(2^1)   2*(2^2)
  Expressed numerically, the deployment rolls out as follows: 2% of the targets, 4% of the
  targets, 8% of the targets, and continues until the configuration has been deployed to all
  targets.
- `"Tags"`: Metadata to assign to the deployment strategy. Tags help organize and
  categorize your AppConfig resources. Each tag consists of a key and an optional value, both
  of which you define.
"""
create_deployment_strategy(DeploymentDurationInMinutes, GrowthFactor, Name, ReplicateTo; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/deploymentstrategies", Dict{String, Any}("DeploymentDurationInMinutes"=>DeploymentDurationInMinutes, "GrowthFactor"=>GrowthFactor, "Name"=>Name, "ReplicateTo"=>ReplicateTo); aws_config=aws_config)
create_deployment_strategy(DeploymentDurationInMinutes, GrowthFactor, Name, ReplicateTo, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/deploymentstrategies", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("DeploymentDurationInMinutes"=>DeploymentDurationInMinutes, "GrowthFactor"=>GrowthFactor, "Name"=>Name, "ReplicateTo"=>ReplicateTo), params)); aws_config=aws_config)

"""
    create_environment(application_id, name)
    create_environment(application_id, name, params::Dict{String,<:Any})

For each application, you define one or more environments. An environment is a logical
deployment group of AppConfig targets, such as applications in a Beta or Production
environment. You can also define environments for application subcomponents such as the
Web, Mobile and Back-end components for your application. You can configure Amazon
CloudWatch alarms for each environment. The system monitors alarms during a configuration
deployment. If an alarm is triggered, the system rolls back the configuration.

# Arguments
- `application_id`: The application ID.
- `name`: A name for the environment.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the environment.
- `"Monitors"`: Amazon CloudWatch alarms to monitor during the deployment process.
- `"Tags"`: Metadata to assign to the environment. Tags help organize and categorize your
  AppConfig resources. Each tag consists of a key and an optional value, both of which you
  define.
"""
create_environment(ApplicationId, Name; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/environments", Dict{String, Any}("Name"=>Name); aws_config=aws_config)
create_environment(ApplicationId, Name, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/environments", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Name"=>Name), params)); aws_config=aws_config)

"""
    create_hosted_configuration_version(application_id, configuration_profile_id, content, content-_type)
    create_hosted_configuration_version(application_id, configuration_profile_id, content, content-_type, params::Dict{String,<:Any})

Create a new configuration in the AppConfig configuration store.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.
- `content`: The content of the configuration or the configuration data.
- `content-_type`: A standard MIME type describing the format of the configuration content.
  For more information, see Content-Type.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the configuration.
- `"Latest-Version-Number"`: An optional locking token used to prevent race conditions from
  overwriting configuration updates when creating a new version. To ensure your data is not
  overwritten when creating multiple hosted configuration versions in rapid succession,
  specify the version of the latest hosted configuration version.
"""
create_hosted_configuration_version(ApplicationId, ConfigurationProfileId, Content, Content_Type; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions", Dict{String, Any}("Content"=>Content, "headers"=>Dict{String, Any}("Content-Type"=>Content_Type)); aws_config=aws_config)
create_hosted_configuration_version(ApplicationId, ConfigurationProfileId, Content, Content_Type, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Content"=>Content, "headers"=>Dict{String, Any}("Content-Type"=>Content_Type)), params)); aws_config=aws_config)

"""
    delete_application(application_id)
    delete_application(application_id, params::Dict{String,<:Any})

Delete an application. Deleting an application does not delete a configuration from a host.

# Arguments
- `application_id`: The ID of the application to delete.

"""
delete_application(ApplicationId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)"; aws_config=aws_config)
delete_application(ApplicationId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)", params; aws_config=aws_config)

"""
    delete_configuration_profile(application_id, configuration_profile_id)
    delete_configuration_profile(application_id, configuration_profile_id, params::Dict{String,<:Any})

Delete a configuration profile. Deleting a configuration profile does not delete a
configuration from a host.

# Arguments
- `application_id`: The application ID that includes the configuration profile you want to
  delete.
- `configuration_profile_id`: The ID of the configuration profile you want to delete.

"""
delete_configuration_profile(ApplicationId, ConfigurationProfileId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)"; aws_config=aws_config)
delete_configuration_profile(ApplicationId, ConfigurationProfileId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)", params; aws_config=aws_config)

"""
    delete_deployment_strategy(deployment_strategy_id)
    delete_deployment_strategy(deployment_strategy_id, params::Dict{String,<:Any})

Delete a deployment strategy. Deleting a deployment strategy does not delete a
configuration from a host.

# Arguments
- `deployment_strategy_id`: The ID of the deployment strategy you want to delete.

"""
delete_deployment_strategy(DeploymentStrategyId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/deployementstrategies/$(DeploymentStrategyId)"; aws_config=aws_config)
delete_deployment_strategy(DeploymentStrategyId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/deployementstrategies/$(DeploymentStrategyId)", params; aws_config=aws_config)

"""
    delete_environment(application_id, environment_id)
    delete_environment(application_id, environment_id, params::Dict{String,<:Any})

Delete an environment. Deleting an environment does not delete a configuration from a host.

# Arguments
- `application_id`: The application ID that includes the environment you want to delete.
- `environment_id`: The ID of the environment you want to delete.

"""
delete_environment(ApplicationId, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/environments/$(EnvironmentId)"; aws_config=aws_config)
delete_environment(ApplicationId, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/environments/$(EnvironmentId)", params; aws_config=aws_config)

"""
    delete_hosted_configuration_version(application_id, configuration_profile_id, version_number)
    delete_hosted_configuration_version(application_id, configuration_profile_id, version_number, params::Dict{String,<:Any})

Delete a version of a configuration from the AppConfig configuration store.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.
- `version_number`: The versions number to delete.

"""
delete_hosted_configuration_version(ApplicationId, ConfigurationProfileId, VersionNumber; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions/$(VersionNumber)"; aws_config=aws_config)
delete_hosted_configuration_version(ApplicationId, ConfigurationProfileId, VersionNumber, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions/$(VersionNumber)", params; aws_config=aws_config)

"""
    get_application(application_id)
    get_application(application_id, params::Dict{String,<:Any})

Retrieve information about an application.

# Arguments
- `application_id`: The ID of the application you want to get.

"""
get_application(ApplicationId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)"; aws_config=aws_config)
get_application(ApplicationId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)", params; aws_config=aws_config)

"""
    get_configuration(application, configuration, environment, client_id)
    get_configuration(application, configuration, environment, client_id, params::Dict{String,<:Any})

Receive information about a configuration.  AWS AppConfig uses the value of the
ClientConfigurationVersion parameter to identify the configuration version on your clients.
If you don’t send ClientConfigurationVersion with each call to GetConfiguration, your
clients receive the current configuration. You are charged each time your clients receive a
configuration. To avoid excess charges, we recommend that you include the
ClientConfigurationVersion value with every call to GetConfiguration. This value must be
saved on your client. Subsequent calls to GetConfiguration must pass this value by using
the ClientConfigurationVersion parameter.

# Arguments
- `application`: The application to get. Specify either the application name or the
  application ID.
- `configuration`: The configuration to get. Specify either the configuration name or the
  configuration ID.
- `environment`: The environment to get. Specify either the environment name or the
  environment ID.
- `client_id`: A unique ID to identify the client for the configuration. This ID enables
  AppConfig to deploy the configuration in intervals, as defined in the deployment strategy.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"client_configuration_version"`: The configuration version returned in the most recent
  GetConfiguration response.  AWS AppConfig uses the value of the ClientConfigurationVersion
  parameter to identify the configuration version on your clients. If you don’t send
  ClientConfigurationVersion with each call to GetConfiguration, your clients receive the
  current configuration. You are charged each time your clients receive a configuration. To
  avoid excess charges, we recommend that you include the ClientConfigurationVersion value
  with every call to GetConfiguration. This value must be saved on your client. Subsequent
  calls to GetConfiguration must pass this value by using the ClientConfigurationVersion
  parameter.   For more information about working with configurations, see Retrieving the
  Configuration in the AWS AppConfig User Guide.
"""
get_configuration(Application, Configuration, Environment, client_id; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(Application)/environments/$(Environment)/configurations/$(Configuration)", Dict{String, Any}("client_id"=>client_id); aws_config=aws_config)
get_configuration(Application, Configuration, Environment, client_id, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(Application)/environments/$(Environment)/configurations/$(Configuration)", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("client_id"=>client_id), params)); aws_config=aws_config)

"""
    get_configuration_profile(application_id, configuration_profile_id)
    get_configuration_profile(application_id, configuration_profile_id, params::Dict{String,<:Any})

Retrieve information about a configuration profile.

# Arguments
- `application_id`: The ID of the application that includes the configuration profile you
  want to get.
- `configuration_profile_id`: The ID of the configuration profile you want to get.

"""
get_configuration_profile(ApplicationId, ConfigurationProfileId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)"; aws_config=aws_config)
get_configuration_profile(ApplicationId, ConfigurationProfileId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)", params; aws_config=aws_config)

"""
    get_deployment(application_id, deployment_number, environment_id)
    get_deployment(application_id, deployment_number, environment_id, params::Dict{String,<:Any})

Retrieve information about a configuration deployment.

# Arguments
- `application_id`: The ID of the application that includes the deployment you want to get.
- `deployment_number`: The sequence number of the deployment.
- `environment_id`: The ID of the environment that includes the deployment you want to get.

"""
get_deployment(ApplicationId, DeploymentNumber, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments/$(DeploymentNumber)"; aws_config=aws_config)
get_deployment(ApplicationId, DeploymentNumber, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments/$(DeploymentNumber)", params; aws_config=aws_config)

"""
    get_deployment_strategy(deployment_strategy_id)
    get_deployment_strategy(deployment_strategy_id, params::Dict{String,<:Any})

Retrieve information about a deployment strategy. A deployment strategy defines important
criteria for rolling out your configuration to the designated targets. A deployment
strategy includes: the overall duration required, a percentage of targets to receive the
deployment during each interval, an algorithm that defines how percentage grows, and bake
time.

# Arguments
- `deployment_strategy_id`: The ID of the deployment strategy to get.

"""
get_deployment_strategy(DeploymentStrategyId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/deploymentstrategies/$(DeploymentStrategyId)"; aws_config=aws_config)
get_deployment_strategy(DeploymentStrategyId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/deploymentstrategies/$(DeploymentStrategyId)", params; aws_config=aws_config)

"""
    get_environment(application_id, environment_id)
    get_environment(application_id, environment_id, params::Dict{String,<:Any})

Retrieve information about an environment. An environment is a logical deployment group of
AppConfig applications, such as applications in a Production environment or in an EU_Region
environment. Each configuration deployment targets an environment. You can enable one or
more Amazon CloudWatch alarms for an environment. If an alarm is triggered during a
deployment, AppConfig roles back the configuration.

# Arguments
- `application_id`: The ID of the application that includes the environment you want to get.
- `environment_id`: The ID of the environment you wnat to get.

"""
get_environment(ApplicationId, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)"; aws_config=aws_config)
get_environment(ApplicationId, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)", params; aws_config=aws_config)

"""
    get_hosted_configuration_version(application_id, configuration_profile_id, version_number)
    get_hosted_configuration_version(application_id, configuration_profile_id, version_number, params::Dict{String,<:Any})

Get information about a specific configuration version.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.
- `version_number`: The version.

"""
get_hosted_configuration_version(ApplicationId, ConfigurationProfileId, VersionNumber; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions/$(VersionNumber)"; aws_config=aws_config)
get_hosted_configuration_version(ApplicationId, ConfigurationProfileId, VersionNumber, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions/$(VersionNumber)", params; aws_config=aws_config)

"""
    list_applications()
    list_applications(params::Dict{String,<:Any})

List all applications in your AWS account.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_applications(; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications"; aws_config=aws_config)
list_applications(params::AbstractDict{String, Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications", params; aws_config=aws_config)

"""
    list_configuration_profiles(application_id)
    list_configuration_profiles(application_id, params::Dict{String,<:Any})

Lists the configuration profiles for an application.

# Arguments
- `application_id`: The application ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_configuration_profiles(ApplicationId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles"; aws_config=aws_config)
list_configuration_profiles(ApplicationId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles", params; aws_config=aws_config)

"""
    list_deployment_strategies()
    list_deployment_strategies(params::Dict{String,<:Any})

List deployment strategies.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_deployment_strategies(; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/deploymentstrategies"; aws_config=aws_config)
list_deployment_strategies(params::AbstractDict{String, Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/deploymentstrategies", params; aws_config=aws_config)

"""
    list_deployments(application_id, environment_id)
    list_deployments(application_id, environment_id, params::Dict{String,<:Any})

Lists the deployments for an environment.

# Arguments
- `application_id`: The application ID.
- `environment_id`: The environment ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_deployments(ApplicationId, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments"; aws_config=aws_config)
list_deployments(ApplicationId, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments", params; aws_config=aws_config)

"""
    list_environments(application_id)
    list_environments(application_id, params::Dict{String,<:Any})

List the environments for an application.

# Arguments
- `application_id`: The application ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_environments(ApplicationId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments"; aws_config=aws_config)
list_environments(ApplicationId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/environments", params; aws_config=aws_config)

"""
    list_hosted_configuration_versions(application_id, configuration_profile_id)
    list_hosted_configuration_versions(application_id, configuration_profile_id, params::Dict{String,<:Any})

View a list of configurations stored in the AppConfig configuration store by version.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"max_results"`: The maximum number of items to return for this call. The call also
  returns a token that you can specify in a subsequent call to get the next set of results.
- `"next_token"`: A token to start the list. Use this token to get the next set of results.
"""
list_hosted_configuration_versions(ApplicationId, ConfigurationProfileId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions"; aws_config=aws_config)
list_hosted_configuration_versions(ApplicationId, ConfigurationProfileId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/hostedconfigurationversions", params; aws_config=aws_config)

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Retrieves the list of key-value tags assigned to the resource.

# Arguments
- `resource_arn`: The resource ARN.

"""
list_tags_for_resource(ResourceArn; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/tags/$(ResourceArn)"; aws_config=aws_config)
list_tags_for_resource(ResourceArn, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("GET", "/tags/$(ResourceArn)", params; aws_config=aws_config)

"""
    start_deployment(application_id, configuration_profile_id, configuration_version, deployment_strategy_id, environment_id)
    start_deployment(application_id, configuration_profile_id, configuration_version, deployment_strategy_id, environment_id, params::Dict{String,<:Any})

Starts a deployment.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.
- `configuration_version`: The configuration version to deploy.
- `deployment_strategy_id`: The deployment strategy ID.
- `environment_id`: The environment ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the deployment.
- `"Tags"`: Metadata to assign to the deployment. Tags help organize and categorize your
  AppConfig resources. Each tag consists of a key and an optional value, both of which you
  define.
"""
start_deployment(ApplicationId, ConfigurationProfileId, ConfigurationVersion, DeploymentStrategyId, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments", Dict{String, Any}("ConfigurationProfileId"=>ConfigurationProfileId, "ConfigurationVersion"=>ConfigurationVersion, "DeploymentStrategyId"=>DeploymentStrategyId); aws_config=aws_config)
start_deployment(ApplicationId, ConfigurationProfileId, ConfigurationVersion, DeploymentStrategyId, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ConfigurationProfileId"=>ConfigurationProfileId, "ConfigurationVersion"=>ConfigurationVersion, "DeploymentStrategyId"=>DeploymentStrategyId), params)); aws_config=aws_config)

"""
    stop_deployment(application_id, deployment_number, environment_id)
    stop_deployment(application_id, deployment_number, environment_id, params::Dict{String,<:Any})

Stops a deployment. This API action works only on deployments that have a status of
DEPLOYING. This action moves the deployment to a status of ROLLED_BACK.

# Arguments
- `application_id`: The application ID.
- `deployment_number`: The sequence number of the deployment.
- `environment_id`: The environment ID.

"""
stop_deployment(ApplicationId, DeploymentNumber, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments/$(DeploymentNumber)"; aws_config=aws_config)
stop_deployment(ApplicationId, DeploymentNumber, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/applications/$(ApplicationId)/environments/$(EnvironmentId)/deployments/$(DeploymentNumber)", params; aws_config=aws_config)

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Metadata to assign to an AppConfig resource. Tags help organize and categorize your
AppConfig resources. Each tag consists of a key and an optional value, both of which you
define. You can specify a maximum of 50 tags for a resource.

# Arguments
- `resource_arn`: The ARN of the resource for which to retrieve tags.
- `tags`: The key-value string map. The valid character set is [a-zA-Z+-=._:/]. The tag key
  can be up to 128 characters and must not start with aws:. The tag value can be up to 256
  characters.

"""
tag_resource(ResourceArn, Tags; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/tags/$(ResourceArn)", Dict{String, Any}("Tags"=>Tags); aws_config=aws_config)
tag_resource(ResourceArn, Tags, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/tags/$(ResourceArn)", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Tags"=>Tags), params)); aws_config=aws_config)

"""
    untag_resource(resource_arn, tag_keys)
    untag_resource(resource_arn, tag_keys, params::Dict{String,<:Any})

Deletes a tag key and value from an AppConfig resource.

# Arguments
- `resource_arn`: The ARN of the resource for which to remove tags.
- `tag_keys`: The tag keys to delete.

"""
untag_resource(ResourceArn, tagKeys; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/tags/$(ResourceArn)", Dict{String, Any}("tagKeys"=>tagKeys); aws_config=aws_config)
untag_resource(ResourceArn, tagKeys, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("DELETE", "/tags/$(ResourceArn)", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("tagKeys"=>tagKeys), params)); aws_config=aws_config)

"""
    update_application(application_id)
    update_application(application_id, params::Dict{String,<:Any})

Updates an application.

# Arguments
- `application_id`: The application ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the application.
- `"Name"`: The name of the application.
"""
update_application(ApplicationId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)"; aws_config=aws_config)
update_application(ApplicationId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)", params; aws_config=aws_config)

"""
    update_configuration_profile(application_id, configuration_profile_id)
    update_configuration_profile(application_id, configuration_profile_id, params::Dict{String,<:Any})

Updates a configuration profile.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The ID of the configuration profile.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the configuration profile.
- `"Name"`: The name of the configuration profile.
- `"RetrievalRoleArn"`: The ARN of an IAM role with permission to access the configuration
  at the specified LocationUri.
- `"Validators"`: A list of methods for validating the configuration.
"""
update_configuration_profile(ApplicationId, ConfigurationProfileId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)"; aws_config=aws_config)
update_configuration_profile(ApplicationId, ConfigurationProfileId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)", params; aws_config=aws_config)

"""
    update_deployment_strategy(deployment_strategy_id)
    update_deployment_strategy(deployment_strategy_id, params::Dict{String,<:Any})

Updates a deployment strategy.

# Arguments
- `deployment_strategy_id`: The deployment strategy ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"DeploymentDurationInMinutes"`: Total amount of time for a deployment to last.
- `"Description"`: A description of the deployment strategy.
- `"FinalBakeTimeInMinutes"`: The amount of time AppConfig monitors for alarms before
  considering the deployment to be complete and no longer eligible for automatic roll back.
- `"GrowthFactor"`: The percentage of targets to receive a deployed configuration during
  each interval.
- `"GrowthType"`: The algorithm used to define how percentage grows over time. AWS
  AppConfig supports the following growth types:  Linear: For this type, AppConfig processes
  the deployment by increments of the growth factor evenly distributed over the deployment
  time. For example, a linear deployment that uses a growth factor of 20 initially makes the
  configuration available to 20 percent of the targets. After 1/5th of the deployment time
  has passed, the system updates the percentage to 40 percent. This continues until 100% of
  the targets are set to receive the deployed configuration.  Exponential: For this type,
  AppConfig processes the deployment exponentially using the following formula: G*(2^N). In
  this formula, G is the growth factor specified by the user and N is the number of steps
  until the configuration is deployed to all targets. For example, if you specify a growth
  factor of 2, then the system rolls out the configuration as follows:  2*(2^0)   2*(2^1)
  2*(2^2)  Expressed numerically, the deployment rolls out as follows: 2% of the targets, 4%
  of the targets, 8% of the targets, and continues until the configuration has been deployed
  to all targets.
"""
update_deployment_strategy(DeploymentStrategyId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/deploymentstrategies/$(DeploymentStrategyId)"; aws_config=aws_config)
update_deployment_strategy(DeploymentStrategyId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/deploymentstrategies/$(DeploymentStrategyId)", params; aws_config=aws_config)

"""
    update_environment(application_id, environment_id)
    update_environment(application_id, environment_id, params::Dict{String,<:Any})

Updates an environment.

# Arguments
- `application_id`: The application ID.
- `environment_id`: The environment ID.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: A description of the environment.
- `"Monitors"`: Amazon CloudWatch alarms to monitor during the deployment process.
- `"Name"`: The name of the environment.
"""
update_environment(ApplicationId, EnvironmentId; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)/environments/$(EnvironmentId)"; aws_config=aws_config)
update_environment(ApplicationId, EnvironmentId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("PATCH", "/applications/$(ApplicationId)/environments/$(EnvironmentId)", params; aws_config=aws_config)

"""
    validate_configuration(application_id, configuration_profile_id, configuration_version)
    validate_configuration(application_id, configuration_profile_id, configuration_version, params::Dict{String,<:Any})

Uses the validators in a configuration profile to validate a configuration.

# Arguments
- `application_id`: The application ID.
- `configuration_profile_id`: The configuration profile ID.
- `configuration_version`: The version of the configuration to validate.

"""
validate_configuration(ApplicationId, ConfigurationProfileId, configuration_version; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/validators", Dict{String, Any}("configuration_version"=>configuration_version); aws_config=aws_config)
validate_configuration(ApplicationId, ConfigurationProfileId, configuration_version, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = appconfig("POST", "/applications/$(ApplicationId)/configurationprofiles/$(ConfigurationProfileId)/validators", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("configuration_version"=>configuration_version), params)); aws_config=aws_config)


text:
import Test: @test, @testset, @test_throws
import Luna: Antiresonant, Capillary, Modes
import Luna.PhysData: wlfreq

@testset "Zeisberger Model" begin
    a = 20e-6
    m = Capillary.MarcatiliMode(a, :Air, 0, (ω; z) -> 1.45)
    w = 0.7e-6
    arm = Antiresonant.ZeisbergerMode(a, :Air, 0, (ω; z) -> 1.45; wallthickness=w)
    arm2 = Antiresonant.ZeisbergerMode(m; wallthickness=w)

    @test Modes.N(m) == Modes.N(arm) == Modes.N(arm2)
    @test Modes.Aeff(m) == Modes.Aeff(arm) == Modes.Aeff(arm2)
    r = a*rand(50)
    θ = 2π*rand(50)
    @test all(Modes.field.(m, (r, θ)) .== Modes.field.(arm, (r, θ)))
    @test all(Modes.field.(m, (r, θ)) .== Modes.field.(arm2, (r, θ)))
    λ = collect(range(390e-9, stop=1450e-9, length=2^11))
    ω = wlfreq.(λ)
    @test all(Modes.neff.(arm, ω) .== Modes.neff.(arm2, ω))

    @test Modes.neff(arm, 2.5e15) ≈ 0.9999193518567425 + 1.87925966056515e-6im
    arm = Antiresonant.ZeisbergerMode(a, :Air, 0, (ω; z) -> 1.45; wallthickness=w, loss=false)
    @test Modes.neff(arm, 2.5e15) == 0.9999193518567425
    arm = Antiresonant.ZeisbergerMode(a, :Air, 0, (ω; z) -> 1.45; wallthickness=w, loss=0.5)
    @test Modes.neff(arm, 2.5e15) ≈ 0.9999193518567425 + 0.5*1.87925966056515e-6im
    @test_throws ArgumentError Antiresonant.ZeisbergerMode(a, :Air, 0; wallthickness=w, loss=0.5im)
end

##
@testset "Vincetti Model" begin
t = 1e-6
r_ext = 10e-6
δ = 5e-6 # tube spacing
n = 1.44
N = 8 # number of tubes

Rco = Antiresonant.getRco(r_ext, N, δ)

@test Antiresonant.getδ(Rco, r_ext, N) ≈ δ
@test Antiresonant.getr_ext(Rco, N, δ) ≈ r_ext

m = Antiresonant.VincettiMode(Rco; wallthickness=t, tube_radius=r_ext, Ntubes=N, cladn=n)

F = collect(range(0.4, 4.2, 2^14))
λ = @. 2t/F*sqrt(n^2-1)

scale = 0.5
msc = Antiresonant.VincettiMode(Rco; wallthickness=t, tube_radius=r_ext, Ntubes=N, cladn=n,
                                     loss=scale)
@test Modes.α.(msc, wlfreq.(λ)) ≈ scale*Modes.α.(m, wlfreq.(λ))
m0 = Antiresonant.VincettiMode(Rco; wallthickness=t, tube_radius=r_ext, Ntubes=N, cladn=n,
                                     loss=false)
@test all(Modes.α.(m0, wlfreq.(λ)) .== 0)

@test Modes.neff(m, wlfreq(1030e-9)) ≈ 0.9998598623672965 + 3.4579455755137454e-8im

@test Modes.dimlimits(m) == Modes.dimlimits(m.m)
end


text:
module Relational

import Base: select, join

export Relation, select, where, join, groupby, agg, count

include("attribute.jl")

abstract AbstractRelation

immutable Relation <: AbstractRelation
    name::Symbol
    attrs::Tuple
    _lookup::Dict
    Relation(n, attrs) =
       new(n, attrs, Dict([name(a) => a for a in attrs]))
end
Relation(attrs) = Relation(:_, attrs)
fieldset(x::Relation) = x

include("source.jl") # Actual data sources, courtesy DataStreams

### Relation+Attr ###
immutable QualifiedAttr{T} <: Attr{T}
    relation::Relation
    attr::Attr{T}
end
name(x::QualifiedAttr) = name(x.attr)
Base.getindex(s::Relation, x) = QualifiedAttr(s, s._lookup[x])
Base.getindex(s::Relation, x::String) = s[symbol(x)]

Base.getindex(s::AbstractRelation, x) = fieldset(s)[x]
### Projection ###

immutable Select <: AbstractRelation
    from::AbstractRelation
    relation::Relation
end
select(from, fields::Tuple) = Select(from, Relation(fieldset(from).name, fields))
select(from, fields::Attr) = Select(from, Relation(fieldset(from).name, (fields,)))
fieldset(x::Relation) = x.relation

### Selection ###

immutable Where <: AbstractRelation
    source::AbstractRelation
    predicate::Attr{Bool}
end
where(source, attr) = Where(source, attr)
fieldset(x::Where) = fieldset(x.source)

### Join ###

immutable Join <: AbstractRelation
    l::AbstractRelation
    r::AbstractRelation
    link::Attr
end
join(l, r, link) = Join(l,r, link)
fieldset(x::Join) = fieldset(x.l) # TODO: Merge relation

### GroupBy ###

immutable GroupBy <: AbstractRelation
    source::AbstractRelation
    grouping::Tuple{Attr}
end
groupby(source, g) = GroupBy(source, g)
groupby(source, g::Attr) = GroupBy(source, (g,))
fieldset(x::GroupBy) = fieldset(x.source)

### Aggregate ###

immutable Aggregate <: AbstractRelation
    source::AbstractRelation
    aggregator::Aggregator
end
agg(r, a::Aggregator) = Aggregate(r, a)
agg(r, a::Function) = Aggregate(r, ReduceAgg(a))
fieldset(x::Aggregate) = fieldset(x.source)

Base.sum(r::AbstractRelation, f::Attr) = agg(r, Sum(f))
Base.mean(r::AbstractRelation, f::Attr) = agg(r, Avg(f))
count(r::AbstractRelation) = agg(r, Count())

include("show.jl")

end # module


text:
### A Pluto.jl notebook ###
# v0.16.1

using Markdown
using InteractiveUtils

# ╔═╡ d0aeac70-e8b3-4765-9483-e1cbb6233793
begin
	using Pkg
	Pkg.activate("../../")

	md"""
	# Lattice-Free MMI part I: Graphs preparation
	*[Lucas Ondel](https://lucasondel.github.io/), September 2021*

	This is the first step of the Lattice-Free MMI training recipe: preparing the graphs needed by the MMI objective function.
	"""
end

# ╔═╡ e313e3c6-1ba5-11ec-2657-9574ba676d47
begin
	using JLD2
	using MarkovModels
	using PlutoUI
	using Random
	using TOML
end

# ╔═╡ 605fd496-1707-4ce8-84f3-dc2dabe13d4e
TableOfContents()

# ╔═╡ 3bf681b3-9949-4a94-824b-b2f7416751bf
md"""
## Setup
"""

# ╔═╡ ad9289fd-3d73-443d-8a6d-79edcf7d8c0a
md"""
We use the [MarkovModels](https://github.com/lucasondel/MarkovModels.jl) package to build our graphs.
"""

# ╔═╡ b248cbdb-93ac-4e40-92ed-1cbd6f0b8f89
md"""
We load the configuration file of the experiment. By default, we look for the file  `config.toml` in the root directory (i.e. the directory containing this notebook). Alternatively, when calling this notebook as a julia script, you can specify another configuration file by setting the environment variable `SPEECHLAB_LFMMI_CONFIG=/path/to/file`.
"""

# ╔═╡ fb1c9103-ad50-4eeb-b142-ff74f55170c3
rootdir = @__DIR__

# ╔═╡ d8c0ac69-6753-4eb2-bf35-d6a3744eb261
configfile = get(ENV, "SPEECHLAB_LFMMI_CONFIG", joinpath(rootdir, "config.toml"))

# ╔═╡ c16e546d-f83a-4094-bf0c-31457bef0ef5
begin
	@info "Reading configuration from $configfile."
	config = TOML.parsefile(configfile)
end

# ╔═╡ ce7861dd-3868-4fc5-b5c8-339ab1a4981f
md"""
## Input
To build the graphs, we need a *lexicon*, the *list of acoustic units* (i.e. phones), the *topology for each unit* and the *textual transcription* of the training and development set.

### Lexicon
The lexicon contains the phonetic pronunciation of each word of the vocabulary. It has the following format:
```
word1  a b c
word2  e d f
...
```
where `a`, `b`, `c`... are phones in a given language.
!!! note
	A word can have multiple pronunciations. In this case, you can write them on separate lines in the lexicon such as:
	```
	word1 h e l o
	word1 h a l o
    ...
	```

### Units
The list of units is stored in file formatted as:
```
unit1  [tag1 tag2 ...]
unit2  [tag1 tag2 ...]
...
```
Tags are optional user-defined categories. They allow to have unit specific topologies. Here is an example of a units file for the English language:
```
SIL     nonspeech-unit  silence
NSN     nonspeech-unit  nonspoken-noise
SPN     speech-unit     spoken-noise
AA      speech-unit     phone
AE      speech-unit     phone
AH      speech-unit     phone
AO      speech-unit     phone
AW      speech-unit     phone
AY      speech-unit     phone
B       speech-unit     phone
```

### HMM topologies
For each unit, we specify the HMM topology. The path (relative to the root directory) of the topology file is given in in the configuration file by the key <graphs.hmm_topologies>. In this file, you need to specify the HMM states and their arcs. For instance:
```TOML
states = [
	{ initweight = 1.0, finalweight = 0.5 },
	{ initweight = 0.0, finalweight = 0.5 },
]

arcs = [
	{ src = 1, dest = 2, weight = 0.5 },
	{ src = 2, dest = 2, weight = 0.5 },
]
```
If you want to have a particular topology for a given units, you can use the tags specified in the units file. For instance, let's say we have the following units file:
```
SIL     nonspeech-unit  silence
NSN     nonspeech-unit  nonspoken-noise
AA      speech-unit     vowel
AE      speech-unit     vowel
AE      speech-unit     consonant
```
Then we can have different topologies given the category tags.
```TOML

# This is for units that don't match specific categories.
states = [
	{ initweight = 1.0, finalweight = 0.5 },
	{ initweight = 0.0, finalweight = 0.5 },
]

arcs = [
	{ src = 1, dest = 2, weight = 0.5 },
	{ src = 2, dest = 2, weight = 0.5 },
]

# For all nonspeech-units
[nonspeech-units]
states = [
	{ initweight = 1.0, finalweight = 0.5 },
]

arcs = [
	{ src = 1, dest = 1, weight = 0.5 },
]

# For vowels
[speech-units.vowels]
states = [
	{ initweight = 1.0, finalweight = 0.0 },
	{ initweight = 0.0, finalweight = 0.33 },
	{ initweight = 0.0, finalweight = 0.5 },
]

arcs = [
	{ src = 1, dest = 2, weight = 0.5 },
	{ src = 2, dest = 2, weight = 0.33 },
	{ src = 2, dest = 3, weight = 0.33 },
  	{ src = 3, dest = 3, weight = 0.5 },
]
```

### Transcription
The text transcription simply a text file with one line per utterance:
```
utt1  w1 w2 w3 ...
utt2  w1 w2 w3 ...
...
```

We expect these different files to be organized as:
```
<dataset.dir>/
+-- <dataset.name>/
|   +-- lang/
|   |   +-- units
|   |   +-- lexicon
|   +-- dev/
|   |   +-- text
|   +-- train/
|   |   +-- text
```
The keys `<dataset.dir> and <dataset.name> are taken from the configuration file.
!!! note
	`<dataset.dir>` should be an absolute directory.
"""

# ╔═╡ a55eaabf-cb79-4130-affa-a64c3bce1526
lexiconfile = joinpath(config["dataset"]["dir"], config["dataset"]["name"],
					   "lang", "lexicon")

# ╔═╡ 537679cc-3104-4bbb-aab8-21d80ed03ced
unitsfile = joinpath(config["dataset"]["dir"], config["dataset"]["name"],
					 "lang", "units")

# ╔═╡ 43c0ce2c-c28d-4b77-a2ac-75edacc8774f
topofile = joinpath(rootdir, config["graphs"]["hmm_topologies"])

# ╔═╡ 8bd0b6af-9aff-43e7-a06f-934b2525d11a
begin
	@info "Reading topologies from $topofile."
	topologies = TOML.parsefile(topofile)
end

# ╔═╡ cf5fc8b6-e14d-49b2-9c7d-c8e58e80c118
md"""
We have two options to generate the graphs: either we build the graph by "sampling" an alignment sequence or we build the graph of all possible sequence. The generating mode is set is decided by the key `<graphs.sample>`.
"""

# ╔═╡ 6384a034-32ca-4e9e-8a20-d7b044868cca
sample = config["graphs"]["sample"]

# ╔═╡ d6e65144-247f-4c77-b22d-4e59f2433201
train_trans = joinpath(config["dataset"]["dir"], config["dataset"]["name"],
					   "train", "text")

# ╔═╡ 873dd558-f3fa-494f-8fe5-4e353244a44e
dev_trans = joinpath(config["dataset"]["dir"], config["dataset"]["name"],
					   "dev", "text")

# ╔═╡ 59e08585-dff3-4390-a5e8-eb29ffa5b3c2
md"""
## Output
This notebook will generate:
  1. the mapping between the HMMs' states and the pdf-id.
  2. the alignment (or numerator) graphs, one for each utterance.
  3. the phonotactic language model (or denominator) graph shared by all utterances.
The output will have the following structure:
```
<rootdir>/
+-- <graphs.dir>/
|   +-- <dataset.name>/
|   |   +-- denominator_fsm.jld2
|   |   +-- dev_alignments_fsms.jld2
|   |   +-- pdfid_mapping.jld2
|   |   +-- train_alignments_fsms.jld2
```
The keys `<graphs.dir>` and `<dataset.name>` are taken from the configuration file. If it doesn't exists, the path `<rootdir>/<graphs.dir>/<dataset.name>` will be created.
"""

# ╔═╡ 27f7f58c-b27d-49fe-808d-b6028b8825f4
graphsdir = joinpath(rootdir, config["graphs"]["dir"], config["dataset"]["name"])

# ╔═╡ 11db08d7-b24f-4892-8dc4-ee4ff41eef72
mkpath(graphsdir)

# ╔═╡ 8dad4f95-da50-420a-9a80-847374567180
md"""
## HMMs

For each units of the language, we generate a HMM.
"""

# ╔═╡ 3ce30988-b86f-4405-bf9c-814962a19943
hmmfile = joinpath(graphsdir, "hmms.jld2")

# ╔═╡ d372e6e8-8126-42b6-a88e-f0cc5999ad56
md"""
## Lexicon

Now we convert text input lexicon into a FSM-based representation.
"""

# ╔═╡ 6327eb1c-61a8-4bcf-812c-9c5b05707b50
md"""
## Numerator graphs

We create the alignment graphs that will be the numerator graphs of the LF-MMI
loss function. In the mean time, we estimate a n-gram language model for the
denominator graph.

The parameters are:
  * `sil_sym` the silence word symbol
  * `oov_sym` the special symbol to indicate out-of-vocabulary words
  * `between_silprob` the probability to have silence between word
  * `edge_silprob` the probability to have silence at the edge of an utterance
  * `ngram_order` the order of the n-gram language model
"""

# ╔═╡ 5cb1f14f-521e-4378-8089-9b6a546e0594
sil_sym = config["graphs"]["sil_sym"]

# ╔═╡ b1d8df0f-62ba-40c3-ac04-02a2bff6af97
oov_sym = config["graphs"]["oov_sym"]

# ╔═╡ 1ce9985f-79c6-4de4-9fb1-66f9f70c64f3
between_silprob = config["graphs"]["between_silprob"]

# ╔═╡ 0ea4bd97-676b-4c22-ac98-b67c2c052324
edge_silprob = config["graphs"]["edge_silprob"]

# ╔═╡ a7dbdbc1-8397-426b-be12-973dfe812346
ngram_order = config["graphs"]["ngram_order"]

# ╔═╡ f89f0d32-834e-4ff0-9cff-125e159c31b4
train_alifile = joinpath(graphsdir, "train_alignments_fsms.jld2")

# ╔═╡ b0eadb5b-7601-4181-9c9f-7692c5be404d
dev_alifile = joinpath(graphsdir, "dev_alignments_fsms.jld2")

# ╔═╡ 78d343e0-e66c-4e20-a646-47d4ffa5253f
function sample_ali(sentence, silsym, between_silprob, edge_silprob)
	retval = []
	if rand() < edge_silprob push!(retval, silsym) end
	for (i, word) in enumerate(sentence)
		push!(retval, word)
		if i < length(sentence) && rand() < between_silprob
			push!(retval, silsym)
		end
	end
	if rand() < edge_silprob push!(retval, silsym) end
	retval
end

# ╔═╡ 789c536c-edc5-44f7-9105-27db55b63cbc
md"""
## Denominator graph

We use the n-gram counts from the previous step to build a phonotactic language model FSM. This language model has no back-off.
"""

# ╔═╡ 329f8e51-eb9a-48ba-ac84-ecc4bc1a6dc4
 den_fsmfile = joinpath(graphsdir, "denominator_fsm.jld2")

# ╔═╡ e1a64ab5-72c6-4a50-8e20-a472a1159021
md"""
## Utilities
"""

# ╔═╡ 3d4d32a3-b633-4b54-abae-3c169c0c8c4c
md"""
### Loading data
"""

# ╔═╡ c067a34d-1a5c-4134-96fe-9d97f3f5bf42
function loadunits(file)
	units = []
	open(file, "r") do f
		for line in eachline(f)
			tokens = split(line)
			push!(units, tuple(tokens...))
		end
	end
	units
end

# ╔═╡ bd3301dc-9806-4236-9507-c0290bd5e819
function loadlexicon(file)
	pronuns = Dict()
	open(file, "r") do f
		for line in eachline(f)
			tokens = split(line)
			word, pronun = tokens[1], tokens[2:end]
			plist = get(pronuns, word, [])
			push!(plist, pronun)
			pronuns[word] = plist
		end
	end
	pronuns
end

# ╔═╡ 4dd7c660-34bb-493e-8e6c-e23c45eb2687
begin
	"""
		get_topology(topologies, unit_tuple)

	Get the topology for the given unit. `category` is a sequence
	of category tags. These tags will be use to select the topology.
	"""
	get_topology

	function get_topology(topologies, categories)
		i = 2
		topo = topologies
		while i <= length(categories) && categories[i] ∈ keys(topologies)
			topo = topo[categories[i]]
			i += 1
		end
		topo
	end
end

# ╔═╡ f4a8a935-d6bf-4778-891e-1b4c79d48930
md"""
### Building FSMs
"""

# ╔═╡ c9895639-fabe-4727-9478-e919ab3fbb7e
function makehmm!(pdfid_mapping, unit, topo, pdfid)
	SF = LogSemifield{Float32}
	fsm = VectorFSM{SF}()

	states = Dict()
	for (i, state) in enumerate(topo["states"])
		initweight = SF(log(state["initweight"]))
		finalweight = SF(log(state["finalweight"]))
		s = addstate!(fsm, i; initweight, finalweight)
		states[i] = s
		pdfid_mapping[(unit, i)] = pdfid
		pdfid += 1
	end

	for arc in topo["arcs"]
		src, dest = states[arc["src"]], states[arc["dest"]]
		weight = SF(log(arc["weight"]))
		addarc!(fsm, src, dest, weight)
	end

	fsm, pdfid
end

# ╔═╡ 9bf21a5d-3999-44cb-9556-e625c7126e3e
function makehmms(topologies, unitsfile)
	HMMs = Dict()
	pdfid_mapping = Dict()
	units = loadunits(unitsfile)
	 pdfid = 1
	for unit_tuple in units
		unit, categories = unit_tuple[1], unit_tuple[2:end]
		topo = get_topology(topologies, categories)
		(HMMs[unit], next_pdfid) = makehmm!(pdfid_mapping, unit, topo, pdfid)
		pdfid = next_pdfid
	end
	HMMs, pdfid_mapping
end

# ╔═╡ 532d1672-d160-4103-b3a6-6b951faa221f
begin
	HMMs, pdfid_mapping = makehmms(topologies, unitsfile)
	@info "Created $(length(HMMs)) HMMs."
end

# ╔═╡ c3f3e900-c35a-4386-8ba5-e435387941d2
save(joinpath(graphsdir, "pdfid_mapping.jld2"), "pdfid_mapping", Dict(pdfid_mapping))

# ╔═╡ ab1d6284-5c71-4e46-b5b1-72c75498a7b9
begin
	"""
		LinearFSM(seq)

	Create a linear FSM from a sequence of labels.
	"""
	LinearFSM

	function LinearFSM(seq)
		SF = LogSemifield{Float32}
		fsm = VectorFSM{SF}()

		prev = nothing
		for (i, label) in enumerate(seq)
			initweight = i == 1 ? one(SF) : zero(SF)
			finalweight = i == length(seq) ? one(SF) : zero(SF)
			s = addstate!(fsm, label; initweight, finalweight)
			i > 1 && addarc!(fsm, prev, s)
			prev = s
		end
		fsm
	end
end

# ╔═╡ 30f79da9-8f89-4690-a45e-9d8f0063b6e8
function makelexicon(lexiconfile; sample=false)
	lexicon = Dict()
	pronuns = loadlexicon(lexiconfile)

	# Note: this loop could be parallelized to deal with large lexicon.
	for word in keys(pronuns)
		@debug "Building pronunciation for $word."
		fsms = [LinearFSM(pronun) for pronun in pronuns[word]]
		fsm = sample ? fsms[rand(1:length(fsms))] : union(fsms...)
		lexicon[word] = fsm |> minimize
	end
	lexicon
end

# ╔═╡ dca35409-d9fc-45c9-a826-87f262189211
begin
	@info "Building the lexicon."
	lexicon = makelexicon(lexiconfile; sample)
	@info "Build a pronunciation FSM for $(length(lexicon)) entries."
end

# ╔═╡ 88fff67c-5995-44ca-9a70-d493de00959d
if "HELLO" in keys(lexicon) lexicon["HELLO"] end

# ╔═╡ 0e043217-7087-47f1-910b-937322c78159
begin
	"""
		LanguageModelFSM(ngram_counts)

	Create a language model FSM without back-offs.
	"""
	LanguageModelFSM

	function LanguageModelFSM(ngram_counts)
		SR = LogSemifield{Float32}
		fsm = VectorFSM{SR}()

		iws = Dict()
		fws = Dict()
		for ((src, dest), (iw, w, fw)) in ngram_counts
			if isnothing(src) iws[dest] = iw end
			if fw ≠ zero(w) fws[dest] = fw end
		end

		smap = Dict()
		for ((src, dest), (_, w, _)) in ngram_counts
			dest ∈ keys(smap) && continue
			initweight = get(iws, dest, zero(w))
			finalweight = get(fws, dest, zero(w))
			smap[dest] = addstate!(fsm, dest; initweight, finalweight)
		end

		for ((src, dest), (iw, w, fw)) in ngram_counts
			isnothing(src) && continue
			addarc!(fsm, smap[src], smap[dest], w)
		end

		fsm |> renormalize
		#fsm
	end
end

# ╔═╡ 0dab16d8-98f4-4e19-8663-8e667c1b9d41
function AlignmentFSM(sentence, sil_sym, between_silprob, edge_silprob)
	SF = LogSemifield{Float32}
	fsm = VectorFSM{SF}()

	# Initial optional silence state.
	prev = addstate!(fsm, sil_sym;
					 initweight = SF(log(edge_silprob)),
					 finalweight = zero(SF))

	for (i, word) in enumerate(sentence)
		initweight = i == 1 ? SF(log(1 - edge_silprob)) : zero(SF)
		finalweight = i == length(sentence) ? SF(log(1 - edge_silprob)) : zero(SF)
		s = addstate!(fsm, word; initweight, finalweight)

		if i > 1
			silstate = addstate!(fsm, sil_sym)
			addarc!(fsm, prev, silstate, SF(log(between_silprob)))
			addarc!(fsm, prev, s, SF(log(1-between_silprob)))
			addarc!(fsm, silstate, s)
		else
			addarc!(fsm, prev, s)
		end

		prev = s
	end

	# Final optional silence state.
	silstate = addstate!(fsm, sil_sym;
						 initweight = zero(SF),
						 finalweight = one(SF))
	addarc!(fsm, prev, silstate, SF(log(edge_silprob)))

	fsm
end

# ╔═╡ cac1b00e-4eaa-4057-8cbe-3571a15c8fa9
begin
	"""
		getstates(tup, order)

	Return the state label (source and destination) of the `order`-ngram
	stored in `tup`.
	"""
	getstates

	function getstates(tup, ngram_order)
		if length(tup) == 1
			return (nothing, tup)
		end
		L = length(tup)
		src = tuple(tup[1:min(ngram_order, L)-1]...)
		dest = tuple(tup[max(L-ngram_order+2, 1):end]...)
		src, dest
	end

end

# ╔═╡ d1c050fe-f98d-42a1-8ec5-10196811d262
md"""
### Counting n-grams
"""

# ╔═╡ 8dd44223-0a2d-4eb4-9a15-20654ec1ab76
function enumerate_labels(fsm::AbstractFSM{SF}, state, ngram_order,
						  expand_init = false) where SF
	retval = []
	if expand_init && isinit(state)
		push!(retval, ((state.label[end],), (state.initweight, zero(SF), zero(SF))))
		for i in 2:ngram_order-1
			push!(retval, enumerate_labels(fsm, state, i)...)
		end
	end

	if ngram_order == 1
		push!(retval, ((state.label[end],), (zero(SF), one(SF), state.finalweight)))
		return retval
	end

	for a in arcs(fsm, state)
		aw = a.weight
		for (seqs, (iw, w, fw)) in enumerate_labels(fsm, a.dest, ngram_order-1)
			push!(retval, (tuple(state.label[end], seqs...), (iw, aw * w, fw)))
		end
	end

	retval
end

# ╔═╡ 242f61df-22b1-43ae-b638-870d98aa8dce
function count_ngrams(fsm::AbstractFSM, ngram_order)
	seqs = []
	for state in states(fsm)
		for t in enumerate_labels(fsm, state, ngram_order, true)
			push!(seqs, t)
		end
	end
	seqs
end

# ╔═╡ 63374224-5ad9-4927-a299-389665d061ac
function makenums(fout, trans, lexicon, sil_sym, oov_sym, between_silprob,
				  edge_silprob, ngram_order; njobs, sample=false)

	ch_lines = Channel(100)
	ch_numfsms = Channel(100)
	ch_ngrams = Channel(100)

	Threads.@spawn open(trans, "r") do f
        for line in eachline(f)
		    put!(ch_lines, line)
        end
		close(ch_lines)
	end

	function do_job()
		for line in ch_lines
			tokens = split(line)
			uttid = tokens[1]

			@debug "Creating alignment graph for utterance $uttid."

			# Replace unknown words with the OOV symbol.
			sentence = [word ∈ keys(lexicon) ? word : oov_sym
						for word in tokens[2:end]]

			# Word level FSM.
			trans_wrd = if sample
				seq = sample_ali(sentence, sil_sym, between_silprob, edge_silprob)
				LinearFSM(seq)
			else
				AlignmentFSM(sentence, sil_sym, between_silprob,
							 edge_silprob)
			end

			# Phone level FSM.
			trans_phn = HierarchicalFSM(trans_wrd, lexicon)

			# We count the n-grams that will be used later
			# to build the denominator graph.
			rawcounts = count_ngrams(trans_phn, ngram_order)
			for (tup, ws) in rawcounts
				(src, dest) = getstates(tup, ngram_order)
				put!(ch_ngrams, (src, dest, ws))

			end

			# Alignment FSM.
			ali = HierarchicalFSM(trans_phn, HMMs)

			# Compile the FSM to a matrix format.
			matfsm = MatrixFSM(ali, pdfid_mapping, t -> (t[2], t[3]))

			put!(ch_numfsms, (uttid, matfsm))
		end
		close(ch_numfsms)
		close(ch_ngrams)
	end

	Threads.@spawn do_job()
	# for jobid in 1:njobs
	# 	Threads.@spawn do_job()
	# end

	ngram_counts = Dict()
	Threads.@spawn for (src, dest, ws) in ch_ngrams
		c = get(ngram_counts, (src, dest),
			    (zero(ws[1]), zero(ws[1]), zero(ws[1])))
		ngram_counts[(src,dest)] = c .+  ws
	end

	for (uttid, numfsm) in ch_numfsms
		fout[uttid] = numfsm
	end

	ngram_counts
end

# ╔═╡ 6cfb9bd9-e5e9-4983-be9c-4f96711590e1
ngram_counts = jldopen(train_alifile, "w") do fout
	@info "Generating alignments FSMs for the train set."
	makenums(fout, train_trans, lexicon, sil_sym, oov_sym, between_silprob,
			 edge_silprob, ngram_order; njobs = Threads.nthreads(), sample)
end

# ╔═╡ ee118372-9d16-4de6-84b2-9900036f2f8f
jldopen(den_fsmfile, "w") do f
	@info "Creating the $(ngram_order)-gram phonotactic LM."
	fsm = LanguageModelFSM(ngram_counts)
	fsm = HierarchicalFSM(fsm, HMMs)
	f["fsm"] = MatrixFSM(fsm, pdfid_mapping, t -> (t[end-1], t[end]))
end;

# ╔═╡ b64876f4-db49-493a-bf0e-85096dfe3f88
jldopen(dev_alifile, "w") do fout
	@info "Generating alignments FSMs for the dev set."
	makenums(fout, dev_trans, lexicon, sil_sym, oov_sym, between_silprob,
			 edge_silprob, ngram_order; njobs=Threads.nthreads(), sample)
end

# ╔═╡ Cell order:
# ╟─d0aeac70-e8b3-4765-9483-e1cbb6233793
# ╠═605fd496-1707-4ce8-84f3-dc2dabe13d4e
# ╟─3bf681b3-9949-4a94-824b-b2f7416751bf
# ╟─ad9289fd-3d73-443d-8a6d-79edcf7d8c0a
# ╠═e313e3c6-1ba5-11ec-2657-9574ba676d47
# ╟─b248cbdb-93ac-4e40-92ed-1cbd6f0b8f89
# ╠═fb1c9103-ad50-4eeb-b142-ff74f55170c3
# ╠═d8c0ac69-6753-4eb2-bf35-d6a3744eb261
# ╠═c16e546d-f83a-4094-bf0c-31457bef0ef5
# ╟─ce7861dd-3868-4fc5-b5c8-339ab1a4981f
# ╠═a55eaabf-cb79-4130-affa-a64c3bce1526
# ╠═537679cc-3104-4bbb-aab8-21d80ed03ced
# ╠═43c0ce2c-c28d-4b77-a2ac-75edacc8774f
# ╠═8bd0b6af-9aff-43e7-a06f-934b2525d11a
# ╟─cf5fc8b6-e14d-49b2-9c7d-c8e58e80c118
# ╠═6384a034-32ca-4e9e-8a20-d7b044868cca
# ╠═d6e65144-247f-4c77-b22d-4e59f2433201
# ╠═873dd558-f3fa-494f-8fe5-4e353244a44e
# ╟─59e08585-dff3-4390-a5e8-eb29ffa5b3c2
# ╠═27f7f58c-b27d-49fe-808d-b6028b8825f4
# ╠═11db08d7-b24f-4892-8dc4-ee4ff41eef72
# ╟─8dad4f95-da50-420a-9a80-847374567180
# ╠═3ce30988-b86f-4405-bf9c-814962a19943
# ╠═9bf21a5d-3999-44cb-9556-e625c7126e3e
# ╠═532d1672-d160-4103-b3a6-6b951faa221f
# ╠═c3f3e900-c35a-4386-8ba5-e435387941d2
# ╟─d372e6e8-8126-42b6-a88e-f0cc5999ad56
# ╠═30f79da9-8f89-4690-a45e-9d8f0063b6e8
# ╠═dca35409-d9fc-45c9-a826-87f262189211
# ╠═88fff67c-5995-44ca-9a70-d493de00959d
# ╟─6327eb1c-61a8-4bcf-812c-9c5b05707b50
# ╠═5cb1f14f-521e-4378-8089-9b6a546e0594
# ╠═b1d8df0f-62ba-40c3-ac04-02a2bff6af97
# ╠═1ce9985f-79c6-4de4-9fb1-66f9f70c64f3
# ╠═0ea4bd97-676b-4c22-ac98-b67c2c052324
# ╠═a7dbdbc1-8397-426b-be12-973dfe812346
# ╠═f89f0d32-834e-4ff0-9cff-125e159c31b4
# ╠═b0eadb5b-7601-4181-9c9f-7692c5be404d
# ╠═78d343e0-e66c-4e20-a646-47d4ffa5253f
# ╠═63374224-5ad9-4927-a299-389665d061ac
# ╠═6cfb9bd9-e5e9-4983-be9c-4f96711590e1
# ╠═b64876f4-db49-493a-bf0e-85096dfe3f88
# ╟─789c536c-edc5-44f7-9105-27db55b63cbc
# ╠═329f8e51-eb9a-48ba-ac84-ecc4bc1a6dc4
# ╠═ee118372-9d16-4de6-84b2-9900036f2f8f
# ╟─e1a64ab5-72c6-4a50-8e20-a472a1159021
# ╟─3d4d32a3-b633-4b54-abae-3c169c0c8c4c
# ╠═c067a34d-1a5c-4134-96fe-9d97f3f5bf42
# ╠═bd3301dc-9806-4236-9507-c0290bd5e819
# ╟─4dd7c660-34bb-493e-8e6c-e23c45eb2687
# ╟─f4a8a935-d6bf-4778-891e-1b4c79d48930
# ╠═c9895639-fabe-4727-9478-e919ab3fbb7e
# ╠═ab1d6284-5c71-4e46-b5b1-72c75498a7b9
# ╟─0e043217-7087-47f1-910b-937322c78159
# ╠═0dab16d8-98f4-4e19-8663-8e667c1b9d41
# ╠═cac1b00e-4eaa-4057-8cbe-3571a15c8fa9
# ╟─d1c050fe-f98d-42a1-8ec5-10196811d262
# ╠═8dd44223-0a2d-4eb4-9a15-20654ec1ab76
# ╠═242f61df-22b1-43ae-b638-870d98aa8dce

max_stars_repo_name: lucasondel/SpeechRecipes

max_issues_repo_name: lucasondel/SpeechRecipes

max_forks_repo_name: lucasondel/SpeechRecipes


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    RowTable()

The transform that applies the function `Tables.rowtable` to to the input table.
"""
struct RowTable <: Stateless end

apply(::RowTable, table) = Tables.rowtable(table), table

revert(::RowTable, newtable, cache) = cache


text:
include("../../ccode/mpiops.jl")

mpi_init()

function kappa(x, y)
    return 1+x^2+y^2
end

function rhs(x, y)
    -2*x*(1 - x)*(x^2 + y^2 + 1) + 2*x*(-x*y*(1 - y) + y*(1 - x)*(1 - y)) -
             2*y*(1 - y)*(x^2 + y^2 + 1) + 2*y*(-x*y*(1 - x) + x*(1 - x)*(1 - y))
end

function ufunc(x, y)
    x * (1-x) * y * (1-y)
end

mc = MPIConfig(300)
global_to_local, local_to_global = dofmap(mc)
X, Y = get_xy(mc)
f_local = rhs.(X, Y)
# using Random; Random.seed!(233)
# θ = Variable(fc_init([2,20,20,20,1]))
# θ_shared = mpi_bcast(θ)
# κ_local = (fc([X'[:] Y'[:]], [20,20,20,1], θ_shared) + 5.0)|>squeeze
# κ_local = reshape(κ_local, (mc.n, mc.n))
κ_local = kappa.(X, Y)
u_local = poisson_solver(κ_local, f_local, mc)

sess = Session(); init(sess)
U = run(sess, u_local)

# change_directory("data")
@save "data/$(mpi_size())_$(mpi_rank()).jld2" U

if mpi_size()>1
    mpi_finalize()
end


text:
using ProgressMeter

test = """
    on x=-20..26,y=-36..17,z=-47..7
    on x=-20..33,y=-21..23,z=-26..28
    on x=-22..28,y=-29..23,z=-38..16
    on x=-46..7,y=-6..46,z=-50..-1
    on x=-49..1,y=-3..46,z=-24..28
    on x=2..47,y=-22..22,z=-23..27
    on x=-27..23,y=-28..26,z=-21..29
    on x=-39..5,y=-6..47,z=-3..44
    on x=-30..21,y=-8..43,z=-13..34
    on x=-22..26,y=-27..20,z=-29..19
    off x=-48..-32,y=26..41,z=-47..-37
    on x=-12..35,y=6..50,z=-50..-2
    off x=-48..-32,y=-32..-16,z=-15..-5
    on x=-18..26,y=-33..15,z=-7..46
    off x=-40..-22,y=-38..-28,z=23..41
    on x=-16..35,y=-41..10,z=-47..6
    off x=-32..-23,y=11..30,z=-14..3
    on x=-49..-5,y=-3..45,z=-29..18
    off x=18..30,y=-20..-8,z=-3..13
    on x=-41..9,y=-7..43,z=-33..15
    on x=-54112..-39298,y=-85059..-49293,z=-27449..7877
    on x=967..23432,y=45373..81175,z=27513..53682
    """

function parse_input(input::AbstractString)
    lines = split(input, r"\n"; keepempty = false)
    return parse_line.(lines)
end

const LINE_RE =
    let re(c) = Regex("(?<$(c)min>-?[0-9]+)\\.\\.(?<$(c)max>-?[0-9]+)")
        r"^(?<act>on|off) x=" * re('x') * ",y=" * re('y') * ",z=" * re('z')
    end

function parse_line(line::AbstractString)
    m = match(LINE_RE, line)
    @assert m !== nothing
    ca(c) = let a = parse(Int, m["$(c)min"]),
        b = parse(Int, m["$(c)max"])
        a:b
    end
    return (Block(ca('x'), ca('y'), ca('z')), m["act"] == "on")
end

struct Block
    x::UnitRange{Int}
    y::UnitRange{Int}
    z::UnitRange{Int}
end

Base.isempty(b::Block) = isempty(b.x) || isempty(b.y) || isempty(b.z)
Base.length(b::Block) = length(b.x) * length(b.y) * length(b.z)

function Base.intersect(a::Block, b::Block)::Block
    return Block(a.x ∩ b.x, a.y ∩ b.y, a.z ∩ b.z)
end

function Base.setdiff(a::Block, b::Block)
    return filter(x -> !isempty(x ∩ a) && isempty(x ∩ b), slices(a, b))
end

function Base.union(a::Block, b::Block)
    return filter(x -> !isempty(x ∩ a) || !isempty(x ∩ b), slices(a, b))
end

function slices(a::Block, b::Block)::Vector{Block}
    return [Block(x, y, z) for x in slice(a.x, b.x)
                           for y in slice(a.y, b.y)
                           for z in slice(a.z, b.z)]
end

function slice(a::UnitRange{<: T}, b::UnitRange{<: T}) where T
    trirange(i, j, k, l) = filter(!isempty, [i:j - 1, j:k, k + 1:l])
    a.stop < b.start && return [a, b]
    b.stop < a.start && return [b, a]
    return trirange(sort!([a.start, a.stop, b.start, b.stop])...)
end

@assert slice(-5:-2, 1:10) == [-5:-2, 1:10]
@assert slice(1:10, -5:-2) == [-5:-2, 1:10]
@assert slice(1:10, 5:6) == [1:4, 5:6, 7:10]
@assert slice(1:10, -5:3) == [-5:0, 1:3, 4:10]
@assert slice(1:10, 5:20) == [1:4, 5:10, 11:20]

unionall(as::AbstractVector{<: Block}, b::Block) = unionall(as, [b])
function unionall(as::AbstractVector{<: Block}, bs::AbstractVector{<: Block})
    isempty(bs) && return as
    (b, bs...) = bs
    i = findfirst(a -> !isempty(a ∩ b), as)
    i == nothing && return unionall([as; b], bs)
    a = splice!(as, i)
    return unionall(as, [a ∪ b; bs])
end

q1(procedure) = boot(procedure; bounds = Block(-50:50, -50:50, -50:50))
q2(procedure) = boot(procedure)

function boot(procedure; bounds = nothing)
    blocks = Block[]
    @showprogress for (region, state) in procedure
        if bounds !== nothing
            region = region ∩ bounds
            isempty(region) && continue
        end
        blocks = (state ?
            unionall(blocks, region) :
            vcat(map(block -> setdiff(block, region), blocks)...)) |> simplify!
    end
    return sum(length, blocks)
end

function simplify!(bs::V)::V where {V <: AbstractVector{<: Block}}
    i = 1
    while i < length(bs)
        j = findnext(b -> mergeable(bs[i], b), bs, i + 1)
        while j != nothing
            bs[i] = merge(bs[i], bs[j])
            deleteat!(bs, j)
            j = findnext(b -> mergeable(bs[i], b), bs, j)
        end
        i += 1
    end
    return bs
end

function mergeable(a::Block, b::Block)::Bool
    return a.x == b.x && a.y == b.y && mergeable(a.z, b.z) ||
        a.x == b.x && a.z == b.z && mergeable(a.y, b.y) ||
        a.y == b.y && a.z == b.z && mergeable(a.x, b.x)
end

function mergeable(a::UnitRange, b::UnitRange)::Bool
    return a.stop + 1 == b.start || b.stop + 1 == a.start
end

function Base.merge(a::Block, b::Block)::Block
    return Block(merge(a.x, b.x), merge(a.y, b.y), merge(a.z, b.z))
end

function Base.merge(a::UnitRange, b::UnitRange)::UnitRange
    @assert a == b || mergeable(a, b)
    return min(a.start, b.start):max(a.stop, b.stop)
end

let proc1 = parse_input(test),
    proc2 = parse_input(read("day22-test.in", String))

    @assert q1(proc1) == 590784
    @assert q1(proc2) == 474140
    @assert q2(proc2) == 2758514936282235
end

procedure = parse_input(read("day22.in", String))
println("Q1: ", q1(procedure))
println("Q2: ", q2(procedure))  # approx. 1 min

max_stars_repo_name: dhanak/competitive-coding

max_issues_repo_name: dhanak/competitive-coding

max_forks_repo_name: dhanak/competitive-coding


text:
using Test
using OpenQuantumSystems
using SparseArrays, LinearAlgebra

@testset "interaction picture" begin

    D(op1::Array, op2::Array) = abs(norm(op1 - op2))
    D(x1::StateVector, x2::StateVector) = norm(x2 - x1)
    D(op1::AbstractOperator, op2::AbstractOperator) =
        abs(tracedistance_nh(dense(op1), dense(op2)))
    D(op1::AbstractSuperOperator, op2::AbstractSuperOperator) =
        abs(tracedistance_nh(dense(op1), dense(op2)))

    # TODO: change to macro
    mode1 = Mode(0.2, 1.0)
    Energy = [0.0, 200.0]
    mol1 = Molecule([mode1], 2, Energy)
    mol2 = Molecule([mode1], 2, Energy)
    agg = Aggregate([mol1, mol2])
    aggInds = getIndices(agg)
    aggIndsLen = length(aggInds)
    basis = GenericBasis([aggIndsLen])
    FCFact = getFranckCondonFactors(agg, aggInds)

    Ham_I = getAggHamInteraction(agg, aggInds, FCFact)
    Ham_0 = getAggHamSystemBath(agg, aggInds, FCFact)
    Ham = getAggHamiltonian(agg, aggInds, FCFact)

    Ham_0_lambda, Ham_0_S = eigen(Ham_0.data)
    Ham_0_Sinv = inv(Ham_0_S)
    Ham_0_lambda = diagm(Ham_0_lambda)

    for t in [0.0, 1.0, 2.0]
        U_II_t = getInteractionHamIPicture(Ham_0, Ham_I, t)
        U_op = evolutionOperator(Ham_0, t)
        U_II_t_ref = U_op' * Ham_I * U_op
        @test 1e-14 > D(U_II_t_ref, U_II_t)
        U_II_t = getInteractionHamIPictureA(Ham_0, Ham_I, t)
        @test 1e-14 > D(U_II_t_ref.data, U_II_t)
        U_II_t = getInteractionHamIPictureA(Ham_I.data, Ham_0_lambda, Ham_0_S, Ham_0_Sinv, t)
        @test 1e-14 > D(U_II_t_ref.data, U_II_t)
    end

end # testset