text:
using Test, RelationalData

@test RelationalData.Heading().names == ()

@test RelationalData.Heading().t == Tuple{}

@test RelationalData.Heading(()).names == ()

@test RelationalData.Heading(()).t == Tuple{}

@test RelationalData.Heading((a = Int, b = String)) == RelationalData.Heading((b = String, a = Int))

@test RelationalData.Heading((d = 1, e = "foo")) == RelationalData.Heading((e = "bar", d = 2))

@test RelationalData.Heading((a = Int, b = String)) == RelationalData.Heading((b = "bar", a = 2))

@test RelationalData.Heading((c = Int, a = Int, b = String)) == RelationalData.Heading((b = "bar", a = 2, c = 5))

@test RelationalData.shapeto((b = "bar", a = 2, c = 5), RelationalData.Heading((c = Int, a = Int, b = String))) == (a = 2, b = "bar", c = 5)

@test RelationalData.shapeto((a = 2, c = 500), RelationalData.Heading((c = Int, a = Int8))) == (a = Int8(2), c = 500)

@test_throws InexactError RelationalData.shapeto((a = 500, c = 500), RelationalData.Heading((c = Int, a = Int8)))

@test_throws MethodError RelationalData.shapeto((c = 500), RelationalData.Heading((c = Int, a = Int8)))

@test_throws DomainError RelationalData.shapeto((a = 2, b = 50, c = 500), RelationalData.Heading((c = Int, a = Int8)))

@test_throws ErrorException RelationalData.shapeto((b = 500, c = 500), RelationalData.Heading((c = Int, a = Int)))

@test (typeof(Relation((a=Int64, b=String, c=Float64))).parameters...,) == ((:a, :b, :c), Tuple{Int64, String, Float64})

@test (typeof(Relation(Set([(a=1, b="foo", c=3.14)]))).parameters...,) == ((:a, :b, :c), Tuple{Int64, String, Float64})

@test (typeof(Relation(Set([(a=Int8(1), b="foo"), (a=Int8(3), b="foo")]))).parameters...,) == ((:a, :b), Tuple{Int8, String})

@test (typeof(Relation((a=Int8(1), b="foo"), (a=Int8(3), b="foo"))).parameters...,) == ((:a, :b), Tuple{Int8, String})

@test Relation((a=1, b="foo"), (a=3, b="bar")).body == Set([(a=1, b="foo"), (a=3, b="bar")])

@test Relation((a=Int64, b=String), (1, "foo"), (3, "bar")).body == Set([(a=1, b="foo"), (a=3, b="bar")])

@test Relation((a=Int8, b=String), (1, "foo"), (3, "bar")).body == Set([(a=Int8(1), b="foo"), (a=Int8(3), b="bar")])

@test Relation((b=String, a=Int64), ("foo", 1), ("bar", 3)).body == Set([(a=1, b="foo"), (a=3, b="bar")])

@test Relation((a=Int64, b=String), [(1, "foo"), (3, "bar")]).body == Set([(a=1, b="foo"), (a=3, b="bar")])

@test Relation((a=Int8, b=String), [(1, "foo"), (3, "bar")]).body == Set([(a=Int8(1), b="foo"), (a=Int8(3), b="bar")])

@test Relation((b=String, a=Int64), [("foo", 1), ("bar", 3)]).body == Set([(a=1, b="foo"), (a=3, b="bar")])

@test (typeof(Relation()).parameters...,) == ((), Tuple{})

@test length(Relation().body) == 0

@test (typeof(Relation(())).parameters...,) == ((), Tuple{})

@test length(Relation(()).body) == 1

@test (typeof(Relation([])).parameters...,) == ((), Tuple{})

@test length(Relation([]).body) == 0

@test (typeof(Relation([()])).parameters...,) == ((), Tuple{})

@test length(Relation([()]).body) == 1

@test (typeof(Relation([(a=1, b="foo"), (a=3.14, b="bar"), (b="qux", a=6)])).parameters...,) == ((:a, :b), Tuple{Float64, String})

@test [t.a for t in Relation((a=1, b=2), (a=5, b=9))] == [1,5]

@test filter(t -> t.a < 4, Relation((a=1, b=2), (a=5, b=9))) == Relation((a=1, b=2))

@test Relation((a=Int8, b=Int8)) != Relation((a=Int64, b=Int64))

@test restrict(Relation((a=1, b=2), (a=5, b=9)), t -> t.a < 4) == Relation((a=1, b=2))

@test extend(Relation((a=1, b=2), (a=5, b=9)), :sum, t -> t.a+t.b) == Relation((a=1, b=2, sum=3), (a=5, b=9, sum=14))

@test rename(Relation((a=1, b=2), (a=5, b=9)), :a=>:x) == Relation((x=1, b=2), (x=5, b=9))

@test rename(Relation((a=1, b=2), (a=5, b=9)), :a=>:y, :b=>:x) == Relation((y=1, x=2), (y=5, x=9))

@test project(Relation((a=1, b=2, c="foo"), (a=1, b=3, c="foo"), (a=5, b=9, c="bar")), :a, :c) == Relation((a=1, c="foo"), (a=5, c="bar"))

@test_throws ErrorException project(Relation((a=1, b=2, c="foo"), (a=1, b=3, c="foo"), (a=5, b=9, c="bar")), :a, :z)

@test naturaljoin(Relation((a=1, b=2), (a=5, b=9)), Relation((a=1, c="foo"), (a=1, c="bar"), (a=2, c="foo"))) == Relation((a=1, b=2, c="foo"), (a=1, b=2, c="bar"))

@test naturaljoin(Relation((a=1, b=2)), Relation((c="foo",), (c="bar",))) == Relation((a=1, b=2, c="foo"), (a=1, b=2, c="bar"))

@test Relation((a=1, b=2), (a=5, b=9)) ⨝ Relation((a=1, c="foo"), (a=1, c="bar"), (a=2, c="foo")) == Relation((a=1, b=2, c="foo"), (a=1, b=2, c="bar"))

@test naturaljoin(Relation((a=1,), (a=2,)), Relation((c="foo",), (c="bar",)), Relation((b=3.14,))) == Relation((a=1, c="foo", b=3.14), (a=1, c="bar", b=3.14), (a=2, c="foo", b=3.14), (a=2, c="bar", b=3.14), )

@test union(Relation((c="bar", a=1)), Relation((a=4, c="foo"))) == Relation((a=1, c="bar"), (a=4, c="foo"))

@test union(Relation((c="bar", a=1)), Relation((c="bar", a=1))) == Relation((c="bar", a=1))

@test Relation((c="bar", a=1)) ∪ Relation((a=4, c="foo")) == Relation((a=1, c="bar"), (a=4, c="foo"))

@test union(Relation((c="bar", a=1)), Relation((a=4, c="foo")), Relation((a=4, c="foo"))) == Relation((a=1, c="bar"), (a=4, c="foo"))

max_stars_repo_name: tobega/RelationalData

max_issues_repo_name: tobega/RelationalData

max_forks_repo_name: tobega/RelationalData


text:
#  MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
#  Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved.
#  Released under the modified BSD license. See COPYING.md for more details.

using Cbc
using Clp
using JuMP
using MathOptInterface
using TimerOutputs
using SparseArrays
const MOI = MathOptInterface

import JuMP: value

Base.@kwdef mutable struct JuMPSolverData
    optimizer_factory::Any
    basis_status::Dict{ConstraintRef,MOI.BasisStatusCode} = Dict()
    bin_vars::Vector{JuMP.VariableRef} = []
    int_vars::Vector{JuMP.VariableRef} = []
    cb_data::Any = nothing
    cname_to_constr::Dict{String,JuMP.ConstraintRef} = Dict()
    dual_values::Dict{JuMP.ConstraintRef,Float64} = Dict()
    instance::Union{Nothing,PyObject} = nothing
    model::Union{Nothing,JuMP.Model} = nothing
    reduced_costs::Vector{Float64} = []
    sensitivity_report::Any = nothing
    solution::Dict{JuMP.VariableRef,Float64} = Dict()
    varname_to_var::Dict{String,VariableRef} = Dict()
    x::Vector{Float64} = Float64[]
end

"""
    _optimize_and_capture_output!(model; tee=tee)

Optimizes a given JuMP model while capturing the solver log, then returns that log.
If tee=true, prints the solver log to the standard output as the optimization takes place.
"""
function _optimize_and_capture_output!(model; tee::Bool = false)
    logname = tempname()
    logfile = open(logname, "w")
    redirect_stdout(logfile) do
        JuMP.optimize!(model)
        Base.Libc.flush_cstdio()
    end
    close(logfile)
    log = String(read(logname))
    rm(logname)
    if tee
        println(log)
        flush(stdout)
        Base.Libc.flush_cstdio()
    end
    return log
end

function _update_solution!(data::JuMPSolverData)
    vars = JuMP.all_variables(data.model)
    data.solution = Dict(var => JuMP.value(var) for var in vars)
    data.x = JuMP.value.(vars)

    if has_duals(data.model)
        data.reduced_costs = []
        data.basis_status = Dict()

        for var in vars
            rc = 0.0
            if has_upper_bound(var)
                rc += shadow_price(UpperBoundRef(var))
            end
            if has_lower_bound(var)
                # FIXME: Remove negative sign
                rc -= shadow_price(LowerBoundRef(var))
            end
            if is_fixed(var)
                rc += shadow_price(FixRef(var))
            end
            push!(data.reduced_costs, rc)
        end

        try
            data.sensitivity_report = lp_sensitivity_report(data.model)
        catch
            @warn "Sensitivity analysis is unavailable; ignoring" maxlog = 1
        end

        basis_status_supported = true
        data.dual_values = Dict()
        for (ftype, stype) in JuMP.list_of_constraint_types(data.model)
            for constr in JuMP.all_constraints(data.model, ftype, stype)
                # Dual values (FIXME: Remove negative sign)
                data.dual_values[constr] = -JuMP.dual(constr)

                # Basis status
                if basis_status_supported
                    try
                        data.basis_status[constr] =
                            MOI.get(data.model, MOI.ConstraintBasisStatus(), constr)
                    catch
                        @warn "Basis status is unavailable; ignoring" maxlog = 1
                        basis_status_supported = false
                        data.basis_status = Dict()
                    end
                end
            end
        end

    else
        data.reduced_costs = []
        data.dual_values = Dict()
        data.sensitivity_report = nothing
        data.basis_status = Dict()
    end
end

function add_constraints(
    data::JuMPSolverData;
    lhs::SparseMatrixCSC,
    rhs::Vector{Float64},
    senses::Vector{String},
    names::Vector{String},
)::Nothing
    lhs_exprs = lhs * JuMP.all_variables(data.model)
    for (i, lhs_expr) in enumerate(lhs_exprs)
        if senses[i] == ">"
            constr = @constraint(data.model, lhs_expr >= rhs[i])
        elseif senses[i] == "<"
            constr = @constraint(data.model, lhs_expr <= rhs[i])
        elseif senses[i] == "="
            constr = @constraint(data.model, lhs_expr == rhs[i])
        else
            error("unknown sense: $sense")
        end
        set_name(constr, names[i])
        data.cname_to_constr[names[i]] = constr
    end
    data.solution = Dict()
    data.x = Float64[]
    return
end

function are_constraints_satisfied(
    data::JuMPSolverData;
    lhs::SparseMatrixCSC,
    rhs::Vector{Float64},
    senses::Vector{String},
    tol::Float64 = 1e-5,
)::Vector{Bool}
    result = Bool[]
    lhs_value = lhs * data.x
    for (i, sense) in enumerate(senses)
        sense = senses[i]
        if sense == "<"
            push!(result, lhs_value[i] <= rhs[i] + tol)
        elseif sense == ">"
            push!(result, lhs_value[i] >= rhs[i] - tol)
        elseif sense == "<"
            push!(result, abs(rhs[i] - lhs_value[i]) <= tol)
        else
            error("unknown sense: $sense")
        end
    end
    return result
end

function build_test_instance_knapsack()
    weights = [23.0, 26.0, 20.0, 18.0]
    prices = [505.0, 352.0, 458.0, 220.0]
    capacity = 67.0

    model = Model()
    n = length(weights)
    @variable(model, x[0:n-1], Bin)
    @variable(model, z, lower_bound = 0.0, upper_bound = capacity)
    @objective(model, Max, sum(x[i-1] * prices[i] for i = 1:n))
    @constraint(model, eq_capacity, sum(x[i-1] * weights[i] for i = 1:n) - z == 0)

    return JuMPInstance(model).py
end

function build_test_instance_infeasible()
    model = Model()
    @variable(model, x, Bin)
    @objective(model, Max, x)
    @constraint(model, x >= 2)
    return JuMPInstance(model).py
end

function remove_constraints(data::JuMPSolverData, names::Vector{String})::Nothing
    for name in names
        constr = data.cname_to_constr[name]
        delete(data.model, constr)
        delete!(data.cname_to_constr, name)
    end
    return
end

function solve(
    data::JuMPSolverData;
    tee::Bool = false,
    iteration_cb = nothing,
    lazy_cb = nothing,
)
    model = data.model
    wallclock_time = 0
    log = ""

    if lazy_cb !== nothing
        function lazy_cb_wrapper(cb_data)
            data.cb_data = cb_data
            lazy_cb(nothing, nothing)
            data.cb_data = nothing
        end
        MOI.set(model, MOI.LazyConstraintCallback(), lazy_cb_wrapper)
    end

    while true
        wallclock_time += @elapsed begin
            log *= _optimize_and_capture_output!(model, tee = tee)
        end
        if is_infeasible(data)
            break
        end
        if iteration_cb !== nothing
            iteration_cb() || break
        else
            break
        end
    end

    if is_infeasible(data)
        data.solution = Dict()
        data.x = Float64[]
        primal_bound = nothing
        dual_bound = nothing
    else
        _update_solution!(data)
        primal_bound = JuMP.objective_value(model)
        dual_bound = JuMP.objective_bound(model)
    end
    if JuMP.objective_sense(model) == MOI.MIN_SENSE
        sense = "min"
        lower_bound = dual_bound
        upper_bound = primal_bound
    else
        sense = "max"
        lower_bound = primal_bound
        upper_bound = dual_bound
    end
    return miplearn.solvers.internal.MIPSolveStats(
        mip_lower_bound = lower_bound,
        mip_upper_bound = upper_bound,
        mip_sense = sense,
        mip_wallclock_time = wallclock_time,
        mip_nodes = 1,
        mip_log = log,
        mip_warm_start_value = nothing,
    )
end

function solve_lp(data::JuMPSolverData; tee::Bool = false)
    for var in data.bin_vars
        ~is_fixed(var) || continue
        unset_binary(var)
        set_upper_bound(var, 1.0)
        set_lower_bound(var, 0.0)
    end
    for var in data.int_vars
        ~is_fixed(var) || continue
        unset_integer(var)
    end
    # If the optimizer is Cbc, we need to replace it by Clp,
    # otherwise dual values are not available.
    # https://github.com/jump-dev/Cbc.jl/issues/50
    is_cbc = (data.optimizer_factory == Cbc.Optimizer)
    if is_cbc
        set_optimizer(data.model, Clp.Optimizer)
    end
    wallclock_time = @elapsed begin
        log = _optimize_and_capture_output!(data.model, tee = tee)
    end
    if is_infeasible(data)
        data.solution = Dict()
        obj_value = nothing
    else
        _update_solution!(data)
        obj_value = objective_value(data.model)
    end
    if is_cbc
        set_optimizer(data.model, data.optimizer_factory)
    end
    for var in data.bin_vars
        ~is_fixed(var) || continue
        set_binary(var)
    end
    for var in data.int_vars
        ~is_fixed(var) || continue
        set_integer(var)
    end
    return miplearn.solvers.internal.LPSolveStats(
        lp_value = obj_value,
        lp_log = log,
        lp_wallclock_time = wallclock_time,
    )
end

function set_instance!(
    data::JuMPSolverData,
    instance;
    model::Union{Nothing,JuMP.Model},
)::Nothing
    data.instance = instance
    if model === nothing
        model = instance.to_model()
    end
    data.model = model
    data.bin_vars = [var for var in JuMP.all_variables(model) if JuMP.is_binary(var)]
    data.int_vars = [var for var in JuMP.all_variables(model) if JuMP.is_integer(var)]
    data.varname_to_var = Dict(JuMP.name(var) => var for var in JuMP.all_variables(model))
    JuMP.set_optimizer(model, data.optimizer_factory)
    data.cname_to_constr = Dict()
    for (ftype, stype) in JuMP.list_of_constraint_types(model)
        for constr in JuMP.all_constraints(model, ftype, stype)
            name = JuMP.name(constr)
            length(name) > 0 || continue
            data.cname_to_constr[name] = constr
        end
    end
    return
end

function fix!(data::JuMPSolverData, solution)
    for (varname, value) in solution
        value !== nothing || continue
        var = data.varname_to_var[varname]
        JuMP.fix(var, value, force = true)
    end
end

function set_warm_start!(data::JuMPSolverData, solution)
    for (varname, value) in solution
        value !== nothing || continue
        var = data.varname_to_var[varname]
        JuMP.set_start_value(var, value)
    end
end

function is_infeasible(data::JuMPSolverData)
    return JuMP.termination_status(data.model) in
           [MOI.INFEASIBLE, MOI.INFEASIBLE_OR_UNBOUNDED]
end

function get_variables(data::JuMPSolverData; with_static::Bool, with_sa::Bool)
    vars = JuMP.all_variables(data.model)
    lb, ub, types = nothing, nothing, nothing
    sa_obj_down, sa_obj_up = nothing, nothing
    sa_lb_down, sa_lb_up = nothing, nothing
    sa_ub_down, sa_ub_up = nothing, nothing
    basis_status = nothing
    values, rc = nothing, nothing

    # Variable names
    names = JuMP.name.(vars)

    # Primal values
    if !isempty(data.solution)
        values = [data.solution[v] for v in vars]
    end

    # Objective function coefficients
    obj = objective_function(data.model)
    obj_coeffs = [v ∈ keys(obj.terms) ? obj.terms[v] : 0.0 for v in vars]

    if with_static
        # Lower bounds
        lb = [
            JuMP.is_binary(v) ? 0.0 : JuMP.has_lower_bound(v) ? JuMP.lower_bound(v) : -Inf for v in vars
        ]

        # Upper bounds
        ub = [
            JuMP.is_binary(v) ? 1.0 : JuMP.has_upper_bound(v) ? JuMP.upper_bound(v) : Inf for v in vars
        ]

        # Variable types
        types = [JuMP.is_binary(v) ? "B" : JuMP.is_integer(v) ? "I" : "C" for v in vars]
    end

    # Sensitivity analysis
    if data.sensitivity_report !== nothing
        sa_obj_down, sa_obj_up = Float64[], Float64[]
        sa_lb_down, sa_lb_up = Float64[], Float64[]
        sa_ub_down, sa_ub_up = Float64[], Float64[]

        for (i, v) in enumerate(vars)
            # Objective function
            (delta_down, delta_up) = data.sensitivity_report[v]
            push!(sa_obj_down, delta_down + obj_coeffs[i])
            push!(sa_obj_up, delta_up + obj_coeffs[i])

            # Lower bound
            if has_lower_bound(v)
                constr = LowerBoundRef(v)
                (delta_down, delta_up) = data.sensitivity_report[constr]
                push!(sa_lb_down, lower_bound(v) + delta_down)
                push!(sa_lb_up, lower_bound(v) + delta_up)
            else
                push!(sa_lb_down, -Inf)
                push!(sa_lb_up, -Inf)
            end

            # Upper bound
            if has_upper_bound(v)
                constr = JuMP.UpperBoundRef(v)
                (delta_down, delta_up) = data.sensitivity_report[constr]
                push!(sa_ub_down, upper_bound(v) + delta_down)
                push!(sa_ub_up, upper_bound(v) + delta_up)
            else
                push!(sa_ub_down, Inf)
                push!(sa_ub_up, Inf)
            end
        end
    end

    # Basis status
    if !isempty(data.basis_status)
        basis_status = []
        for v in vars
            basis_status_v = "B"
            if has_lower_bound(v)
                constr = LowerBoundRef(v)
                if data.basis_status[constr] == MOI.NONBASIC
                    basis_status_v = "L"
                end
            end
            if has_upper_bound(v)
                constr = UpperBoundRef(v)
                if data.basis_status[constr] == MOI.NONBASIC
                    basis_status_v = "U"
                end
            end
            push!(basis_status, basis_status_v)
        end
    end

    rc = isempty(data.reduced_costs) ? nothing : data.reduced_costs

    vf = miplearn.solvers.internal.Variables(
        basis_status = to_str_array(basis_status),
        lower_bounds = lb,
        names = to_str_array(names),
        obj_coeffs = with_static ? obj_coeffs : nothing,
        reduced_costs = rc,
        sa_lb_down = with_sa ? sa_lb_down : nothing,
        sa_lb_up = with_sa ? sa_lb_up : nothing,
        sa_obj_down = with_sa ? sa_obj_down : nothing,
        sa_obj_up = with_sa ? sa_obj_up : nothing,
        sa_ub_down = with_sa ? sa_ub_down : nothing,
        sa_ub_up = with_sa ? sa_ub_up : nothing,
        types = to_str_array(types),
        upper_bounds = ub,
        values = values,
    )
    return vf
end

function get_constraints(
    data::JuMPSolverData;
    with_static::Bool,
    with_sa::Bool,
    with_lhs::Bool,
)
    names = String[]
    senses, rhs = String[], Float64[]
    lhs_rows, lhs_cols, lhs_values = Int[], Int[], Float64[]
    dual_values, slacks = nothing, nothing
    basis_status = nothing
    sa_rhs_up, sa_rhs_down = nothing, nothing

    if !isempty(data.dual_values)
        dual_values = Float64[]
    end
    if !isempty(data.basis_status)
        basis_status = []
    end
    if data.sensitivity_report !== nothing
        sa_rhs_up, sa_rhs_down = Float64[], Float64[]
    end

    constr_index = 1
    for (ftype, stype) in JuMP.list_of_constraint_types(data.model)
        for constr in JuMP.all_constraints(data.model, ftype, stype)
            cset = MOI.get(constr.model.moi_backend, MOI.ConstraintSet(), constr.index)
            cf = MOI.get(constr.model.moi_backend, MOI.ConstraintFunction(), constr.index)

            # Names
            name = JuMP.name(constr)
            length(name) > 0 || continue
            push!(names, name)

            # LHS, RHS and sense
            if ftype == VariableRef
                # nop
            elseif ftype == AffExpr
                if stype == MOI.EqualTo{Float64}
                    rhs_c = cset.value
                    push!(senses, "=")
                elseif stype == MOI.LessThan{Float64}
                    rhs_c = cset.upper
                    push!(senses, "<")
                elseif stype == MOI.GreaterThan{Float64}
                    rhs_c = cset.lower
                    push!(senses, ">")
                else
                    error("Unsupported set: $stype")
                end
                push!(rhs, rhs_c)
                for term in cf.terms
                    push!(lhs_cols, term.variable_index.value)
                    push!(lhs_rows, constr_index)
                    push!(lhs_values, term.coefficient)
                end
                constr_index += 1
            else
                error("Unsupported constraint type: ($ftype, $stype)")
            end

            # Dual values
            if !isempty(data.dual_values)
                push!(dual_values, data.dual_values[constr])
            end

            # Basis status
            if !isempty(data.basis_status)
                b = data.basis_status[constr]
                if b == MOI.NONBASIC
                    push!(basis_status, "N")
                elseif b == MOI.BASIC
                    push!(basis_status, "B")
                else
                    error("Unknown basis status: $b")
                end
            end

            # Sensitivity analysis
            if data.sensitivity_report !== nothing
                (delta_down, delta_up) = data.sensitivity_report[constr]
                push!(sa_rhs_down, rhs_c + delta_down)
                push!(sa_rhs_up, rhs_c + delta_up)
            end
        end
    end

    lhs =
        sparse(lhs_rows, lhs_cols, lhs_values, length(rhs), JuMP.num_variables(data.model))
    if !isempty(data.x)
        lhs_value = lhs * data.x
        slacks = abs.(lhs_value - rhs)
    end

    return miplearn.solvers.internal.Constraints(
        basis_status = to_str_array(basis_status),
        dual_values = dual_values,
        lhs = (with_static && with_lhs) ? sparse(lhs_rows, lhs_cols, lhs_values) : nothing,
        names = to_str_array(names),
        rhs = with_static ? rhs : nothing,
        sa_rhs_down = with_sa ? sa_rhs_down : nothing,
        sa_rhs_up = with_sa ? sa_rhs_up : nothing,
        senses = with_static ? to_str_array(senses) : nothing,
        slacks = slacks,
    )
end

function __init_JuMPSolver__()
    @pydef mutable struct Class <: miplearn.solvers.internal.InternalSolver
        function __init__(self, optimizer_factory)
            self.data = JuMPSolverData(optimizer_factory = optimizer_factory)
        end

        function add_constraints(self, cf)
            add_constraints(
                self.data,
                lhs = convert(SparseMatrixCSC, cf.lhs),
                rhs = cf.rhs,
                senses = from_str_array(cf.senses),
                names = from_str_array(cf.names),
            )
        end

        function are_constraints_satisfied(self, cf; tol = 1e-5)
            return are_constraints_satisfied(
                self.data,
                lhs = convert(SparseMatrixCSC, cf.lhs),
                rhs = cf.rhs,
                senses = from_str_array(cf.senses),
                tol = tol,
            )
        end

        build_test_instance_infeasible(self) = build_test_instance_infeasible()

        build_test_instance_knapsack(self) = build_test_instance_knapsack()

        clone(self) = JuMPSolver(self.data.optimizer_factory)

        fix(self, solution) = fix!(self.data, solution)

        get_solution(self) = isempty(self.data.solution) ? nothing : self.data.solution

        get_constraints(self; with_static = true, with_sa = true, with_lhs = true) =
            get_constraints(
                self.data,
                with_static = with_static,
                with_sa = with_sa,
                with_lhs = with_lhs,
            )

        function get_constraint_attrs(self)
            attrs = [
                "categories",
                "dual_values",
                "lazy",
                "lhs",
                "names",
                "rhs",
                "senses",
                "user_features",
                "slacks",
            ]
            if repr(self.data.optimizer_factory) in ["Gurobi.Optimizer"]
                append!(attrs, ["basis_status", "sa_rhs_down", "sa_rhs_up"])
            end
            return attrs
        end

        get_variables(self; with_static = true, with_sa = true) =
            get_variables(self.data; with_static = with_static, with_sa = with_sa)

        function get_variable_attrs(self)
            attrs = [
                "names",
                "categories",
                "lower_bounds",
                "obj_coeffs",
                "reduced_costs",
                "types",
                "upper_bounds",
                "user_features",
                "values",
            ]
            if repr(self.data.optimizer_factory) in ["Gurobi.Optimizer"]
                append!(
                    attrs,
                    [
                        "basis_status",
                        "sa_obj_down",
                        "sa_obj_up",
                        "sa_lb_down",
                        "sa_lb_up",
                        "sa_ub_down",
                        "sa_ub_up",
                    ],
                )
            end
            return attrs
        end

        is_infeasible(self) = is_infeasible(self.data)

        remove_constraints(self, names) = remove_constraints(self.data, [n for n in names])

        set_instance(self, instance, model = nothing) =
            set_instance!(self.data, instance, model = model)

        set_warm_start(self, solution) = set_warm_start!(self.data, solution)

        solve(
            self;
            tee = false,
            iteration_cb = nothing,
            lazy_cb = nothing,
            user_cut_cb = nothing,
        ) = solve(self.data, tee = tee, iteration_cb = iteration_cb, lazy_cb = lazy_cb)

        solve_lp(self; tee = false) = solve_lp(self.data, tee = tee)
    end
    copy!(JuMPSolver, Class)
end

function value(solver::JuMPSolverData, var::VariableRef)
    if solver.cb_data !== nothing
        return JuMP.callback_value(solver.cb_data, var)
    else
        return JuMP.value(var)
    end
end

function submit(solver::JuMPSolverData, con::AbstractConstraint, name::String = "")
    if solver.cb_data !== nothing
        MOI.submit(solver.model, MOI.LazyConstraint(solver.cb_data), con)
    else
        JuMP.add_constraint(solver.model, con, name)
    end
end

export JuMPSolver, submit

ext: jl


text:
using Hashpipe_jll

const depsfile = joinpath(@__DIR__, "deps.jl")

open(depsfile, "w") do f
    # Path to Hashpipe_jll build hashpipe shared object files:
    deps_string = "const libhashpipe = \"$(Hashpipe_jll.libhashpipe_path)\"\n" # libhashpipe.so
    deps_string *= "const libhashpipestatus = \"$(normpath(joinpath(Hashpipe_jll.libhashpipe_path, "../libhashpipestatus.so")))\"\n"
    print(f, deps_string)
end


text:
module test_nodebooks

using Conda
using DomainSets
using BasisFunctions
using FrameFun
using Plots

function delimit(s::AbstractString)
    println()
    println("############")
    println("# ",s)
    println("############")
end

jupyter = nothing #Sys.which("jupyter")
if jupyter == nothing
    global jupyter = Sys.which(Conda.PYTHONDIR * "/jupyter")
    if jupyter == nothing
        try
            @info "Trying to install jupyter using the Conda package"
            Conda.add("jupyter")
            global jupyter = Sys.which(Conda.PYTHONDIR * "/jupyter")
        catch error
            @warn Conda could not add jupyter
            bt = catch_backtrace()
            msg = sprint(showerror, error, bt)
            @info msg
        end
    end
end

if jupyter == nothing
    @warn "Jupyter not installed and no internet connection to install, abandoning notebook tests."
else
    FRAMEFUNSRC = pathof(FrameFun)
    FRAMEFUNPATH = splitdir(splitdir(FRAMEFUNSRC)[1])[1]
    delimit("Notebooks")

    run(`$jupyter nbconvert --to script $FRAMEFUNPATH'/examples/*.ipynb'`)
    a = readdir("$FRAMEFUNPATH/examples/")
    scripts = a[occursin.(".jl",a)]
    try
        for LINE in scripts
            println("Run $(LINE)")
            include("$FRAMEFUNPATH/examples/"*LINE)
        end
    finally
        for LINE in scripts
            rm("$FRAMEFUNPATH/examples/"*LINE)
        end
    end
end
end


text:
abstract type Injection <: Device end
abstract type WaterDemand <: Injection end
abstract type Tank <: Injection end

## potential todo:  add Emitter type as described in EPANET manual
# abstract type Emitter <: Devise end


text:
export UnconditionalMusicTransformer, MelodyConditionedMusicTransformer

using Flux: unsqueeze, @functor
using MacroTools: @forward

using Transformers.Basic: AbstractTransformer, Embed
using Transformers.Stacks

abstract type MusicTransformerModel <: AbstractTransformer end

struct UnconditionalMusicTransformer{E<:Stack, B<:Stack} <: MusicTransformerModel
    embedding::E
    body::B
end

@functor UnconditionalMusicTransformer

function UnconditionalMusicTransformer(size::Int, heads::Int, ps::Int, layers::Int)
    rem(size, heads) != 0 && error("size not divisible by heads")
    UnconditionalMusicTransformer(size, heads, div(size, heads), ps, layers)
end

function UnconditionalMusicTransformer(size::Int, head::Int, hs::Int, ps::Int, layers::Int)
    UnconditionalMusicTransformer(
        Stack(
            @nntopo_str("indices => e => pe:(e, pe) => input"),
            Embed(size, 310, scale=sqrt(size)),
            PositionEncoding(size, 4096),
            # Perform bottom transformation and add position embedding
            (e, pe) -> (e .+ pe),
        ),
        Stack(
            @nntopo_str("input => $layers => norm => logits"),
            [
                MusicTransformerBlock(size, head, hs, ps; future=false, pdrop=0)
                for i = 1:layers
            ]...,
            LayerNorm(size),
            Dense(size, 310),
        )
    )
end

function prepare_decoder_inputs(mt::UnconditionalMusicTransformer, indices::T) where T
    if isempty(indices)
        vocab_size = size(mt.embedding[1], 1)
        inputs = zeros(Float32, (vocab_size, 1))
    else
        inputs = mt.embedding(indices)
    end

    inputs
end

function (mt::UnconditionalMusicTransformer)(indices::T) where T
    inputs = prepare_decoder_inputs(mt, indices)
    mt.body(inputs)
end

function Base.show(io::IO, mt::UnconditionalMusicTransformer)
    hs = div(size(mt.body[1].mh.iqproj.W)[1], mt.body[1].mh.head)
    h, ps = size(mt.body[1].pw.dout.W)
    num_layers = length(mt.body) - 2 # Ignore output norm and softmax layers
    print(io, "UnconditionalMusicTransformer(")
    print(io, "layers=$num_layers, ")
    print(io, "head=$(mt.body[1].mh.head), ")
    print(io, "head_size=$(hs), ")
    print(io, "pwffn_size=$(ps), ")
    print(io, "size=$(h))")
end

struct MelodyConditionedMusicTransformer{EE<:Stack, E<:Stack, DE<:Stack, D<:Stack} <: MusicTransformerModel
    encoder_embedding::EE
    encoder::E
    decoder_embedding::DE
    decoder::D
end

@functor MelodyConditionedMusicTransformer

function MelodyConditionedMusicTransformer(size::Int, heads::Int, ps::Int,
                                     encoder_layers::Int, decoder_layers::Int)

    rem(size, heads) != 0 && error("size not divisible by heads")
    MelodyConditionedMusicTransformer(size, heads, div(size, heads), ps, encoder_layers, decoder_layers)
end

function MelodyConditionedMusicTransformer(size::Int, head::Int, hs::Int, ps::Int,
                                           encoder_layers::Int, decoder_layers::Int)

    MelodyConditionedMusicTransformer(
        Stack(
            @nntopo_str(
                "(indices, tid):indices => e:tid => te:(e, te) => out => pe:(out, pe) => enc_input"),
            Embed(size, 92, scale=sqrt(size)),
            Embed(size, 32),
            # Add target space embedding
            (e, te) -> Float32.(e .+ te),
            PositionEncoding(size, 4096),
            # Add position embedding
            (e, pe) -> (e .+ pe)
        ),
        Stack(
            @nntopo_str("input => $encoder_layers => output"),
            [
                MusicTransformerBlock(size, head, hs, ps; future=true, pdrop=0)
                for i = 1:encoder_layers
            ]...,
            LayerNorm(size),
        ),
        Stack(
            @nntopo_str("indices => e => pe:(e, pe) => dec_input"),
            Embed(size, 310, scale=sqrt(size)),
            PositionEncoding(size, 4096),
            # Add position embedding
            (e, pe) -> (e .+ pe)
        ),
        Stack(
            @nntopo_str(
                "((input, m) => input:(input, m)) => $decoder_layers:input => norm => output"),
            [
                MusicTransformerDecoder(size, head, hs, ps; pdrop=0)
                for i = 1:decoder_layers
            ]...,
            LayerNorm(size),
            Dense(size, 310), # softmax embedding layer
        )
    )
end

function prepare_decoder_inputs(mt::MelodyConditionedMusicTransformer, targets::Array{Int, N}) where N
    if isempty(targets)
        vocab_size = size(mt.decoder_embedding[1], 1)
        decoder_inputs = zeros(Float32, (vocab_size, 1))
    else
        decoder_inputs = mt.decoder_embedding(targets)
    end

    decoder_inputs
end

function (mt::MelodyConditionedMusicTransformer)(inputs::Array{Int, N},
                                                 targets::Array{Int, M}) where {N, M}
    target_space_id = [1]
    # Call the encoder layers
    encoder_inputs = mt.encoder_embedding(inputs, target_space_id)
    encoder_context = mt.encoder(encoder_inputs)
    # Call the decoder layers
    logits = mt(targets, encoder_context)

    logits, encoder_context
end

function (mt::MelodyConditionedMusicTransformer)(targets::Array{Int, N},
                                                 encoder_context::Array{Float32, M}) where {N, M}

    decoder_inputs = prepare_decoder_inputs(mt, targets)
    logits = mt.decoder(decoder_inputs, encoder_context)
    logits
end

function Base.show(io::IO, mt::MelodyConditionedMusicTransformer)
    hs = div(size(mt.encoder[1].mh.iqproj.W)[1], mt.encoder[1].mh.head)
    h, ps = size(mt.encoder[1].pw.dout.W)
    encoder_layers = length(mt.encoder) - 1 # Ignore postprocess layernorm
    decoder_layers = length(mt.decoder) - 2 # Ignore postprocess layernorm and softmax embedding layer
    print(io, "MelodyConditionedMusicTransformer(")
    print(io, "encoder_layers=$encoder_layers, ")
    print(io, "decoder_layers=$decoder_layers, ")
    print(io, "head=$(mt.encoder[1].mh.head), ")
    print(io, "head_size=$(hs), ")
    print(io, "pwffn_size=$(ps), ")
    print(io, "size=$(h))")
end


text:
# All Document types share a common metadata profile using DocumentMetadata
abstract type AbstractMetadata end

@auto_hash_equals mutable struct DocumentMetadata <: AbstractMetadata
    language::Languages.Language
    name::String
    author::String
    timestamp::String
    id::String
    publisher::String
    edition_year::String
    published_year::String
    documenttype::String
    note::String
end

DocumentMetadata() = DocumentMetadata(DEFAULT_LANGUAGE, ("" for _ in 1:9)...)

# The abstract Document type
abstract type AbstractDocument{T<:AbstractString, M<:AbstractMetadata} end

# FileDocument type and constructors
@auto_hash_equals mutable struct FileDocument{T} <: AbstractDocument{T, DocumentMetadata}
    filename::T
    metadata::DocumentMetadata
end

FileDocument{T}(f::AbstractString) where T<:AbstractString = begin
    d = FileDocument(T(f), DocumentMetadata())
    d.metadata.name = f
    return d
end

FileDocument(f::AbstractString) = FileDocument{String}(f)

# StringDocument type and constructors
@auto_hash_equals mutable struct StringDocument{T<:AbstractString} <: AbstractDocument{T, DocumentMetadata}
    text::T
    metadata::DocumentMetadata
end

StringDocument{T}(txt::AbstractString) where T<:AbstractString =
    StringDocument(T(txt), DocumentMetadata())

StringDocument(txt::T) where T<:AbstractString =
    StringDocument{T}(txt, DocumentMetadata())

# TokenDocument type and constructors
@auto_hash_equals mutable struct TokenDocument{T<:AbstractString} <: AbstractDocument{T, DocumentMetadata}
    tokens::Vector{T}
    metadata::DocumentMetadata
end

TokenDocument(tkns::Vector{T}) where T <: AbstractString =
    TokenDocument{T}(tkns, DocumentMetadata())

TokenDocument{T}(tkns::Vector{S}) where {T<:AbstractString, S<:AbstractString} =
    TokenDocument{T}(T.(tkns), DocumentMetadata())

TokenDocument{T}(txt::AbstractString,
                 dm::DocumentMetadata=DocumentMetadata();
                 method::Symbol=DEFAULT_TOKENIZER
                ) where T<:AbstractString =
    TokenDocument{T}(T.(tokenize(txt, method=method)), dm)

TokenDocument(txt::AbstractString,
              dm::DocumentMetadata=DocumentMetadata();
              method::Symbol=DEFAULT_TOKENIZER) =
    TokenDocument(tokenize(txt, method=method), dm)

# NGramDocument type and constructors
@auto_hash_equals mutable struct NGramDocument{T<:AbstractString} <: AbstractDocument{T, DocumentMetadata}
    ngrams::Dict{T,Int}
    n::Int
    metadata::DocumentMetadata
end

NGramDocument(ng::Dict{T, Int}, n::Int=DEFAULT_NGRAM_COMPLEXITY
             ) where T <: AbstractString =
    NGramDocument{T}(ng, n, DocumentMetadata())

NGramDocument{T}(txt::AbstractString,
                 dm::DocumentMetadata=DocumentMetadata(),
                 n::Int=DEFAULT_NGRAM_COMPLEXITY;
                 tokenizer::Symbol=DEFAULT_TOKENIZER
                ) where T<:AbstractString =
    NGramDocument(ngramize(dm.language, T.(tokenize(txt, method=tokenizer)), n), n, dm)

NGramDocument(txt::AbstractString,
              dm::DocumentMetadata=DocumentMetadata(),
              n::Int=DEFAULT_NGRAM_COMPLEXITY;
              tokenizer::Symbol=DEFAULT_TOKENIZER)=
    NGramDocument(ngramize(dm.language, tokenize(txt, method=tokenizer), n), n, dm)

# Union type that refers to a generic, non-abstract document type
const GenericDocument{T} = Union{
                                 FileDocument{T},
                                 StringDocument{T},
                                 TokenDocument{T},
                                 NGramDocument{T}
                                } where T<:AbstractString

# Easier Document() constructor that decides types based on inputs
Document(str::AbstractString) = isfile(str) ? FileDocument(str) : StringDocument(str)

Document(tkns::Vector{T}) where {T <: AbstractString} = TokenDocument(tkns)

Document(ng::Dict{String, Int}) = NGramDocument(ng)

"""
    text(d)

Access the text of document `d` if possible.
"""
text(d::AbstractString) = d

text(fd::FileDocument) = begin
    !isfile(fd.filename) && error("Can't find file: $(fd.filename)")
    read(fd.filename, String)
end

text(sd::StringDocument) = sd.text

text(td::TokenDocument) = begin
    @warn("TokenDocument's can only approximate the original text")
    join(td.tokens, " ")
end

text(ngd::NGramDocument) =
    error("The text of an NGramDocument cannot be reconstructed")

"""
    text!(d, new_text)

Replace the original text of document `d` with `new_text`.
"""
text!(sd::StringDocument{T}, new_text::T) where T<:AbstractString =
    (sd.text = new_text)

text!(d::AbstractDocument, new_text::AbstractString) =
    error("The text of a $(typeof(d)) cannot be edited")

"""
    tokens(d [; method=DEFAULT_TOKENIZER])

Access the tokens of document `d` as a token array. The `method` keyword
argument specifies the type of tokenization to perform. Available
options are `:default` and `:stringanalysis`.
"""
tokens(d::AbstractString; method::Symbol=DEFAULT_TOKENIZER) =
    tokenize(d, method=method)

tokens(d::(Union{FileDocument, StringDocument}); method::Symbol=DEFAULT_TOKENIZER) =
    tokens(text(d), method=method)

tokens(d::TokenDocument; method::Symbol=DEFAULT_TOKENIZER) = d.tokens

tokens(d::NGramDocument; method::Symbol=DEFAULT_TOKENIZER) =
    error("The tokens of an NGramDocument cannot be reconstructed")

"""
    tokens!(d, new_tokens)

Replace the original tokens of document `d` with `new_tokens`.
"""
tokens!(d::TokenDocument{T}, new_tokens::Vector{T}) where T<:AbstractString =
    (d.tokens = new_tokens)

tokens!(d::AbstractDocument, new_tokens::Vector{T}) where T<:AbstractString =
    error("The tokens of a $(typeof(d)) cannot be directly edited")

"""
    ngrams(d, n=DEFAULT_GRAM_COMPLEXITY [; tokenizer=DEFAULT_TOKENIZER])

Access the document text of `d` as n-gram counts. The ngrams contain
at most `n` tokens which are obtained using `tokenizer`.
"""
ngrams(d::AbstractString,
       n::Int=DEFAULT_NGRAM_COMPLEXITY;
       tokenizer::Symbol=DEFAULT_TOKENIZER) =
    ngramize(DEFAULT_LANGUAGE, tokens(d, method=tokenizer), n)

ngrams(d::NGramDocument,
       n::Int=DEFAULT_NGRAM_COMPLEXITY;
       tokenizer::Symbol=DEFAULT_TOKENIZER) = d.ngrams

ngrams(d::AbstractDocument,
       n::Int=DEFAULT_NGRAM_COMPLEXITY;
       tokenizer::Symbol=DEFAULT_TOKENIZER) =
    ngramize(language(d), tokens(d, method=tokenizer), n)

ngrams(d::Vector{S},
       n::Int=DEFAULT_NGRAM_COMPLEXITY;
       tokenizer::Symbol=DEFAULT_TOKENIZER) where S<:AbstractString =
    ngramize(DEFAULT_LANGUAGE, d, n)

"""
    ngrams!(d, new_ngrams)

Replace the original n-grams of document `d` with `new_ngrams`.
"""
ngrams!(d::NGramDocument{T}, new_ngrams::Dict{T, Int}) where T<:AbstractString =
    (d.ngrams = new_ngrams)

ngrams!(d::AbstractDocument, new_ngrams::Dict) =
    error("The n-grams of $(typeof(d)) cannot be directly edited")

# Length describes length of document in characters
Base.length(d::NGramDocument) =
    error("NGramDocument's do not have a well-defined length")

Base.length(d::AbstractDocument) = length(text(d))

# Length describes length of document in characters
ngram_complexity(ngd::NGramDocument) = ngd.n

ngram_complexity(d::AbstractDocument) =
    error("$(typeof(d))'s have no n-gram complexity")

# getindex() methods: StringDocument("This is text and that is not")["is"]
Base.getindex(d::AbstractDocument, term::AbstractString) = get(ngrams(d), term, 0)

# Conversion rules
Base.convert(::Type{FileDocument{T}}, d::FileDocument
            ) where T<:AbstractString =
    FileDocument(T.(d.filename), d.metadata)

Base.convert(::Type{StringDocument{T}}, d::Union{FileDocument, StringDocument}
            ) where T<:AbstractString =
    StringDocument(T.(text(d)), d.metadata)

Base.convert(::Type{TokenDocument{T}}, d::Union{TokenDocument, FileDocument, StringDocument}
            ) where T<:AbstractString =
    TokenDocument(T.(tokens(d)), d.metadata)

Base.convert(::Type{NGramDocument{T}}, d::Union{TokenDocument, FileDocument, StringDocument}
            ) where T<:AbstractString =
    NGramDocument(ngramize(language(d), T.(tokens(d))), DEFAULT_NGRAM_COMPLEXITY, d.metadata)

Base.convert(::Type{NGramDocument{T}}, d::NGramDocument{T2}
            ) where {T<:AbstractString, T2<:AbstractString} =
    NGramDocument(Dict{T, Int}(ngrams(d)), d.n, d.metadata)

"""
    abstract_convert(document::AbstractDocument, parameter::Union{Nothing, Type{T}})

Tries converting `document::AbstractDocument` to one of the concrete types
with witch `StringAnalysis` works i.e. `StringDocument{T}`, `TokenDocument{T}`,
`NGramDocument{T}`. A user-defined `convert` method between the `typeof(document)`
and the concrete types should be defined.
"""
function abstract_convert(document::AbstractDocument,
                          parameter::Union{Nothing, Type{T}}=nothing
                         ) where T<:AbstractString
    if parameter != nothing
        known_types = [FileDocument{T}, StringDocument{T},
                       TokenDocument{T}, NGramDocument{T}]
    else
        known_types = [FileDocument, StringDocument,
                       TokenDocument, NGramDocument]
    end
    local newdoc
    converted = false
    for typ in known_types
        try
            newdoc = convert(typ, document)
            converted = true
            break
        catch e
        end
    end
    !converted && throw(ErrorException(
        "Could not convert the $(typeof(document)) "*
        "to any GenericDocument type."))
    return newdoc
end


text:
@testset "FO Constructor" begin
    fo = FO(:c,2,Up,false)
    @test isa(fo,FO)
    fo = FO(:c,-1,Dn,true)
    @test isa(fo,FO)
end

@testset "PFO Constructor" begin
    fo = FO(:c,2,Up,false)
    pfo = PFO(fo,symbols(:n1))
    @test isa(pfo,PFO)
    fo = FO(:d,-2,Sl,true)
    pfo = PFO(fo,symbols(:n3))
    @test isa(pfo,PFO)
end


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: greengrassv2
using AWS.Compat
using AWS.UUIDs

"""
    batch_associate_client_device_with_core_device(core_device_thing_name)
    batch_associate_client_device_with_core_device(core_device_thing_name, params::Dict{String,<:Any})

Associate a list of client devices with a core device. Use this API operation to specify
which client devices can discover a core device through cloud discovery. With cloud
discovery, client devices connect to IoT Greengrass to retrieve associated core devices'
connectivity information and certificates. For more information, see Configure cloud
discovery in the IoT Greengrass V2 Developer Guide.  Client devices are local IoT devices
that connect to and communicate with an IoT Greengrass core device over MQTT. You can
connect client devices to a core device to sync MQTT messages and data to Amazon Web
Services IoT Core and interact with client devices in Greengrass components. For more
information, see Interact with local IoT devices in the IoT Greengrass V2 Developer Guide.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"entries"`: The list of client devices to associate.
"""
function batch_associate_client_device_with_core_device(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/associateClientDevices";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function batch_associate_client_device_with_core_device(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/associateClientDevices",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    batch_disassociate_client_device_from_core_device(core_device_thing_name)
    batch_disassociate_client_device_from_core_device(core_device_thing_name, params::Dict{String,<:Any})

Disassociate a list of client devices from a core device. After you disassociate a client
device from a core device, the client device won't be able to use cloud discovery to
retrieve the core device's connectivity information and certificates.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"entries"`: The list of client devices to disassociate.
"""
function batch_disassociate_client_device_from_core_device(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/disassociateClientDevices";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function batch_disassociate_client_device_from_core_device(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/disassociateClientDevices",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    cancel_deployment(deployment_id)
    cancel_deployment(deployment_id, params::Dict{String,<:Any})

Cancels a deployment. This operation cancels the deployment for devices that haven't yet
received it. If a device already received the deployment, this operation doesn't change
anything for that device.

# Arguments
- `deployment_id`: The ID of the deployment.

"""
function cancel_deployment(deploymentId; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "POST",
        "/greengrass/v2/deployments/$(deploymentId)/cancel";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function cancel_deployment(
    deploymentId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/deployments/$(deploymentId)/cancel",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_component_version()
    create_component_version(params::Dict{String,<:Any})

Creates a component. Components are software that run on Greengrass core devices. After you
develop and test a component on your core device, you can use this operation to upload your
component to IoT Greengrass. Then, you can deploy the component to other core devices. You
can use this operation to do the following:    Create components from recipes  Create a
component from a recipe, which is a file that defines the component's metadata, parameters,
dependencies, lifecycle, artifacts, and platform capability. For more information, see IoT
Greengrass component recipe reference in the IoT Greengrass V2 Developer Guide. To create a
component from a recipe, specify inlineRecipe when you call this operation.    Create
components from Lambda functions  Create a component from an Lambda function that runs on
IoT Greengrass. This creates a recipe and artifacts from the Lambda function's deployment
package. You can use this operation to migrate Lambda functions from IoT Greengrass V1 to
IoT Greengrass V2. This function only accepts Lambda functions that use the following
runtimes:   Python 2.7 – python2.7    Python 3.7 – python3.7    Python 3.8 –
python3.8    Java 8 – java8    Node.js 10 – nodejs10.x    Node.js 12 – nodejs12.x
To create a component from a Lambda function, specify lambdaFunction when you call this
operation.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"clientToken"`: A unique, case-sensitive identifier that you can provide to ensure that
  the request is idempotent. Idempotency means that the request is successfully processed
  only once, even if you send the request multiple times. When a request succeeds, and you
  specify the same client token for subsequent successful requests, the IoT Greengrass V2
  service returns the successful response that it caches from the previous request. IoT
  Greengrass V2 caches successful responses for idempotent requests for up to 8 hours.
- `"inlineRecipe"`: The recipe to use to create the component. The recipe defines the
  component's metadata, parameters, dependencies, lifecycle, artifacts, and platform
  compatibility. You must specify either inlineRecipe or lambdaFunction.
- `"lambdaFunction"`: The parameters to create a component from a Lambda function. You must
  specify either inlineRecipe or lambdaFunction.
- `"tags"`: A list of key-value pairs that contain metadata for the resource. For more
  information, see Tag your resources in the IoT Greengrass V2 Developer Guide.
"""
function create_component_version(; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "POST",
        "/greengrass/v2/createComponentVersion",
        Dict{String,Any}("clientToken" => string(uuid4()));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_component_version(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/createComponentVersion",
        Dict{String,Any}(
            mergewith(_merge, Dict{String,Any}("clientToken" => string(uuid4())), params)
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    create_deployment(target_arn)
    create_deployment(target_arn, params::Dict{String,<:Any})

Creates a continuous deployment for a target, which is a Greengrass core device or group of
core devices. When you add a new core device to a group of core devices that has a
deployment, IoT Greengrass deploys that group's deployment to the new device. You can
define one deployment for each target. When you create a new deployment for a target that
has an existing deployment, you replace the previous deployment. IoT Greengrass applies the
new deployment to the target devices. Every deployment has a revision number that indicates
how many deployment revisions you define for a target. Use this operation to create a new
revision of an existing deployment. This operation returns the revision number of the new
deployment when you create it. For more information, see the Create deployments in the IoT
Greengrass V2 Developer Guide.

# Arguments
- `target_arn`: The ARN of the target IoT thing or thing group.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"clientToken"`: A unique, case-sensitive identifier that you can provide to ensure that
  the request is idempotent. Idempotency means that the request is successfully processed
  only once, even if you send the request multiple times. When a request succeeds, and you
  specify the same client token for subsequent successful requests, the IoT Greengrass V2
  service returns the successful response that it caches from the previous request. IoT
  Greengrass V2 caches successful responses for idempotent requests for up to 8 hours.
- `"components"`: The components to deploy. This is a dictionary, where each key is the
  name of a component, and each key's value is the version and configuration to deploy for
  that component.
- `"deploymentName"`: The name of the deployment.
- `"deploymentPolicies"`: The deployment policies for the deployment. These policies define
  how the deployment updates components and handles failure.
- `"iotJobConfiguration"`: The job configuration for the deployment configuration. The job
  configuration specifies the rollout, timeout, and stop configurations for the deployment
  configuration.
- `"tags"`: A list of key-value pairs that contain metadata for the resource. For more
  information, see Tag your resources in the IoT Greengrass V2 Developer Guide.
"""
function create_deployment(targetArn; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "POST",
        "/greengrass/v2/deployments",
        Dict{String,Any}("targetArn" => targetArn, "clientToken" => string(uuid4()));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function create_deployment(
    targetArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/deployments",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "targetArn" => targetArn, "clientToken" => string(uuid4())
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_component(arn)
    delete_component(arn, params::Dict{String,<:Any})

Deletes a version of a component from IoT Greengrass.  This operation deletes the
component's recipe and artifacts. As a result, deployments that refer to this component
version will fail. If you have deployments that use this component version, you can remove
the component from the deployment or update the deployment to use a valid version.

# Arguments
- `arn`: The ARN of the component version.

"""
function delete_component(arn; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "DELETE",
        "/greengrass/v2/components/$(arn)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_component(
    arn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "DELETE",
        "/greengrass/v2/components/$(arn)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    delete_core_device(core_device_thing_name)
    delete_core_device(core_device_thing_name, params::Dict{String,<:Any})

Deletes a Greengrass core device, which is an IoT thing. This operation removes the core
device from the list of core devices. This operation doesn't delete the IoT thing. For more
information about how to delete the IoT thing, see DeleteThing in the IoT API Reference.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

"""
function delete_core_device(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "DELETE",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function delete_core_device(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "DELETE",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    describe_component(arn)
    describe_component(arn, params::Dict{String,<:Any})

Retrieves metadata for a version of a component.

# Arguments
- `arn`: The ARN of the component version.

"""
function describe_component(arn; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/metadata";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function describe_component(
    arn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/metadata",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_component(arn)
    get_component(arn, params::Dict{String,<:Any})

Gets the recipe for a version of a component. Core devices can call this operation to
identify the artifacts and requirements to install a component.

# Arguments
- `arn`: The ARN of the component version.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"recipeOutputFormat"`: The format of the recipe.
"""
function get_component(arn; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_component(
    arn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_component_version_artifact(arn, artifact_name)
    get_component_version_artifact(arn, artifact_name, params::Dict{String,<:Any})

Gets the pre-signed URL to download a public component artifact. Core devices call this
operation to identify the URL that they can use to download an artifact to install.

# Arguments
- `arn`: The ARN of the component version. Specify the ARN of a public component version.
- `artifact_name`: The name of the artifact. You can use the GetComponent operation to
  download the component recipe, which includes the URI of the artifact. The artifact name is
  the section of the URI after the scheme. For example, in the artifact URI
  greengrass:SomeArtifact.zip, the artifact name is SomeArtifact.zip.

"""
function get_component_version_artifact(
    arn, artifactName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/artifacts/$(artifactName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_component_version_artifact(
    arn,
    artifactName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/artifacts/$(artifactName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_core_device(core_device_thing_name)
    get_core_device(core_device_thing_name, params::Dict{String,<:Any})

Retrieves metadata for a Greengrass core device.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

"""
function get_core_device(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_core_device(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    get_deployment(deployment_id)
    get_deployment(deployment_id, params::Dict{String,<:Any})

Gets a deployment. Deployments define the components that run on Greengrass core devices.

# Arguments
- `deployment_id`: The ID of the deployment.

"""
function get_deployment(deploymentId; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/deployments/$(deploymentId)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function get_deployment(
    deploymentId,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/deployments/$(deploymentId)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_client_devices_associated_with_core_device(core_device_thing_name)
    list_client_devices_associated_with_core_device(core_device_thing_name, params::Dict{String,<:Any})

Retrieves a paginated list of client devices that are associated with a core device.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
"""
function list_client_devices_associated_with_core_device(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/associatedClientDevices";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_client_devices_associated_with_core_device(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/associatedClientDevices",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_component_versions(arn)
    list_component_versions(arn, params::Dict{String,<:Any})

Retrieves a paginated list of all versions for a component. Greater versions are listed
first.

# Arguments
- `arn`: The ARN of the component version.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
"""
function list_component_versions(arn; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/versions";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_component_versions(
    arn, params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components/$(arn)/versions",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_components()
    list_components(params::Dict{String,<:Any})

Retrieves a paginated list of component summaries. This list includes components that you
have permission to view.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
- `"scope"`: The scope of the components to list. Default: PRIVATE
"""
function list_components(; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/components";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_components(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/components",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_core_devices()
    list_core_devices(params::Dict{String,<:Any})

Retrieves a paginated list of Greengrass core devices.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
- `"status"`: The core device status by which to filter. If you specify this parameter, the
  list includes only core devices that have this status. Choose one of the following options:
     HEALTHY – The IoT Greengrass Core software and all components run on the core device
  without issue.    UNHEALTHY – The IoT Greengrass Core software or a component is in a
  failed state on the core device.
- `"thingGroupArn"`: The ARN of the IoT thing group by which to filter. If you specify this
  parameter, the list includes only core devices that are members of this thing group.
"""
function list_core_devices(; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_core_devices(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_deployments()
    list_deployments(params::Dict{String,<:Any})

Retrieves a paginated list of deployments.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"historyFilter"`: The filter for the list of deployments. Choose one of the following
  options:    ALL – The list includes all deployments.    LATEST_ONLY – The list includes
  only the latest revision of each deployment.   Default: LATEST_ONLY
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
- `"targetArn"`: The ARN of the target IoT thing or thing group.
"""
function list_deployments(; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "GET",
        "/greengrass/v2/deployments";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_deployments(
    params::AbstractDict{String}; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/deployments",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_effective_deployments(core_device_thing_name)
    list_effective_deployments(core_device_thing_name, params::Dict{String,<:Any})

Retrieves a paginated list of deployment jobs that IoT Greengrass sends to Greengrass core
devices.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
"""
function list_effective_deployments(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/effectiveDeployments";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_effective_deployments(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/effectiveDeployments",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_installed_components(core_device_thing_name)
    list_installed_components(core_device_thing_name, params::Dict{String,<:Any})

Retrieves a paginated list of the components that a Greengrass core device runs.

# Arguments
- `core_device_thing_name`: The name of the core device. This is also the name of the IoT
  thing.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"maxResults"`: The maximum number of results to be returned per paginated request.
- `"nextToken"`: The token to be used for the next set of paginated results.
"""
function list_installed_components(
    coreDeviceThingName; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/installedComponents";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_installed_components(
    coreDeviceThingName,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/greengrass/v2/coreDevices/$(coreDeviceThingName)/installedComponents",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    list_tags_for_resource(resource_arn)
    list_tags_for_resource(resource_arn, params::Dict{String,<:Any})

Retrieves the list of tags for an IoT Greengrass resource.

# Arguments
- `resource_arn`: The ARN of the resource.

"""
function list_tags_for_resource(
    resourceArn; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "GET",
        "/tags/$(resourceArn)";
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function list_tags_for_resource(
    resourceArn,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "GET",
        "/tags/$(resourceArn)",
        params;
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    resolve_component_candidates(component_candidates, platform)
    resolve_component_candidates(component_candidates, platform, params::Dict{String,<:Any})

Retrieves a list of components that meet the component, version, and platform requirements
of a deployment. Greengrass core devices call this operation when they receive a deployment
to identify the components to install. This operation identifies components that meet all
dependency requirements for a deployment. If the requirements conflict, then this operation
returns an error and the deployment fails. For example, this occurs if component A requires
version >2.0.0 and component B requires version <2.0.0 of a component dependency.
When you specify the component candidates to resolve, IoT Greengrass compares each
component's digest from the core device with the component's digest in the Amazon Web
Services Cloud. If the digests don't match, then IoT Greengrass specifies to use the
version from the Amazon Web Services Cloud.  To use this operation, you must use the data
plane API endpoint and authenticate with an IoT device certificate. For more information,
see IoT Greengrass endpoints and quotas.

# Arguments
- `component_candidates`: The list of components to resolve.
- `platform`: The platform to use to resolve compatible components.

"""
function resolve_component_candidates(
    componentCandidates, platform; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/resolveComponentCandidates",
        Dict{String,Any}(
            "componentCandidates" => componentCandidates, "platform" => platform
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function resolve_component_candidates(
    componentCandidates,
    platform,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/greengrass/v2/resolveComponentCandidates",
        Dict{String,Any}(
            mergewith(
                _merge,
                Dict{String,Any}(
                    "componentCandidates" => componentCandidates, "platform" => platform
                ),
                params,
            ),
        );
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    tag_resource(resource_arn, tags)
    tag_resource(resource_arn, tags, params::Dict{String,<:Any})

Adds tags to an IoT Greengrass resource. If a tag already exists for the resource, this
operation updates the tag's value.

# Arguments
- `resource_arn`: The ARN of the resource to tag.
- `tags`: A list of key-value pairs that contain metadata for the resource. For more
  information, see Tag your resources in the IoT Greengrass V2 Developer Guide.

"""
function tag_resource(resourceArn, tags; aws_config::AbstractAWSConfig=global_aws_config())
    return greengrassv2(
        "POST",
        "/tags/$(resourceArn)",
        Dict{String,Any}("tags" => tags);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function tag_resource(
    resourceArn,
    tags,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "POST",
        "/tags/$(resourceArn)",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("tags" => tags), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end

"""
    untag_resource(resource_arn, tag_keys)
    untag_resource(resource_arn, tag_keys, params::Dict{String,<:Any})

Removes a tag from an IoT Greengrass resource.

# Arguments
- `resource_arn`: The ARN of the resource to untag.
- `tag_keys`: A list of keys for tags to remove from the resource.

"""
function untag_resource(
    resourceArn, tagKeys; aws_config::AbstractAWSConfig=global_aws_config()
)
    return greengrassv2(
        "DELETE",
        "/tags/$(resourceArn)",
        Dict{String,Any}("tagKeys" => tagKeys);
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end
function untag_resource(
    resourceArn,
    tagKeys,
    params::AbstractDict{String};
    aws_config::AbstractAWSConfig=global_aws_config(),
)
    return greengrassv2(
        "DELETE",
        "/tags/$(resourceArn)",
        Dict{String,Any}(mergewith(_merge, Dict{String,Any}("tagKeys" => tagKeys), params));
        aws_config=aws_config,
        feature_set=SERVICE_FEATURE_SET,
    )
end


text:
export accessible, coaccessible, trim

function accessible(wfst::Wfst)
    stack = copy(wfst.initial_states)
    discovered_states = Any[]
    discovered_arcs = Arc[]
    # Classic DFS
    while length(stack) > 0
        state = pop!(stack)
        if !(state in discovered_states)
            push!(discovered_states, state)
            for arc in wfst.arcs
                if arc.from == state
                    push!(stack, arc.to)
                    push!(discovered_arcs, arc)
                end
            end
        end
    end

    println(wfst)
    new = deepcopy(wfst)
    new.states = Set(discovered_states)
    new.arcs = Set(discovered_arcs)
    println(new)
    return new
end

function coaccessible(wfst::Wfst)
    stack = copy(wfst.final_states)
    discovered_states = Any[]
    discovered_arcs = Arc[]
    # Classic DFS, just reversing the direction we follow arcs
    while length(stack) > 0
        state = pop!(stack)
        if !(state in discovered_states)
            push!(discovered_states, state)
            for arc in wfst.arcs
                # Note the reversal of arc.to and arc.from
                if arc.to == state
                    push!(stack, arc.from)
                    push!(discovered_arcs, arc)
                end
            end
        end
    end

    new = deepcopy(wfst)
    new.states = Set(discovered_states)
    new.arcs = Set(discovered_arcs)
    return new
end

function trim(wfst::Wfst)
    return coaccessible(accessible(wfst))
end


text:
using SubmodularMaximization
using Distributions
using Statistics

table = SubmodularMaximization.NormalLookup(increment=0.0001, max=4.0)

maxerror = maximum(0.0:0.00001:5) do x
  trueval = pdf(Normal(0.0, 1.0), -x)
  lookupval = SubmodularMaximization.evaluate(table, error= -x, stddev=1.0)

  abs(trueval - lookupval)
end
println("Max error: ", maxerror)

let sigma = 3.0, error = 5.0
  @show trueval = pdf(Normal(0.0, sigma), error)
  @show lookupval = SubmodularMaximization.evaluate(table, error=error, stddev=sigma)
  @show abs(trueval - lookupval)
end

max_stars_repo_name: mcorah/MultiAgentSensing

max_issues_repo_name: mcorah/MultiAgentSensing

max_forks_repo_name: mcorah/MultiAgentSensing


text:
println("hello, this is Julia")

println(" 傻 胖子！！！ ")

a = 2
b = 3
println("2 * 3  = " , a*b)


text:
println("Loading packages...")
@time begin
  using Cytof5
  using Random, Distributions
  # TODO: Get rid of this dep
  using RCall
  using BSON
  using ArgParse
  import Cytof5.Model.logger
  include("util.jl")
end
println("Done loading packages.")

# TODO: review
# ARG PARSING

# Define behavior for vector numerical args (input as space delimited string)
ArgParse.parse_item(::Type{Vector{T}}, x::AbstractString) where {T <: Number} = parse.(T, split(x))

function parse_cmd()
  s = ArgParseSettings()

  @add_arg_table s begin
    "--simdat_path"
      arg_type = String
      required = true
    "--MCMC_ITER"
      arg_type = Int
      default = 1000
    "--BURN"
      arg_type = Int
      default = 10000
    "--K_MCMC"
      arg_type = Int
      required = true
    "--L0_MCMC"
      arg_type = Int
      required = true
    "--L1_MCMC"
      arg_type = Int
      required = true

    # Missin mechanism
    "--pBounds"
      arg_type = Vector{Float64}
      default = [.05, .8, .05] # paper
    "--yQuantiles"
      arg_type = Vector{Float64}
      default = [0.0, .25, .5] # paper

    "--RESULTS_DIR"
      arg_type = String
      required = true
    "--SEED"
      arg_type = Int
      default = 0
    "--EXP_NAME"
      arg_type = String
    "--printFreq"
      arg_type = Int
      default = 50
  end

  PARSED_ARGS = parse_args(s)

  if PARSED_ARGS["EXP_NAME"] == nothing
    logger("EXP_NAME not defined. Making default experiment name.")
    MAIN_ARGS = filter(d -> !(d.first in ("RESULTS_DIR", "EXP_NAME")), PARSED_ARGS)
    PARSED_ARGS["EXP_NAME"] = join(["$k$v" for (k, v) in MAIN_ARGS], '_')
  end

  return PARSED_ARGS
end

PARSED_ARGS = parse_cmd()
for (k,v) in PARSED_ARGS
  logger("$k => $v")
end

SIMDAT_PATH = PARSED_ARGS["simdat_path"]

MCMC_ITER = PARSED_ARGS["MCMC_ITER"]
BURN = PARSED_ARGS["BURN"]

K_MCMC = PARSED_ARGS["K_MCMC"]
L0_MCMC = PARSED_ARGS["L0_MCMC"]
L1_MCMC = PARSED_ARGS["L1_MCMC"]
L_MCMC = Dict(0 => L0_MCMC, 1 => L1_MCMC)

# missing mechanism
yQuantiles = PARSED_ARGS["yQuantiles"]
pBounds = PARSED_ARGS["pBounds"]

EXP_NAME = PARSED_ARGS["EXP_NAME"]
SEED = PARSED_ARGS["SEED"]
RESULTS_DIR = PARSED_ARGS["RESULTS_DIR"]
printFreq = PARSED_ARGS["printFreq"]
# END OF ARG PARSING

# CREATE RESULTS DIR
OUTDIR = "$(RESULTS_DIR)/$(EXP_NAME)/"
mkpath(OUTDIR)

# Set random seed
Random.seed!(SEED);

logger("Load simulated data ...");
BSON.@load SIMDAT_PATH simdat
dat = Cytof5.Model.Data(simdat[:y])

logger("Generating priors ...");
@time c = Cytof5.Model.defaultConstants(dat, K_MCMC, L_MCMC,
                                        tau0=10.0, tau1=10.0,
                                        sig2_prior=InverseGamma(3.0, 2.0),
                                        alpha_prior=Gamma(0.1, 10.0),
                                        yQuantiles=yQuantiles, pBounds=pBounds,
                                        similarity_Z=Cytof5.Model.sim_fn_abs(10000),
                                        probFlip_Z=2.0 / (dat.J * K_MCMC),
                                        noisyDist=Normal(0.0, 3.16))
Cytof5.Model.printConstants(c)
println("dat.I: $(dat.I)")
println("dat.J: $(dat.J)")
println("dat.N: $(dat.N)")

# Plot missing mechanism
logger("Plot missing mechanism")
util.plotPdf("$(OUTDIR)/prob_miss.pdf")
R"par(mfrow=c($(dat.I), 1))"
for i in 1:dat.I
  util.plotProbMiss(c.beta, i)
end
R"par(mfrow=c(1,1))"
util.devOff()

logger("Generating initial state ...");
# @time init = Cytof5.Model.genInitialState(c, dat)
logger("use smart init ...")
@time init = Cytof5.Model.smartInit(c, dat)

# Plot initial Z
util.plotPdf("$(OUTDIR)/Z_init.pdf")
addGridLines(J::Int, K::Int, col="grey") = util.abline(v=(1:K) .+ .5, h=(1:J) .+ .5, col=col)
util.myImage(init.Z, xlab="Features", ylab="Markers", addL=false, f=Z->addGridLines(dat.J, c.K))
util.devOff()

# Fit Model
nsamps_to_thin(nsamps::Int, nmcmc::Int) = max(1, div(nmcmc, nsamps))

@time out, lastState, ll, metrics, dden=
  Cytof5.Model.cytof5_fit(init, c, dat,
                          monitors=[[:Z, :lam, :W,
                                     :sig2, :delta,
                                     :alpha, :v,
                                     :eta, :eps],
                                    [:y_imputed, :gam]],
                          thins=[2, nsamps_to_thin(10, MCMC_ITER)],
                          nmcmc=MCMC_ITER, nburn=BURN,
                          computeLPML=true, computeDIC=true,
                          computedden=true, thin_dden=nsamps_to_thin(200, MCMC_ITER),
                          use_repulsive=false,
                          joint_update_Z=true,
                          printFreq=10, flushOutput=true)

logger("Saving Data ...");
println("length of dden: $(length(dden))")
# @save "$(OUTDIR)/output.jld2" out dat ll lastState c dat metrics
BSON.@save "$(OUTDIR)/output.bson" out ll lastState c metrics init dden simdat

logger("MCMC Completed.");

#= Test
julia --color=yes sim.jl --I=3 --J=32 --N_factor=100 --K=8 --K_MCMC=10 --L0_MCMC=5 --L1_MCMC=5 \
      --MCMC_ITER=10 --BURN=10 --RESULTS_DIR=results --EXP_NAME=bla
=#


text:
# No primatives
function scgal_full_mc(
        C_input,
        b_input,
        A!,
        A_adj!;
        n,
        d,
        R = 30,
        scale_X = 1.0,
        scale_C = 1.0,
        max_iters = 1_000,
        tol = 1e-10,
        print_iter = 10,
        logging=false,              # stores obj_val, dual_gap, infeas
        logging_primal=false,       # computes Xhat & stores true obj, infeas
        compute_cut=false,          # computes the maxcut value
        rseed=0,
        ηt::Function=t->2.0/(t + 1.0),
        δt::Function=t->1.0,
)
    Random.seed!(0)

    # --- copy variables & scale---
    C = copy(C_input) .* scale_C
    b = copy(b_input) .* scale_X
    norm_b = norm(b_input)

    # --- parameters ---
    # Note: assume scaling factors are such that α = 1
    # α = 1
    t = 1
    β0 = 1
    K = Inf

    # --- init sketch, primal state (z), dual, values ---
    Ω, St = init_sketch(n, R)
    yt = zeros(d)
    zt = zeros(d)
    dual_gap = Inf
    primal_infeas = Inf
    obj_val = 0

    # --- cache & other memory allocations ---
    qmax = min(n-1, Int(ceil(max_iters^0.25 * log(n))))
    cache = (
        A_X = zeros(d),
        dual_update = zeros(d),
        sketch_update = zeros(R),
        evec=init_cache_evec(n, qmax),
        tmp = zeros(n)
    )
    Dt = copy(C)
    v = zeros(n)
    if logging_primal
        log_cache = zeros(n, R)
    end

    # --- logging ---
    if logging
        dual_gap_log = zeros(max_iters)
        obj_val_log = zeros(max_iters)
        primal_infeas_log = zeros(max_iters)
        time_sec_log = zeros(max_iters)
    end
    # These require a (pseudo)reconstruction of the sketch
    if logging_primal
        obj_val_Xhat_log = zeros(max_iters)
        primal_infeas_Xhat_log = zeros(max_iters)
        if compute_cut
            cut_value_log = zeros(max_iters)
        end
    end

    # --- Keep track of things ---
    # (credit: https://github.com/ZIB-IOL/FrankWolfe.jl)
    headers = ["Iteration", "Primal", "Dual Gap", "Infeas", "Time"]
    print_header(headers)

    # --------------------------------------------------------------------------
    # --------------------- ITERATIONS -----------------------------------------
    # --------------------------------------------------------------------------
    time_start = time_ns()
    while t <= max_iters #&& dual_gap >= max(tol, eps())
        # --- Parameter updates (from Yurtsever et al Alg 6.1) ---
        βt = β0 * sqrt(t + 1)
        η = ηt(t)
        δ = δt(t)

        # --- Gradient computation ---
        # * Non-allocating *
        cache.A_X .= zt .- b
        cache.A_X .*= βt
        cache.A_X .+= yt
        # C entries are already in Dt -- just update diagonal
        # NOTE: This only works for MAXCUT as implemented
        for i in 1:n
            @views Dt[i,i] = (1.0 - δ)*Dt[i,i] + δ*(C[i,i] + cache.A_X[i])
        end

        # --- Eigenvector computation ---
        # * Custom Method *
        q = Int(ceil(t^0.25 * log(n)))
        ξ = approx_min_evec!(v, Dt, n=n, q=q, cache=cache.evec)
        # * ArnoldiMethod.jl (more stable than Lanzcos) *
        # F = partialschur(Dt, nev=1, which=SR(), tol=sqrt(n)*eps())
        # !F[2].converged && error("Eigvec computation did not converge")
        # ξ, vv = partialeigen(F[1])
        # v .= vv[:,1]

        # --- "Primal" update ---
        zt .-= η.*zt
        # TODO: use primative (3)
        #   NOTE: This only works for MAXCUT as implemented (sorry -- was lazy)
        zt .+= η .* v .* v

        # --- Sketch update ---
        # St = (1-η)*St + η*v*v'*Ω
        rank_one_update!(St, Ω, v, η; cache=cache.sketch_update)

        # --- Dual update (& primal infeasibility) ---
        cache.dual_update .= zt .- b
        # eq (6.4)
        primal_infeas = norm(cache.dual_update)
        γ = min(β0, 4β0*sqrt(t+1)/(t+1)^2 / primal_infeas^2)
        #TODO: get rid of scaling??
        primal_infeas /= (scale_X * (1 + norm_b))
        @. yt += γ*cache.dual_update

        # --- compute objective values for output ---
        mul!(cache.tmp, C, v)
        obj_val = (1-η)*obj_val + η*dot(v, cache.tmp)

        # --- duality gap ---
        @. cache.dual_update = yt + βt*cache.dual_update
        dual_gap = obj_val + dot(cache.dual_update, cache.A_X) - ξ[1]

        # --- logging ---
        time_sec = (time_ns() - time_start) / 1e9
        if logging
            obj_val_log[t] = obj_val
            dual_gap_log[t] = dual_gap
            primal_infeas_log[t] = primal_infeas
            time_sec_log[t] = time_sec
        end
        if logging_primal && t > 15
            U, Λ = reconstruct(Ω, St, correction=true)
            obj_val_Xhat_log[t] = compute_objective(C, U, Λ; cache=log_cache)
            primal_infeas_Xhat_log[t] = compute_primal_infeas_mc(U, Λ; cache=log_cache)
            if compute_cut
                cut_value_log[t] = max_cut_val(C, U, Λ)
            end
        end

        # --- printing ---
        if t == 1 || t % print_iter == 0
            print_iter_func((
                string(t),
                obj_val * 1/scale_C * 1/scale_X,
                dual_gap,
                primal_infeas,
                time_sec
            ))
        end

        t += 1
    end

    # --- Prepare Output ---
    solve_time = (time_ns() - time_start) / 1e9
    @printf("\nSolved in %6.3fs\n", solve_time)
    # Reconstruct Xhat = U*Λ*U'
    U, Λ = reconstruct(Ω, St, correction=true)

    print_footer()

    # Constructs log & returns solution object
    if !compute_cut
        cut_value_log = nothing
    end
    if logging && logging_primal
        scgal_log = SCGALLog(
            dual_gap_log,
            obj_val_log,
            primal_infeas_log,
            time_sec_log,
            obj_val_Xhat_log,
            primal_infeas_Xhat_log,
            cut_value_log
        )
    elseif logging
        scgal_log = SCGALLog(
            dual_gap_log,
            obj_val_log,
            primal_infeas_log,
            time_sec_log,
            nothing,
            nothing,
            nothing
        )
    else
        scgal_log = nothing
    end

    return SCGALSolution(U, Λ, St, Ω, yt, zt, scgal_log)

end

# TODO: better to add this as an option for the above
# ------------------------------------------------------------------------
# ------------------------------------------------------------------------
# ------------------------------------------------------------------------
# Uses primatives defined in (2.4) in the paper
#=
function scgal(
        C_u!,           # u     -> C*u
        Aadj_zu!,       # (u,z) -> A_adj(z)*u
        A_uu!,          # u     -> A(u*u')
        b_input,
        n,
        d,
        R = 30,
        scale_X = 1,
        scale_C = 1,
        max_iters = 1_000,
        tol = 1e-10,
        print_iter = 10,
)

    # Copy input
    b = copy(b_input) .* scale_X
    norm_b = norm(b_input)

    α = 1
    t = 1
    β0 = 1
    K = Inf

    dual_gap = Inf
    obj_val = 0

    Ω, St = init_sketch(n, R)
    yt = zeros(d)
    zt = zeros(d)

    # TODO: maybe better to rename to indicate size?
    qmax = min(n-1, Int(ceil(max_iters^0.25 * log(n))))
    cache = (
        A_X = zeros(d),
        dual_update = zeros(d),
        sketch_update = zeros(R),
        evec = init_cache_evec(n, qmax),
        tmp = zeros(n)
    )
    Dt = copy(C)
    v = zeros(n)

    # Keep track of things (credit: https://github.com/ZIB-IOL/FrankWolfe.jl)
    headers = ["Iteration", "Primal", "Dual", "Dual Gap", "Infeas", "Time"]
    print_header(headers)
    time_start = time_ns()
    while t <= max_iters #&& dual_gap >= max(tol, eps())
        # --- Parameter updates (from Yurtsever et al Alg 6.1) ---
        βt = β0 * sqrt(t + 1)
        η = 2/(t + 1)

        # --- Gradient computation ---
        # * Non-allocating *
        cache.A_X .= zt .- b
        cache.A_X .*= βt
        cache.A_X .+= yt
        # C entries are already in Dt -- just update diagonal
        # for i in 1:n
        #     Dt[i,i] = C[i,i] + cache.A_X[i]
        # end

        # --- Eigenvector computation ---
        # * Custom Method *
        q = Int(ceil(10 * t^0.25 * log(n)))
        ξ = approx_min_evec!(v, C_u!, Aadj_zu!, cache.AX, n=n, q=q, cache=cache.evec)
        # * ArnoldiMethod.jl (more stable than Lanzcos) *
        # F = partialschur(Dt, nev=1, which=SR(), tol=sqrt(n)*eps())
        # !F[2].converged && error("Eigvec computation did not converge")
        # ξ, vv = partialeigen(F[1])
        # v .= vv[:,1]

        # --- "Primal" update ---
        zt .-= η.*zt
        A_uu!(cache.dual_update, v)
        zt .+= η .* cache.dual_update

        # --- Sketch update ---
        rank_one_update!(St, Ω, v, η; cache=cache.sketch_update)

        # --- Dual update (& primal infeasibility) ---
        cache.dual_update .= zt .- b
        primal_infeas = norm(cache.dual_update) * 1/scale_X / (1 + norm_b)
        γ = min(β0, βt*η^2 / primal_infeas^2)
        @. yt += γ*cache.dual_update

        # --- compute objective values for output ---
        # TODO: make this efficient using primative 1
        C_u!(cache.tmp, v)
        obj_update = dot(v, cache.tmp)
        obj_val = (1-η)*obj_val + η*obj_update
        dual_val = dot(b, yt) * 1/scale_X

        # --- duality gap ---
        @. cache.dual_update = yt + βt*cache.dual_update
        dual_gap = obj_val + dot(cache.dual_update, cache.A_X) - ξ[1]

        # --- printing ---
        if t == 1 || t % print_iter == 0
            print_iter_func((
                string(t),
                obj_val * 1/scale_C * 1/scale_X,
                dual_val,
                dual_gap,
                primal_infeas,
                (time_ns() - time_start) / 1e9
            ))
        end
        t += 1
    end

    # --- Prepare Output ---
    solve_time = (time_ns() - time_start) / 1e9
    # Reconstruct Xhat = U*Λ*U'
    U, Λ = reconstruct(Ω, St)
    @. Λ.diag += (1.0 - tr(Λ))*ones(R) / R

    # TODO: return a soution object
    # primal var (U, Λ), dual var, AX
    return U, Λ, yt, zt

end
=#