text:
function planar_rx()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 9,
        3 => 4,
        4 => 8,
        5 => 6,
        6 => 14,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 2),
        2 => HalfEdge(2, 1),
        3 => HalfEdge(1, 3),
        4 => HalfEdge(3, 1),
        5 => HalfEdge(1, 5),
        6 => HalfEdge(5, 1),
        7 => HalfEdge(3, 4),
        8 => HalfEdge(4, 3),
        9 => HalfEdge(2, 6),
        10 => HalfEdge(6, 2),
        11 => HalfEdge(4, 6),
        12 => HalfEdge(6, 4),
        13 => HalfEdge(5, 6),
        14 => HalfEdge(6, 5),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 3,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,
        13 => 0,
        10 => 0,

        1 => 1,
        9 => 1,
        12 => 1,
        8 => 1,
        4 => 1,

        3 => 2,
        7 => 2,
        11 => 2,
        14 => 2,
        6 => 2,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 13,
        13 => 10,
        10 => 2,

        1 => 9,
        9 => 12,
        12 => 8,
        8 => 4,
        4 => 1,

        3 => 7,
        7 => 11,
        11 => 14,
        14 => 6,
        6 => 3,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
        7 => 8,
        8 => 7,
        9 => 10,
        10 => 9,
        11 => 12,
        12 => 11,
        13 => 14,
        14 => 13,
    )
    vs_isolated = Dict{Int, Int}()
    rx = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        6, 14, 2
    )
    return rx
end

function planar_rz()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 9,
        3 => 4,
        4 => 6,
        5 => 14,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 2),
        2 => HalfEdge(2, 1),
        3 => HalfEdge(1, 3),
        4 => HalfEdge(3, 1),
        5 => HalfEdge(1, 4),
        6 => HalfEdge(4, 1),
        7 => HalfEdge(2, 3),
        8 => HalfEdge(3, 2),
        9 => HalfEdge(2, 5),
        10 => HalfEdge(5, 2),
        11 => HalfEdge(3, 5),
        12 => HalfEdge(5, 3),
        13 => HalfEdge(4, 5),
        14 => HalfEdge(5, 4),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 8,
        3 => 3,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,
        13 => 0,
        10 => 0,

        1 => 1,
        7 => 1,
        4 => 1,

        8 => 2,
        9 => 2,
        12 => 2,

        3 => 3,
        11 => 3,
        14 => 3,
        6 => 3,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 13,
        13 => 10,
        10 => 2,

        1 => 7,
        7 => 4,
        4 => 1,

        8 => 9,
        9 => 12,
        12 => 8,

        3 => 11,
        11 => 14,
        14 => 6,
        6 => 3,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
        7 => 8,
        8 => 7,
        9 => 10,
        10 => 9,
        11 => 12,
        12 => 11,
        13 => 14,
        14 => 13,
    )
    vs_isolated = Dict{Int, Int}()
    rz = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        5, 14, 3
    )
    return rz
end

function tait_rx(θ::QuonComplex)
    θ = to_quon_const(θ)
    g = planar_rx()
    p = QuonParam{QuonComplex}(θ, true)
    quon_params = Dict(7 => p, 8 => p)
    inputs = Int[1]
    outputs = Int[6]
    genuses = Set([2, 5])
    locations = Dict(
        1 => (1.0, 0.0),
        2 => (0.0, 1.0),
        3 => (1.0, 1.0),
        4 => (1.0, 2.0),
        5 => (2.0, 1.0),
        6 => (1.0, 3.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end
tait_rx(θ) = tait_rx(ComplexF64(θ))

function tait_rz(θ::QuonComplex)
    θ = to_quon_const(θ)
    g = planar_rz()
    p = QuonParam{QuonComplex}(θ, false)
    quon_params = Dict(7 => p, 8 => p)
    inputs = Int[1]
    outputs = Int[5]
    genuses = Set([2, 4])
    locations = Dict(
        1 => (1.0, 0.0),
        2 => (0.0, 1.0),
        3 => (1.0, 1.0),
        4 => (2.0, 1.0),
        5 => (1.0, 2.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end
tait_rz(θ) = tait_rz(ComplexF64(θ))

function planar_id()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 7,
        3 => 4,
        4 => 6,
        5 => 12,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 2),
        2 => HalfEdge(2, 1),
        3 => HalfEdge(1, 3),
        4 => HalfEdge(3, 1),
        5 => HalfEdge(1, 4),
        6 => HalfEdge(4, 1),
        7 => HalfEdge(2, 5),
        8 => HalfEdge(5, 2),
        9 => HalfEdge(3, 5),
        10 => HalfEdge(5, 3),
        11 => HalfEdge(4, 5),
        12 => HalfEdge(5, 4),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 3,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,
        11 => 0,
        8 => 0,

        1 => 1,
        7 => 1,
        10 => 1,
        4 => 1,

        3 => 2,
        9 => 2,
        12 => 2,
        6 => 2,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 11,
        11 => 8,
        8 => 2,

        1 => 7,
        7 => 10,
        10 => 4,
        4 => 1,

        3 => 9,
        9 => 12,
        12 => 6,
        6 => 3,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
        7 => 8,
        8 => 7,
        9 => 10,
        10 => 9,
        11 => 12,
        12 => 11,
    )
    vs_isolated = Dict{Int, Int}()
    id = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        5, 12, 2
    )
    return id
end

function tait_id()
    g = planar_id()
    quon_params = Dict{Int, QuonParam{QuonComplex}}()
    inputs = Int[1]
    outputs = Int[5]
    genuses = Set([2, 4])
    locations = Dict(
        1 => (1.0, 0.0),
        2 => (0.0, 1.0),
        3 => (1.0, 1.0),
        4 => (2.0, 1.0),
        5 => (1.0, 2.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end

function tait_hadamard()
    z1 = tait_rz(HalfPi)
    x = tait_rx(HalfPi)
    z2 = tait_rz(HalfPi)
    contract!(z1, x)
    contract!(z1, z2)
    z1.locations[13] = (0.0, 1.5)
    z1.locations[15] = (2.0, 1.5)
    return z1
end

function planar_copy()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 7,
        3 => 13,
        4 => 8,
        5 => 6,
        6 => 2,
        7 => 4,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 6),
        2 => HalfEdge(6, 1),
        3 => HalfEdge(1, 7),
        4 => HalfEdge(7, 1),
        5 => HalfEdge(1, 5),
        6 => HalfEdge(5, 1),
        7 => HalfEdge(2, 4),
        8 => HalfEdge(4, 2),
        9 => HalfEdge(2, 7),
        10 => HalfEdge(7, 2),
        11 => HalfEdge(2, 6),
        12 => HalfEdge(6, 2),
        13 => HalfEdge(3, 5),
        14 => HalfEdge(5, 3),
        15 => HalfEdge(3, 7),
        16 => HalfEdge(7, 3),
        17 => HalfEdge(3, 4),
        18 => HalfEdge(4, 3),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 7,
        3 => 3,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,
        14 => 0,
        17 => 0,
        8 => 0,
        11 => 0,

        1 => 1,
        12 => 1,
        9 => 1,
        4 => 1,

        7 => 2,
        18 => 2,
        15 => 2,
        10 => 2,

        3 => 3,
        16 => 3,
        13 => 3,
        6 => 3,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 14,
        14 => 17,
        17 => 8,
        8 => 11,
        11 => 2,

        1 => 12,
        12 => 9,
        9 => 4,
        4 => 1,

        7 => 18,
        18 => 15,
        15 => 10,
        10 => 7,

        3 => 16,
        16 => 13,
        13 => 6,
        6 => 3,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
        7 => 8,
        8 => 7,
        9 => 10,
        10 => 9,
        11 => 12,
        12 => 11,
        13 => 14,
        14 => 13,
        15 => 16,
        16 => 15,
        17 => 18,
        18 => 17,
    )
    vs_isolated = Dict{Int, Int}()
    copy_tensor = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        7, 18, 3
    )
    return copy_tensor
end

function tait_copy()
    g = planar_copy()
    quon_params = Dict{Int, QuonParam{QuonComplex}}()
    inputs = Int[1]
    outputs = Int[2, 3]
    genuses = Set([4, 5, 6])
    locations = Dict(
        1 => (1.0, 0.0),
        2 => (0.0, 2.0),
        3 => (2.0, 2.0),
        4 => (1.0, 2.0),
        5 => (2.0, 1.0),
        6 => (0.0, 1.0),
        7 => (1.0, 1.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end

function tait_cz()
    h = tait_hadamard()
    c1 = tait_copy()
    c2 = tait_copy()
    contract!(c1, h, [3], [1])
    contract!(c1, c2, [23], [2])
    c1.locations[2] = (1.0, 3.0)
    c1.locations[6] = (0.0, 1.0)
    c1.locations[29] = (2.0, 0.0)
    c1.locations[24] = (3.0, 0.0)
    c1.locations[26] = (3.0, 3.0)
    c1.locations[28] = (4.0, 1.0)
    c1.locations[30] = (3.0, 1.0)
    c1.locations[27] = (2.0, 2.0)

    return c1
end

function planar_zero_state()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 2,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 2),
        2 => HalfEdge(2, 1),
        3 => HalfEdge(1, 2),
        4 => HalfEdge(2, 1),
        5 => HalfEdge(1, 2),
        6 => HalfEdge(2, 1),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 3,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,

        1 => 1,
        4 => 1,

        3 => 2,
        6 => 2,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 2,

        1 => 4,
        4 => 1,

        3 => 6,
        6 => 3,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
    )
    vs_isolated = Dict{Int, Int}()
    zero_state = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        2, 6, 2
    )
    return zero_state
end

function tait_zero_state()
    g = planar_zero_state()
    quon_params = Dict{Int, QuonParam{QuonComplex}}()
    inputs = Int[]
    outputs = Int[2]
    genuses = Set([1])
    locations = Dict(
        1 => (1.0, 0.0),
        2 => (1.0, 1.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end

function planar_swap()
    v2he = Dict{Int, Int}(
        1 => 1,
        2 => 7,
        3 => 13,
        4 => 19,
        5 => 2,
        6 => 8,
        7 => 14,
        8 => 6,
        9 => 4,
        10 => 10,
        11 => 16,
        12 => 22,
        13 => 25,
        14 => 33,
        15 => 41,
        16 => 49,
        17 => 30,
    )
    half_edges = Dict{Int, HalfEdge}(
        1 => HalfEdge(1, 5),
        2 => HalfEdge(5, 1),
        3 => HalfEdge(1, 9),
        4 => HalfEdge(9, 1),
        5 => HalfEdge(1, 8),
        6 => HalfEdge(8, 1),
        7 => HalfEdge(2, 6),
        8 => HalfEdge(6, 2),
        9 => HalfEdge(2, 10),
        10 => HalfEdge(10, 2),
        11 => HalfEdge(2, 5),
        12 => HalfEdge(5, 2),
        13 => HalfEdge(3, 7),
        14 => HalfEdge(7, 3),
        15 => HalfEdge(3, 11),
        16 => HalfEdge(11, 3),
        17 => HalfEdge(3, 6),
        18 => HalfEdge(6, 3),
        19 => HalfEdge(4, 8),
        20 => HalfEdge(8, 4),
        21 => HalfEdge(4, 12),
        22 => HalfEdge(12, 4),
        23 => HalfEdge(4, 7),
        24 => HalfEdge(7, 4),
        25 => HalfEdge(13, 5),
        26 => HalfEdge(5, 13),
        27 => HalfEdge(13, 10),
        28 => HalfEdge(10, 13),
        29 => HalfEdge(13, 17),
        30 => HalfEdge(17, 13),
        31 => HalfEdge(13, 9),
        32 => HalfEdge(9, 13),
        33 => HalfEdge(14, 6),
        34 => HalfEdge(6, 14),
        35 => HalfEdge(14, 11),
        36 => HalfEdge(11, 14),
        37 => HalfEdge(14, 17),
        38 => HalfEdge(17, 14),
        39 => HalfEdge(14, 10),
        40 => HalfEdge(10, 14),
        41 => HalfEdge(15, 7),
        42 => HalfEdge(7, 15),
        43 => HalfEdge(15, 12),
        44 => HalfEdge(12, 15),
        45 => HalfEdge(15, 17),
        46 => HalfEdge(17, 15),
        47 => HalfEdge(15, 11),
        48 => HalfEdge(11, 15),
        49 => HalfEdge(16, 8),
        50 => HalfEdge(8, 16),
        51 => HalfEdge(16, 9),
        52 => HalfEdge(9, 16),
        53 => HalfEdge(16, 17),
        54 => HalfEdge(17, 16),
        55 => HalfEdge(16, 12),
        56 => HalfEdge(12, 16),
    )

    f2he = Dict{Int, Int}(
        0 => 2,
        1 => 1,
        2 => 3,
        3 => 7,
        4 => 9,
        5 => 13,
        6 => 15,
        7 => 19,
        8 => 21,
        9 => 29,
        10 => 27,
        11 => 35,
        12 => 43,
    )
    he2f = Dict{Int, Int}(
        2 => 0,
        5 => 0,
        20 => 0,
        23 => 0,
        14 => 0,
        17 => 0,
        8 => 0,
        11 => 0,

        1 => 1,
        26 => 1,
        31 => 1,
        4 => 1,

        3 => 2,
        52 => 2,
        49 => 2,
        6 => 2,

        7 => 3,
        34 => 3,
        39 => 3,
        10 => 3,

        9 => 4,
        28 => 4,
        25 => 4,
        12 => 4,

        13 => 5,
        42 => 5,
        47 => 5,
        16 => 5,

        15 => 6,
        36 => 6,
        33 => 6,
        18 => 6,

        19 => 7,
        50 => 7,
        55 => 7,
        22 => 7,

        21 => 8,
        44 => 8,
        41 => 8,
        24 => 8,

        29 => 9,
        54 => 9,
        51 => 9,
        32 => 9,

        27 => 10,
        40 => 10,
        37 => 10,
        30 => 10,

        35 => 11,
        48 => 11,
        45 => 11,
        38 => 11,

        43 => 12,
        56 => 12,
        53 => 12,
        46 => 12,
    )
    next = Dict{Int, Int}(
        2 => 5,
        5 => 20,
        20 => 23,
        23 => 14,
        14 => 17,
        17 => 8,
        8 => 11,
        11 => 2,

        1 => 26,
        26 => 31,
        31 => 4,
        4 => 1,

        3 => 52,
        52 => 49,
        49 => 6,
        6 => 3,

        7 => 34,
        34 => 39,
        39 => 10,
        10 => 7,

        9 => 28,
        28 => 25,
        25 => 12,
        12 => 9,

        13 => 42,
        42 => 47,
        47 => 16,
        16 => 13,

        15 => 36,
        36 => 33,
        33 => 18,
        18 => 15,

        19 => 50,
        50 => 55,
        55 => 22,
        22 => 19,

        21 => 44,
        44 => 41,
        41 => 24,
        24 => 21,

        29 => 54,
        54 => 51,
        51 => 32,
        32 => 29,

        27 => 40,
        40 => 37,
        37 => 30,
        30 => 27,

        35 => 48,
        48 => 45,
        45 => 38,
        38 => 35,

        43 => 56,
        56 => 53,
        53 => 46,
        46 => 43,
    )
    twin = Dict{Int, Int}(
        1 => 2,
        2 => 1,
        3 => 4,
        4 => 3,
        5 => 6,
        6 => 5,
        7 => 8,
        8 => 7,
        9 => 10,
        10 => 9,
        11 => 12,
        12 => 11,
        13 => 14,
        14 => 13,
        15 => 16,
        16 => 15,
        17 => 18,
        18 => 17,
        19 => 20,
        20 => 19,
        21 => 22,
        22 => 21,
        23 => 24,
        24 => 23,
        25 => 26,
        26 => 25,
        27 => 28,
        28 => 27,
        29 => 30,
        30 => 29,
        31 => 32,
        32 => 31,
        33 => 34,
        34 => 33,
        35 => 36,
        36 => 35,
        37 => 38,
        38 => 37,
        39 => 40,
        40 => 39,
        41 => 42,
        42 => 41,
        43 => 44,
        44 => 43,
        45 => 46,
        46 => 45,
        47 => 48,
        48 => 47,
        49 => 50,
        50 => 49,
        51 => 52,
        52 => 51,
        53 => 54,
        54 => 53,
        55 => 56,
        56 => 55,
    )
    vs_isolated = Dict{Int, Int}()
    swap = PlanarMultigraph(v2he, half_edges,
        f2he, he2f,
        next, twin, vs_isolated,
        17, 56, 12
    )
    return swap
end

function tait_swap()
    g = planar_swap()
    p_para = QuonParam{QuonComplex}(HalfPi, true)
    p_orth = QuonParam{QuonComplex}(HalfPi, false)
    quon_params = Dict{Int, QuonParam{QuonComplex}}(
        25 => p_para,
        26 => p_para,
        27 => p_orth,
        28 => p_orth,
        29 => p_para,
        30 => p_para,
        31 => p_orth,
        32 => p_orth,

        33 => p_orth,
        34 => p_orth,
        35 => p_para,
        36 => p_para,
        37 => p_orth,
        38 => p_orth,
        39 => p_para,
        40 => p_para,

        41 => p_para,
        42 => p_para,
        43 => p_orth,
        44 => p_orth,
        45 => p_para,
        46 => p_para,
        47 => p_orth,
        48 => p_orth,

        49 => p_orth,
        50 => p_orth,
        51 => p_para,
        52 => p_para,
        53 => p_orth,
        54 => p_orth,
        55 => p_para,
        56 => p_para,
    )
    inputs = Int[1, 4]
    outputs = Int[2, 3]
    genuses = Set([5, 6, 7, 8])
    locations = Dict(
        1 => (0.0, 0.0),
        8 => (2.0, 0.0),
        4 => (4.0, 0.0),
        9 => (1.0, 1.0),
        16 => (2.0, 1.0),
        12 => (3.0, 1.0),
        5 => (0.0, 2.0),
        13 => (1.0, 2.0),
        17 => (2.0, 2.0),
        15 => (3.0, 2.0),
        7 => (4.0, 2.0),
        10 => (1.0, 3.0),
        14 => (2.0, 3.0),
        11 => (3.0, 3.0),
        2 => (0.0, 4.0),
        6 => (2.0, 4.0),
        3 => (4.0, 4.0),
    )

    return Tait{QuonParam{QuonComplex}}(g, quon_params, inputs, outputs, genuses, locations)
end

function tait_cnot(ctrl, loc)
    z_copy = tait_copy()
    x_copy = contract!(tait_hadamard(), tait_copy())
    h = tait_hadamard()
    for v in copy(x_copy.outputs)
        contract!(x_copy, h, [v], copy(h.inputs))
    end

    if ctrl < loc
        push!(x_copy.inputs, pop!(x_copy.outputs))
        left, right = z_copy, x_copy
    elseif ctrl > loc
        push!(z_copy.inputs, pop!(z_copy.outputs))
        left, right = x_copy, z_copy
    else
        error("`ctrl` and `loc` should be different")
    end

    s = tait_swap()
    for _ in 1:(abs(ctrl - loc) - 1)
        contract!(left, s, [left.outputs[end]], [s.inputs[1]])
    end

    return contract!(left, right, [left.outputs[end]], [right.inputs[1]])
end

function tait_cz(ctrl, loc)
    @assert ctrl != loc
    left = tait_copy()
    right = tait_copy()
    h = tait_hadamard()
    contract!(left, h, [left.outputs[end]], copy(h.inputs))
    push!(right.inputs, pop!(right.outputs))
    s = tait_swap()
    for _ in 1:(abs(ctrl - loc) - 1)
        contract!(left, s, [left.outputs[end]], [s.inputs[1]])
    end

    return contract!(left, right, [left.outputs[end]], [right.inputs[1]])
end


text:
using JuMP
using CPLEX
include("parser.jl")
include("base.jl")

ext: jl

max_stars_repo_name: VSpitzer/RORT

max_issues_repo_name: VSpitzer/RORT

max_forks_repo_name: VSpitzer/RORT


text:
"""
This file contains all the front-end scripts in ClimateSatellite.jl that are for general use regardless of satellite and product type, which includes:
    * detection of home directory on machine
    * opening an FTP request to arthuhou and jsimpson (GPM / TRMM data) servers
    * creation of folders (both data folders and temporary directories)

"""

## FTP Functions
function pmmftpopen(server::AbstractString,email::AbstractString)
    @info "$(Dates.now()) - Opening FTP request to $(server).pps.eosdis.nasa.gov."
    return FTP("ftp://$(email):$(email)@$(server).pps.eosdis.nasa.gov")
end

function pmmftpclose(ftp)
    @info "$(Dates.now()) - Closing FTP request."
    close(ftp)
end

## Other Functions
function isprod(info::Dict,product::AbstractString)
    occursin(product,info["short"])
end

function clisatextractdate(startdate::TimeType,finish::TimeType);

    yrs = Dates.year(start);  mos = Dates.month(start);  dys = Dates.day(start);
    yrf = Dates.year(finish); mof = Dates.month(finish); dyf = Dates.day(finish);
    ndy = Dates.value((finish-start)/Dates.day(1));
    dvecs = Date(yrs,mos); dvecf = Date(yrf,mof);

    dvec = convert(Array,dvecs:Month(1):dvecf);

    return dvec,dys,dyf,ndy

end

function clisatrmtmp(flist::Array{<:AbstractString,2},fol::AbstractString)

    for ii = 1 : length(flist)
        fii = joinpath(fol,"$(flist[ii])"); if isfile(fii); rm(fii) end
    end

end

function clisattmpfol(info::Dict)

    fol = joinpath(info["root"],"tmp");

    if !isdir(fol)
        @debug "$(Dates.now()) - Creating temporary directory $(fol)."; mkpath(fol);
    end

    return fol

end

## DateString Aliasing
yrmo2dir(date::TimeType) = Dates.format(date,dateformat"yyyy/mm")
yrmo2str(date::TimeType) = Dates.format(date,dateformat"yyyymm")
yr2str(date::TimeType)   = Dates.format(date,dateformat"yyyy")
ymd2str(date::TimeType)  = Dates.format(date,dateformat"yyyymmdd")

## Real2Integer Conversion for Packing
function real2int16!(
    outarray::Array{Int16}, inarray::Array{<:Real};
    offset::Real, scale::Real
)

    if size(outarray) != size(inarray)
        dout = [i for i in size(outarray)];
        din  = [i for i in size(inarray)];
        if (dout[1:end-1] != din[1:end] && dout[1:end] != din[1:end-1]) ||
            prod(dout) != prod(din)
            error("$(Dates.now()) - output array is not of the same size as the input array")
        end
    end

    for ii = 1 : length(inarray)

        inarray[ii] = (inarray[ii] - offset) / scale

        if inarray[ii] < -32767; inarray[ii] = -32768
        elseif inarray[ii] > 32767; inarray[ii] = -32768
        end

        outarray[ii] = round(Int16,inarray[ii])

    end

    return

end

## Check email
function checkemail(email::AbstractString)
    if email == ""
        error("$(Dates.now()) - Usage of this dataset requires an email address for login.  However, no email was provided.")
    end
end


text:
#################################################################################
# Comments
#
# - Ideally the net_injection variables would be bounded.  This can be done using an adhoc data model extention
# - the `instantiate_*_expr_model` functions combine `PM.instantiate_model` and the `build_*` methods
#################################################################################
# Model Definitions

""
function instantiate_nip_expr_model(data::Dict{String, Any}, model_constructor; kwargs...)
    return PM.instantiate_model(data, model_constructor, instantiate_nip_expr; kwargs...)
end

""
# replicates PM.build_mn_opf
function instantiate_nip_expr(pm::PM.AbstractPowerModel)
    for n in eachindex(PM.nws(pm))
        @assert !PM.ismulticonductor(pm, nw = n)
        PM.variable_bus_voltage(pm, nw = n)
        PM.variable_branch_power(pm, nw = n; bounded = false)
        PM.variable_dcline_power(pm, nw = n)

        PM.constraint_model_voltage(pm, nw = n)

        for i in PM.ids(pm, :ref_buses, nw = n)
            PM.constraint_theta_ref(pm, i, nw = n)
        end

        for i in PM.ids(pm, :bus, nw = n)
            constraint_power_balance_ni_expr(pm, i, nw = n)
        end

        for i in PM.ids(pm, :branch, nw = n)
            PM.constraint_ohms_yt_from(pm, i, nw = n)
            PM.constraint_ohms_yt_to(pm, i, nw = n)

            PM.constraint_voltage_angle_difference(pm, i, nw = n)
        end

        for i in PM.ids(pm, :dcline, nw = n)
            PM.constraint_dcline_power_losses(pm, i, nw = n)
        end
    end

    return
end

""
function instantiate_nip_ptdf_expr_model(
    data::Dict{String, Any},
    model_constructor;
    kwargs...,
)
    return PM.instantiate_model(
        data,
        PM.DCPPowerModel,
        instantiate_nip_ptdf_expr;
        ref_extensions = [PM.ref_add_connected_components!, PM.ref_add_sm!],
        kwargs...,
    )
end

""
# replicates PM.build_opf_ptdf
function instantiate_nip_ptdf_expr(pm::PM.AbstractPowerModel)
    for n in eachindex(PM.nws(pm))
        @assert !PM.ismulticonductor(pm, nw = n)

        #PM.variable_gen_power(pm) #connect P__* with these

        for i in PM.ids(pm, :bus, nw = n)
            if !haskey(PM.var(pm, n), :inj_p)
                PM.var(pm, n)[:inj_p] = Dict{Int, Any}()
            end
            PM.var(pm, n, :inj_p)[i] = PM.ref(pm, :bus, i, nw = n)["inj_p"] # use :nodal_balance_expr
        end

        PM.constraint_model_voltage(pm, nw = n)

        # this constraint is implicit in this model
        #for i in PM.ids(pm, :ref_buses, nw = n)
        #    PM.constraint_theta_ref(pm, i, nw = n)
        #end

        # done later with PSI copper_plate
        #for i in PM.ids(pm, :bus, nw = n)
        #    constraint_power_balance_ni_expr(pm, i, nw = n)
        #end
        for (i, branch) in PM.ref(pm, :branch, nw = n)
            if haskey(branch, "rate_a")
                PM.expression_branch_power_ohms_yt_from_ptdf(pm, i, nw = n)
                PM.expression_branch_power_ohms_yt_to_ptdf(pm, i, nw = n)
            end

            # done in PSI construct_branch!
            #PM.constraint_thermal_limit_from(pm, i, nw = n)
            #PM.constraint_thermal_limit_to(pm, i, nw = n)
        end
    end

    return
end

function instantiate_bfp_expr_model(data::Dict{String, Any}, model_constructor; kwargs...)
    return PM.instantiate_model(data, model_constructor, instantiate_bfp_expr; kwargs...)
end

""
# replicates PM.build_mn_opf_bf_strg
function instantiate_bfp_expr(pm::PM.AbstractPowerModel)
    for n in eachindex(PM.nws(pm))
        @assert !PM.ismulticonductor(pm, nw = n)
        PM.variable_bus_voltage(pm, nw = n)
        PM.variable_branch_power(pm, nw = n; bounded = false)
        PM.variable_dcline_power(pm, nw = n)

        PM.constraint_model_current(pm, nw = n)

        for i in PM.ids(pm, :ref_buses, nw = n)
            PM.constraint_theta_ref(pm, i, nw = n)
        end

        for i in PM.ids(pm, :bus, nw = n)
            constraint_power_balance_ni_expr(pm, i, nw = n)
        end

        for i in PM.ids(pm, :branch, nw = n)
            PM.constraint_power_losses(pm, i, nw = n)
            PM.constraint_voltage_magnitude_difference(pm, i, nw = n)

            PM.constraint_voltage_angle_difference(pm, i, nw = n)
        end

        for i in PM.ids(pm, :dcline, nw = n)
            PM.constraint_dcline_power_losses(pm, i, nw = n)
        end
    end

    return
end

function instantiate_vip_expr_model(data::Dict{String, Any}, model_constructor; kwargs...)
    throw(error("VI Models not currently supported"))
end

#################################################################################
# Model Extention Functions

""
function constraint_power_balance_ni_expr(
    pm::PM.AbstractPowerModel,
    i::Int;
    nw::Int = pm.cnw,
)
    if !haskey(PM.con(pm, nw), :power_balance_p)
        PM.con(pm, nw)[:power_balance_p] = Dict{Int, JuMP.ConstraintRef}()
    end
    if !haskey(PM.con(pm, nw), :power_balance_q)
        PM.con(pm, nw)[:power_balance_q] = Dict{Int, JuMP.ConstraintRef}()
    end

    bus_arcs = PM.ref(pm, nw, :bus_arcs, i)
    bus_arcs_dc = PM.ref(pm, nw, :bus_arcs_dc, i)

    inj_p_expr = PM.ref(pm, nw, :bus, i, "inj_p")
    inj_q_expr = PM.ref(pm, nw, :bus, i, "inj_q")

    constraint_power_balance_ni_expr(
        pm,
        nw,
        i,
        bus_arcs,
        bus_arcs_dc,
        inj_p_expr,
        inj_q_expr,
    )

    return
end

""
function constraint_power_balance_ni_expr(
    pm::PM.AbstractPowerModel,
    n::Int,
    i::Int,
    bus_arcs,
    bus_arcs_dc,
    inj_p_expr,
    inj_q_expr,
)
    p = PM.var(pm, n, :p)
    q = PM.var(pm, n, :q)
    p_dc = PM.var(pm, n, :p_dc)
    q_dc = PM.var(pm, n, :q_dc)

    PM.con(pm, n, :power_balance_p)[i] = JuMP.@constraint(
        pm.model,
        sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == inj_p_expr
    )
    PM.con(pm, n, :power_balance_q)[i] = JuMP.@constraint(
        pm.model,
        sum(q[a] for a in bus_arcs) + sum(q_dc[a_dc] for a_dc in bus_arcs_dc) == inj_q_expr
    )

    return
end

""
function constraint_current_balance_ni_expr(
    pm::PM.AbstractPowerModel,
    i::Int;
    nw::Int = pm.cnw,
)
    if !haskey(PM.con(pm, nw), :kcl_cr)
        PM.con(pm, nw)[:kcl_cr] = Dict{Int, JuMP.ConstraintRef}()
    end
    if !haskey(PM.con(pm, nw), :kcl_ci)
        PM.con(pm, nw)[:kcl_ci] = Dict{Int, JuMP.ConstraintRef}()
    end

    bus_arcs = PM.ref(pm, nw, :bus_arcs, i)
    bus_arcs_dc = PM.ref(pm, nw, :bus_arcs_dc, i)

    inj_p_expr = PM.ref(pm, nw, :bus, i, "inj_p")
    inj_q_expr = PM.ref(pm, nw, :bus, i, "inj_q")

    constraint_current_balance_ni_expr(
        pm,
        nw,
        i,
        bus_arcs,
        bus_arcs_dc,
        inj_p_expr,
        inj_q_expr,
    )

    return
end

""
function constraint_current_balance_ni_expr(
    pm::PM.AbstractPowerModel,
    n::Int,
    i::Int,
    bus_arcs,
    bus_arcs_dc,
    inj_p_expr,
    inj_q_expr,
)
    p = PM.var(pm, n, :p)
    q = PM.var(pm, n, :q)
    p_dc = PM.var(pm, n, :p_dc)
    q_dc = PM.var(pm, n, :q_dc)

    PM.con(pm, n, :power_balance_p)[i] = JuMP.@constraint(
        pm.model,
        sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == inj_p_expr
    )
    PM.con(pm, n, :power_balance_q)[i] = JuMP.@constraint(
        pm.model,
        sum(q[a] for a in bus_arcs) + sum(q_dc[a_dc] for a_dc in bus_arcs_dc) == inj_q_expr
    )

    return
end

"active power only models ignore reactive power variables"
function variable_reactive_net_injection(pm::PM.AbstractActivePowerModel; kwargs...)
    return
end

""
function constraint_power_balance_ni_expr(
    pm::PM.AbstractActivePowerModel,
    n::Int,
    i::Int,
    bus_arcs,
    bus_arcs_dc,
    inj_p_expr,
    _,
)
    p = PM.var(pm, n, :p)
    p_dc = PM.var(pm, n, :p_dc)

    PM.con(pm, n, :power_balance_p)[i] = JuMP.@constraint(
        pm.model,
        sum(p[a] for a in bus_arcs) + sum(p_dc[a_dc] for a_dc in bus_arcs_dc) == inj_p_expr
    )

    return
end

""
function powermodels_network!(
    optimization_container::OptimizationContainer,
    system_formulation::Type{S},
    sys::PSY.System,
    template::OperationsProblemTemplate,
    instantiate_model = instantiate_nip_expr_model,
) where {S <: PM.AbstractPowerModel}
    time_steps = model_time_steps(optimization_container)
    pm_data, PM_map = pass_to_pm(sys, template, time_steps[end])
    buses = PSY.get_components(PSY.Bus, sys)

    remove_undef!(optimization_container.expressions[:nodal_balance_active])
    remove_undef!(optimization_container.expressions[:nodal_balance_reactive])

    for t in time_steps, bus in buses
        pm_data["nw"]["$(t)"]["bus"]["$(bus.number)"]["inj_p"] =
            optimization_container.expressions[:nodal_balance_active][bus.number, t]
        pm_data["nw"]["$(t)"]["bus"]["$(bus.number)"]["inj_q"] =
            optimization_container.expressions[:nodal_balance_reactive][bus.number, t]
    end

    optimization_container.pm = instantiate_model(
        pm_data,
        system_formulation,
        jump_model = optimization_container.JuMPmodel,
    )
    optimization_container.pm.ext[:PMmap] = PM_map

    return
end

""
function powermodels_network!(
    optimization_container::OptimizationContainer,
    system_formulation::Type{S},
    sys::PSY.System,
    template::OperationsProblemTemplate,
    instantiate_model = instantiate_nip_expr_model,
) where {S <: PM.AbstractActivePowerModel}
    time_steps = model_time_steps(optimization_container)
    pm_data, PM_map = pass_to_pm(sys, template, time_steps[end])
    buses = PSY.get_components(PSY.Bus, sys)

    remove_undef!(optimization_container.expressions[:nodal_balance_active])

    for t in time_steps, bus in buses
        pm_data["nw"]["$(t)"]["bus"]["$(PSY.get_number(bus))"]["inj_p"] =
            optimization_container.expressions[:nodal_balance_active][
                PSY.get_number(bus),
                t,
            ]
        # pm_data["nw"]["$(t)"]["bus"]["$(bus.number)"]["inj_q"] = 0.0
    end

    optimization_container.pm = instantiate_model(
        pm_data,
        system_formulation,
        jump_model = optimization_container.JuMPmodel,
    )
    optimization_container.pm.ext[:PMmap] = PM_map

    return
end

#### PM accessor functions ########

function PMvarmap(::Type{S}) where {S <: PM.AbstractDCPModel}
    pm_var_map = Dict{Type, Dict{Symbol, Union{String, NamedTuple}}}()

    pm_var_map[PSY.Bus] = Dict(:va => THETA)
    pm_var_map[PSY.ACBranch] =
        Dict(:p => (from_to = FlowActivePowerVariable, to_from = nothing))
    pm_var_map[PSY.DCBranch] =
        Dict(:p_dc => (from_to = FlowActivePowerVariable, to_from = nothing))

    return pm_var_map
end

function PMvarmap(::Type{S}) where {S <: PM.AbstractActivePowerModel}
    pm_var_map = Dict{Type, Dict{Symbol, Union{String, NamedTuple}}}()

    pm_var_map[PSY.Bus] = Dict(:va => THETA)
    pm_var_map[PSY.ACBranch] = Dict(:p => FlowActivePowerFromToVariable)
    pm_var_map[PSY.DCBranch] = Dict(
        :p_dc => (
            from_to = FlowActivePowerFromToVariable,
            to_from = FlowActivePowerToFromVariable,
        ),
    )

    return pm_var_map
end

function PMvarmap(::Type{PTDFPowerModel})
    pm_var_map = Dict{Type, Dict{Symbol, Union{String, NamedTuple}}}()

    pm_var_map[PSY.Bus] = Dict()
    pm_var_map[PSY.ACBranch] = Dict()
    pm_var_map[PSY.DCBranch] = Dict()

    return pm_var_map
end

function PMvarmap(::Type{S}) where {S <: PM.AbstractPowerModel}
    pm_var_map = Dict{Type, Dict{Symbol, Union{String, NamedTuple}}}()

    pm_var_map[PSY.Bus] = Dict(:va => THETA, :vm => VM)
    pm_var_map[PSY.ACBranch] = Dict(
        :p => (
            from_to = FlowActivePowerFromToVariable,
            to_from = FlowActivePowerToFromVariable,
        ),
        :q => (
            from_to = FlowReactivePowerFromToVariable,
            to_from = FlowReactivePowerToFromVariable,
        ),
    )
    pm_var_map[PSY.DCBranch] = Dict(
        :p_dc => (from_to = FlowActivePowerVariable, to_from = nothing),
        :q_dc => (
            from_to = FlowReactivePowerFromToVariable,
            to_from = FlowReactivePowerToFromVariable,
        ),
    )

    return pm_var_map
end

function PMconmap(::Type{S}) where {S <: PM.AbstractActivePowerModel}
    pm_con_map = Dict{Type, Dict{Symbol, String}}()

    pm_con_map[PSY.Bus] = Dict(:power_balance_p => NODAL_BALANCE_ACTIVE)
    return pm_con_map
end

function PMconmap(::Type{S}) where {S <: PM.AbstractPowerModel}
    pm_con_map = Dict{Type, Dict{Symbol, String}}()

    pm_con_map[PSY.Bus] = Dict(
        :power_balance_p => NODAL_BALANCE_ACTIVE,
        :power_balance_q => NODAL_BALANCE_REACTIVE,
    )
    return pm_con_map
end

function PMexprmap(::Type{S}) where {S <: PM.AbstractPowerModel}
    pm_expr_map = Dict{
        Type,
        NamedTuple{
            (:pm_expr, :psi_con),
            Tuple{Dict{Symbol, Union{String, NamedTuple}}, Symbol},
        },
    }()

    return pm_expr_map
end

function PMexprmap(::Type{PTDFPowerModel})
    pm_expr_map = Dict{
        Type,
        NamedTuple{
            (:pm_expr, :psi_con),
            Tuple{Dict{Symbol, Union{String, NamedTuple}}, Symbol},
        },
    }()

    pm_expr_map[PSY.ACBranch] = (
        pm_expr = Dict(:p => (from_to = FlowActivePowerVariable, to_from = nothing)),
        psi_con = Symbol(NETWORK_FLOW),
    )

    return pm_expr_map
end

function add_pm_var_refs!(
    optimization_container::OptimizationContainer,
    system_formulation::Type{S},
    ::PSY.System,
) where {S <: PM.AbstractPowerModel}
    time_steps = model_time_steps(optimization_container)
    bus_dict = optimization_container.pm.ext[:PMmap].bus
    ACbranch_dict = optimization_container.pm.ext[:PMmap].arcs
    ACbranch_types = typeof.(values(ACbranch_dict))
    DCbranch_dict = optimization_container.pm.ext[:PMmap].arcs_dc
    DCbranch_types = typeof.(values(DCbranch_dict))

    pm_var_types = keys(PM.var(optimization_container.pm, 1))

    pm_var_map = PMvarmap(system_formulation)

    for (pm_v, ps_v) in pm_var_map[PSY.Bus]
        if pm_v in pm_var_types
            container = PSI.container_spec(
                JuMP.VariableRef,
                [PSY.get_name(b) for b in values(bus_dict)],
                time_steps,
            )
            assign_variable!(optimization_container, ps_v, PSY.Bus, container)
            for t in time_steps, (pm_bus, bus) in bus_dict
                name = PSY.get_name(bus)
                container[name, t] = PM.var(optimization_container.pm, t, pm_v)[pm_bus] # pm_vars[pm_v][pm_bus]
            end
        end
    end

    add_pm_var_refs!(
        optimization_container,
        PSY.ACBranch,
        ACbranch_types,
        ACbranch_dict,
        pm_var_map,
        pm_var_types,
        time_steps,
    )
    add_pm_var_refs!(
        optimization_container,
        PSY.DCBranch,
        DCbranch_types,
        DCbranch_dict,
        pm_var_map,
        pm_var_types,
        time_steps,
    )
end

function add_pm_var_refs!(
    optimization_container::OptimizationContainer,
    d_class::Type,
    device_types::Vector,
    pm_map::Dict,
    pm_var_map::Dict,
    pm_var_types::Base.KeySet,
    time_steps::UnitRange{Int},
)
    for d_type in Set(device_types)
        devices = [d for d in pm_map if typeof(d[2]) == d_type]
        for (pm_v, ps_v) in pm_var_map[d_class]
            if pm_v in pm_var_types
                for dir in fieldnames(typeof(ps_v))
                    var_type = getfield(ps_v, dir)
                    var_type === nothing && continue
                    container = PSI.container_spec(
                        JuMP.VariableRef,
                        [PSY.get_name(d[2]) for d in devices],
                        time_steps,
                    )
                    assign_variable!(
                        optimization_container,
                        make_variable_name(var_type, d_type),
                        container,
                    )
                    for t in time_steps, (pm_d, d) in devices
                        var =
                            PM.var(optimization_container.pm, t, pm_v, getfield(pm_d, dir))
                        container[PSY.get_name(d), t] = var
                    end
                end
            end
        end
    end
end

function add_pm_con_refs!(
    optimization_container::OptimizationContainer,
    system_formulation::Type{S},
    ::PSY.System,
) where {S <: PM.AbstractPowerModel}
    time_steps = model_time_steps(optimization_container)
    bus_dict = optimization_container.pm.ext[:PMmap].bus

    pm_con_names = [
        k for k in keys(PM.con(optimization_container.pm, 1)) if
        !isempty(PM.con(optimization_container.pm, 1, k))
    ]

    pm_con_map = PMconmap(system_formulation)
    for (pm_v, ps_v) in pm_con_map[PSY.Bus]
        if pm_v in pm_con_names
            container = PSI.add_cons_container!(
                optimization_container,
                make_constraint_name(ps_v, PSY.Bus),
                [PSY.get_name(b) for b in values(bus_dict)],
                time_steps,
            )
            for t in time_steps, (pm_bus, bus) in bus_dict
                name = PSY.get_name(bus)
                container[name, t] = PM.con(optimization_container.pm, t, pm_v)[pm_bus]
            end
        end
    end
end

function add_pm_expr_refs!(
    optimization_container::OptimizationContainer,
    system_formulation::Type{S},
    ::PSY.System,
) where {S <: PM.AbstractPowerModel}
    time_steps = model_time_steps(optimization_container)

    ACbranch_dict = optimization_container.pm.ext[:PMmap].arcs
    ACbranch_types = typeof.(values(ACbranch_dict))

    pm_var_types = keys(PM.var(optimization_container.pm, 1))
    pm_expr_map = PMexprmap(system_formulation)

    add_pm_expr_refs!(
        optimization_container,
        PSY.ACBranch,
        ACbranch_types,
        ACbranch_dict,
        pm_expr_map,
        pm_var_types,
        time_steps,
    )
end

function add_pm_expr_refs!(
    optimization_container::OptimizationContainer,
    d_class::Type,
    device_types::Vector,
    pm_map::Dict,
    pm_expr_map::Dict,
    pm_var_types::Base.KeySet,
    time_steps::UnitRange{Int},
)
    for d_type in Set(device_types)
        for (pm_expr_var, ps_v) in pm_expr_map[d_class].pm_expr
            if pm_expr_var in pm_var_types
                pm_devices = keys(PM.var(optimization_container.pm, pm_expr_var, nw = 1))
                mapped_pm_devices = Vector()
                mapped_ps_devices = Vector{d_type}()
                for d in pm_map
                    if typeof(d[2]) == d_type &&
                       d[1].from_to ∈ pm_devices &&
                       d[1].to_from ∈ pm_devices
                        push!(mapped_pm_devices, d[1])
                        push!(mapped_ps_devices, d[2])
                    end
                end
                isempty(mapped_ps_devices) && continue
                mapped_ps_device_names = PSY.get_name.(mapped_ps_devices)

                # add variable in psi
                # add psi_var = pm_expr_var as constraint
                for dir in fieldnames(typeof(ps_v))
                    var_type = getfield(ps_v, dir)
                    var_type === nothing && continue

                    add_variable!(
                        optimization_container,
                        var_type(),
                        mapped_ps_devices,
                        StaticBranchUnbounded(),
                    )
                    psi_var_container = get_variable(
                        optimization_container,
                        make_variable_name(var_type, d_type),
                    )

                    con_name = make_constraint_name(pm_expr_map[d_class].psi_con, d_type)
                    psi_con_container = add_cons_container!(
                        optimization_container,
                        con_name,
                        mapped_ps_device_names,
                        time_steps,
                    )
                    for t in time_steps,
                        (pm_d, name) in zip(mapped_pm_devices, mapped_ps_device_names)

                        pm_expr = PM.var(
                            optimization_container.pm,
                            t,
                            pm_expr_var,
                            getfield(pm_d, dir),
                        )
                        psi_con_container[name, t] = JuMP.@constraint(
                            optimization_container.JuMPmodel,
                            psi_var_container[name, t] == pm_expr
                        )
                    end
                end
            end
        end
    end
end


text: # Tests various functions for linear optics


text:
using DashBootstrapComponents, DashHtmlComponents

items = [
    dbc_dropdownmenuitem("First"),
    dbc_dropdownmenuitem(divider = true),
    dbc_dropdownmenuitem("Second"),
];

dropdown = html_div([
    dbc_dropdownmenu(
        label = "large dropdown",
        size = "lg",
        children = items,
        className = "mb-3",
    ),
    dbc_dropdownmenu(label = "normal dropdown", children = items, className = "mb-3"),
    dbc_dropdownmenu(label = "small dropdown", size = "sm", children = items),
]);

max_stars_repo_name: glsdown/dash-bootstrap-components

max_issues_repo_name: glsdown/dash-bootstrap-components

max_forks_repo_name: glsdown/dash-bootstrap-components


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

const DEFAULT_MAX_DEPTH = 6

"""
    Arrow.write(io::IO, tbl)
    Arrow.write(file::String, tbl)
    tbl |> Arrow.write(io_or_file)

Write any [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible `tbl` out as arrow formatted data.
Providing an `io::IO` argument will cause the data to be written to it
in the ["streaming" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format), unless `file=true` keyword argument is passed.
Providing a `file::String` argument will result in the ["file" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) being written.

Multiple record batches will be written based on the number of
`Tables.partitions(tbl)` that are provided; by default, this is just
one for a given table, but some table sources support automatic
partitioning. Note you can turn multiple table objects into partitions
by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that
each table must have the exact same `Tables.Schema`.

By default, `Arrow.write` will use multiple threads to write multiple
record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set).

Supported keyword arguments to `Arrow.write` include:
  * `colmetadata=nothing`: the metadata that should be written as the table's columns' `custom_metadata` fields; must either be `nothing` or an `AbstractDict` of `column_name::Symbol => column_metadata` where `column_metadata` is an iterable of `<:AbstractString` pairs.
  * `compress`: possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding
  * `alignment::Int=8`: specify the number of bytes to align buffers to when written in messages; strongly recommended to only use alignment values of 8 or 64 for modern memory cache line optimization
  * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written; to dict encode specific columns, wrap the column/array in `Arrow.DictEncode(col)`
  * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; other language implementations [may not support this](https://arrow.apache.org/docs/status.html)
  * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout
  * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed
  * `maxdepth::Int=$DEFAULT_MAX_DEPTH`: deepest allowed nested serialization level; this is provided by default to prevent accidental infinite recursion with mutually recursive data structures
  * `metadata=Arrow.getmetadata(tbl)`: the metadata that should be written as the table's schema's `custom_metadata` field; must either be `nothing` or an iterable of `<:AbstractString` pairs.
  * `ntasks::Int`: number of concurrent threaded tasks to allow while writing input partitions out as arrow record batches; default is no limit; to disable multithreaded writing, pass `ntasks=1`
  * `file::Bool=false`: if a an `io` argument is being written to, passing `file=true` will cause the arrow file format to be written instead of just IPC streaming
"""
function write end

write(io_or_file; kw...) = x -> write(io_or_file, x; kw...)

function write(file_path, tbl; metadata=getmetadata(tbl), colmetadata=nothing, largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, ntasks=Inf, file::Bool=true)
    open(file_path, "w") do io
        write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, metadata, colmetadata)
    end
    return file_path
end

function write(io::IO, tbl; metadata=getmetadata(tbl), colmetadata=nothing, largelists::Bool=false, compress::Union{Nothing, Symbol, LZ4FrameCompressor, ZstdCompressor}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, ntasks=Inf, file::Bool=false)
    return write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, metadata, colmetadata)
end

function write(io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, meta, colmeta)
    if ntasks < 1
        throw(ArgumentError("ntasks keyword argument must be > 0; pass `ntasks=1` to disable multithreaded writing"))
    end
    if compress === :lz4
        compress = LZ4_FRAME_COMPRESSOR
    elseif compress === :zstd
        compress = ZSTD_COMPRESSOR
    elseif compress isa Symbol
        throw(ArgumentError("unsupported compress keyword argument value: $compress. Valid values include `:lz4` or `:zstd`"))
    end
    # TODO: we're probably not threadsafe if user passes own single compressor instance + ntasks > 1
    # if ntasks > 1 && compres !== nothing && !(compress isa Vector)
    #     compress = Threads.resize_nthreads!([compress])
    # end
    if writetofile
        @debug 1 "starting write of arrow formatted file"
        Base.write(io, "ARROW1\0\0")
    end
    msgs = OrderedChannel{Message}(ntasks)
    # build messages
    sch = Ref{Tables.Schema}()
    firstcols = Ref{Any}()
    dictencodings = Dict{Int64, Any}() # Lockable{DictEncoding}
    blocks = (Block[], Block[])
    # start message writing from channel
    threaded = ntasks > 1
    tsk = threaded ? (Threads.@spawn for msg in msgs
        Base.write(io, msg, blocks, sch, alignment)
    end) : (@async for msg in msgs
        Base.write(io, msg, blocks, sch, alignment)
    end)
    anyerror = Threads.Atomic{Bool}(false)
    errorref = Ref{Any}()
    @sync for (i, tbl) in enumerate(Tables.partitions(source))
        if anyerror[]
            @error "error writing arrow data on partition = $(errorref[][3])" exception=(errorref[][1], errorref[][2])
            error("fatal error writing arrow data")
        end
        @debug 1 "processing table partition i = $i"
        tblcols = Tables.columns(tbl)
        if i == 1
            cols = toarrowtable(tblcols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, meta, colmeta)
            sch[] = Tables.schema(cols)
            firstcols[] = cols
            put!(msgs, makeschemamsg(sch[], cols), i)
            if !isempty(dictencodings)
                des = sort!(collect(dictencodings); by=x->x.first, rev=true)
                for (id, delock) in des
                    # assign dict encoding ids
                    de = delock.x
                    dictsch = Tables.Schema((:col,), (eltype(de.data),))
                    put!(msgs, makedictionarybatchmsg(dictsch, (col=de.data,), id, false, alignment), i)
                end
            end
            put!(msgs, makerecordbatchmsg(sch[], cols, alignment), i, true)
        else
            if threaded
                Threads.@spawn process_partition(tblcols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta)
            else
                process_partition(tblcols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta)
            end
        end
    end
    if anyerror[]
        @error "error writing arrow data on partition = $(errorref[][3])" exception=(errorref[][1], errorref[][2])
        error("fatal error writing arrow data")
    end
    # close our message-writing channel, no further put!-ing is allowed
    close(msgs)
    # now wait for our message-writing task to finish writing
    wait(tsk)
    # write empty message
    if !writetofile
        Base.write(io, Message(UInt8[], nothing, 0, true, false, Meta.Schema), blocks, sch, alignment)
    end
    if writetofile
        b = FlatBuffers.Builder(1024)
        schfoot = makeschema(b, sch[], firstcols[])
        if !isempty(blocks[1])
            N = length(blocks[1])
            Meta.footerStartRecordBatchesVector(b, N)
            for blk in Iterators.reverse(blocks[1])
                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
            end
            recordbatches = FlatBuffers.endvector!(b, N)
        else
            recordbatches = FlatBuffers.UOffsetT(0)
        end
        if !isempty(blocks[2])
            N = length(blocks[2])
            Meta.footerStartDictionariesVector(b, N)
            for blk in Iterators.reverse(blocks[2])
                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
            end
            dicts = FlatBuffers.endvector!(b, N)
        else
            dicts = FlatBuffers.UOffsetT(0)
        end
        Meta.footerStart(b)
        Meta.footerAddVersion(b, Meta.MetadataVersions.V4)
        Meta.footerAddSchema(b, schfoot)
        Meta.footerAddDictionaries(b, dicts)
        Meta.footerAddRecordBatches(b, recordbatches)
        foot = Meta.footerEnd(b)
        FlatBuffers.finish!(b, foot)
        footer = FlatBuffers.finishedbytes(b)
        Base.write(io, footer)
        Base.write(io, Int32(length(footer)))
        Base.write(io, "ARROW1")
    end
    return io
end

function process_partition(cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta)
    try
        cols = toarrowtable(cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, meta, colmeta)
        if !isempty(cols.dictencodingdeltas)
            for de in cols.dictencodingdeltas
                dictsch = Tables.Schema((:col,), (eltype(de.data),))
                put!(msgs, makedictionarybatchmsg(dictsch, (col=de.data,), de.id, true, alignment), i)
            end
        end
        put!(msgs, makerecordbatchmsg(sch[], cols, alignment), i, true)
    catch e
        errorref[] = (e, catch_backtrace(), i)
        anyerror[] = true
    end
    return
end

struct ToArrowTable
    sch::Tables.Schema
    cols::Vector{Any}
    metadata::Union{Nothing,Base.ImmutableDict{String,String}}
    dictencodingdeltas::Vector{DictEncoding}
end

function toarrowtable(cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, meta, colmeta)
    @debug 1 "converting input table to arrow formatted columns"
    sch = Tables.schema(cols)
    types = collect(sch.types)
    N = length(types)
    newcols = Vector{Any}(undef, N)
    newtypes = Vector{Type}(undef, N)
    dictencodingdeltas = DictEncoding[]
    Tables.eachcolumn(sch, cols) do col, i, nm
        oldcolmeta = getmetadata(col)
        newcolmeta = isnothing(colmeta) ? oldcolmeta : get(colmeta, nm, oldcolmeta)
        newcol = toarrowvector(col, i, dictencodings, dictencodingdeltas, newcolmeta; compression=compress, largelists=largelists, denseunions=denseunions, dictencode=dictencode, dictencodenested=dictencodenested, maxdepth=maxdepth)
        newtypes[i] = eltype(newcol)
        newcols[i] = newcol
    end
    minlen, maxlen = isempty(newcols) ? (0, 0) : extrema(length, newcols)
    minlen == maxlen || throw(ArgumentError("columns with unequal lengths detected: $minlen < $maxlen"))
    meta = _normalizemeta(meta)
    return ToArrowTable(Tables.Schema(sch.names, newtypes), newcols, meta, dictencodingdeltas)
end

Tables.columns(x::ToArrowTable) = x
Tables.rowcount(x::ToArrowTable) = length(x.cols) == 0 ? 0 : length(x.cols[1])
Tables.schema(x::ToArrowTable) = x.sch
Tables.columnnames(x::ToArrowTable) = x.sch.names
Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i]

struct Message
    msgflatbuf
    columns
    bodylen
    isrecordbatch::Bool
    blockmsg::Bool
    headerType
end

struct Block
    offset::Int64
    metaDataLength::Int32
    bodyLength::Int64
end

function Base.write(io::IO, msg::Message, blocks, sch, alignment)
    metalen = padding(length(msg.msgflatbuf), alignment)
    @debug 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen), isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)"
    if msg.blockmsg
        push!(blocks[msg.isrecordbatch ? 1 : 2], Block(position(io), metalen + 8, msg.bodylen))
    end
    # now write the final message spec out
    # continuation byte
    n = Base.write(io, 0xFFFFFFFF)
    # metadata length
    n += Base.write(io, Int32(metalen))
    # message flatbuffer
    n += Base.write(io, msg.msgflatbuf)
    n += writezeros(io, paddinglength(length(msg.msgflatbuf), alignment))
    # message body
    if msg.columns !== nothing
        # write out buffers
        for col in Tables.Columns(msg.columns)
            writebuffer(io, col, alignment)
        end
    end
    return n
end

function makemessage(b, headerType, header, columns=nothing, bodylen=0)
    # write the message flatbuffer object
    Meta.messageStart(b)
    Meta.messageAddVersion(b, Meta.MetadataVersions.V5)
    Meta.messageAddHeaderType(b, headerType)
    Meta.messageAddHeader(b, header)
    Meta.messageAddBodyLength(b, Int64(bodylen))
    # Meta.messageAddCustomMetadata(b, meta)
    # Meta.messageStartCustomMetadataVector(b, num_meta_elems)
    msg = Meta.messageEnd(b)
    FlatBuffers.finish!(b, msg)
    return Message(FlatBuffers.finishedbytes(b), columns, bodylen, headerType == Meta.RecordBatch, headerType == Meta.RecordBatch || headerType == Meta.DictionaryBatch, headerType)
end

function makeschema(b, sch::Tables.Schema{names}, columns) where {names}
    # build Field objects
    N = length(names)
    fieldoffsets = [fieldoffset(b, names[i], columns.cols[i]) for i = 1:N]
    Meta.schemaStartFieldsVector(b, N)
    for off in Iterators.reverse(fieldoffsets)
        FlatBuffers.prependoffset!(b, off)
    end
    fields = FlatBuffers.endvector!(b, N)
    if columns.metadata !== nothing
        kvs = columns.metadata
        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
        for (i, (k, v)) in enumerate(kvs)
            koff = FlatBuffers.createstring!(b, String(k))
            voff = FlatBuffers.createstring!(b, String(v))
            Meta.keyValueStart(b)
            Meta.keyValueAddKey(b, koff)
            Meta.keyValueAddValue(b, voff)
            kvoffs[i] = Meta.keyValueEnd(b)
        end
        Meta.schemaStartCustomMetadataVector(b, length(kvs))
        for off in Iterators.reverse(kvoffs)
            FlatBuffers.prependoffset!(b, off)
        end
        meta = FlatBuffers.endvector!(b, length(kvs))
    else
        meta = FlatBuffers.UOffsetT(0)
    end
    # write schema object
    Meta.schemaStart(b)
    Meta.schemaAddEndianness(b, Meta.Endiannesses.Little)
    Meta.schemaAddFields(b, fields)
    Meta.schemaAddCustomMetadata(b, meta)
    return Meta.schemaEnd(b)
end

function makeschemamsg(sch::Tables.Schema, columns)
    @debug 1 "building schema message: sch = $sch"
    b = FlatBuffers.Builder(1024)
    schema = makeschema(b, sch, columns)
    return makemessage(b, Meta.Schema, schema)
end

function fieldoffset(b, name, col)
    nameoff = FlatBuffers.createstring!(b, string(name))
    T = eltype(col)
    nullable = T >: Missing
    # check for custom metadata
    if getmetadata(col) !== nothing
        kvs = getmetadata(col)
        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
        for (i, (k, v)) in enumerate(kvs)
            koff = FlatBuffers.createstring!(b, String(k))
            voff = FlatBuffers.createstring!(b, String(v))
            Meta.keyValueStart(b)
            Meta.keyValueAddKey(b, koff)
            Meta.keyValueAddValue(b, voff)
            kvoffs[i] = Meta.keyValueEnd(b)
        end
        Meta.fieldStartCustomMetadataVector(b, length(kvs))
        for off in Iterators.reverse(kvoffs)
            FlatBuffers.prependoffset!(b, off)
        end
        meta = FlatBuffers.endvector!(b, length(kvs))
    else
        meta = FlatBuffers.UOffsetT(0)
    end
    # build dictionary
    if isdictencoded(col)
        encodingtype = indtype(col)
        IT, inttype, _ = arrowtype(b, encodingtype)
        Meta.dictionaryEncodingStart(b)
        Meta.dictionaryEncodingAddId(b, Int64(getid(col)))
        Meta.dictionaryEncodingAddIndexType(b, inttype)
        # TODO: support isOrdered?
        Meta.dictionaryEncodingAddIsOrdered(b, false)
        dict = Meta.dictionaryEncodingEnd(b)
    else
        dict = FlatBuffers.UOffsetT(0)
    end
    type, typeoff, children = arrowtype(b, col)
    if children !== nothing
        Meta.fieldStartChildrenVector(b, length(children))
        for off in Iterators.reverse(children)
            FlatBuffers.prependoffset!(b, off)
        end
        children = FlatBuffers.endvector!(b, length(children))
    else
        Meta.fieldStartChildrenVector(b, 0)
        children = FlatBuffers.endvector!(b, 0)
    end
    # build field object
    if isdictencoded(col)
        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type, inttype = $IT, dictionary id = $(getid(col))"
    else
        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type"
    end
    Meta.fieldStart(b)
    Meta.fieldAddName(b, nameoff)
    Meta.fieldAddNullable(b, nullable)
    Meta.fieldAddTypeType(b, type)
    Meta.fieldAddType(b, typeoff)
    Meta.fieldAddDictionary(b, dict)
    Meta.fieldAddChildren(b, children)
    Meta.fieldAddCustomMetadata(b, meta)
    return Meta.fieldEnd(b)
end

struct FieldNode
    length::Int64
    null_count::Int64
end

struct Buffer
    offset::Int64
    length::Int64
end

function makerecordbatchmsg(sch::Tables.Schema{names, types}, columns, alignment) where {names, types}
    b = FlatBuffers.Builder(1024)
    recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
    return makemessage(b, Meta.RecordBatch, recordbatch, columns, bodylen)
end

function makerecordbatch(b, sch::Tables.Schema{names, types}, columns, alignment) where {names, types}
    nrows = Tables.rowcount(columns)

    compress = nothing
    fieldnodes = FieldNode[]
    fieldbuffers = Buffer[]
    bufferoffset = 0
    for col in Tables.Columns(columns)
        if col isa Compressed
            compress = compressiontype(col)
        end
        bufferoffset = makenodesbuffers!(col, fieldnodes, fieldbuffers, bufferoffset, alignment)
    end
    @debug 1 "building record batch message: nrows = $nrows, sch = $sch, compress = $compress"

    # write field nodes objects
    FN = length(fieldnodes)
    Meta.recordBatchStartNodesVector(b, FN)
    for fn in Iterators.reverse(fieldnodes)
        Meta.createFieldNode(b, fn.length, fn.null_count)
    end
    nodes = FlatBuffers.endvector!(b, FN)

    # write buffer objects
    bodylen = 0
    BN = length(fieldbuffers)
    Meta.recordBatchStartBuffersVector(b, BN)
    for buf in Iterators.reverse(fieldbuffers)
        Meta.createBuffer(b, buf.offset, buf.length)
        bodylen += padding(buf.length, alignment)
    end
    buffers = FlatBuffers.endvector!(b, BN)

    # compression
    if compress !== nothing
        Meta.bodyCompressionStart(b)
        Meta.bodyCompressionAddCodec(b, compress)
        Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethods.BUFFER)
        compression = Meta.bodyCompressionEnd(b)
    else
        compression = FlatBuffers.UOffsetT(0)
    end

    # write record batch object
    @debug 1 "built record batch message: nrows = $nrows, nodes = $fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
    Meta.recordBatchStart(b)
    Meta.recordBatchAddLength(b, Int64(nrows))
    Meta.recordBatchAddNodes(b, nodes)
    Meta.recordBatchAddBuffers(b, buffers)
    Meta.recordBatchAddCompression(b, compression)
    return Meta.recordBatchEnd(b), bodylen
end

function makedictionarybatchmsg(sch, columns, id, isdelta, alignment)
    @debug 1 "building dictionary message: id = $id, sch = $sch, isdelta = $isdelta"
    b = FlatBuffers.Builder(1024)
    recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
    Meta.dictionaryBatchStart(b)
    Meta.dictionaryBatchAddId(b, Int64(id))
    Meta.dictionaryBatchAddData(b, recordbatch)
    Meta.dictionaryBatchAddIsDelta(b, isdelta)
    dictionarybatch = Meta.dictionaryBatchEnd(b)
    return makemessage(b, Meta.DictionaryBatch, dictionarybatch, columns, bodylen)
end

max_stars_repo_name: jakkosdev/arrow-julia

max_issues_repo_name: jakkosdev/arrow-julia

max_forks_repo_name: jakkosdev/arrow-julia


text:
"""
    log_histogram(logger, name, data, step)

Logs a histogram under the tag `name` on given `step`. The histogram must be
passed as a tuple holding the `N+1` bin edges and the height of the `N` bins.

You can also pass the raw data, and a binning algorithm from `StatsBase.jl` will
be used to bin the data.
"""
function log_histogram(logger::Logger, name::String, data::Tuple{Vector,Vector};
                       step=nothing)
    summ    = SummaryCollection()
    push!(summ.value,  histogram_summary(name, first(data), last(data)))
    write_event(logger.file, make_event(logger, summ, step=step))
end

"""
    log_histogram(logger, name, data::Vector, step)

Bins the values found in `data` and logs them as an histogram under the tag
`name`.
"""
function log_histogram(logger::Logger, name::String, data::Vector;
                       step=nothing)
    summ    = SummaryCollection()
    hvals = fit(Histogram, data)
    push!(summ.value, histogram_summary(name, collect(hvals.edges[1]), hvals.weights))
    write_event(logger.file, make_event(logger, summ, step=step))
end

"""
    log_vector(logger, name, data::Vector, step)

Logs the vector found in `data` as an histogram under the name `name`.
"""
function log_vector(logger::Logger, name::String, data::Vector; step=nothing)
    summ    = SummaryCollection()
    push!(summ.value, histogram_summary(name, collect(0:length(data)),data))
    write_event(logger.file, make_event(logger, summ, step))
end

function histogram_summary(name::String, edges::Vector{T1}, hist_vals::Vector{T2}) where {T1<:Number, T2<:Number}
    @assert length(edges) == length(hist_vals)+1

    hp = HistogramProto(min=minimum(edges), max=maximum(edges),
                        bucket_limit=Vector{Float64}(),
                        bucket=Vector{Float64}())
    for val=edges[2:end]
        push!(hp.bucket_limit, val)
    end
    for val=hist_vals
        push!(hp.bucket, val)
    end

    Summary_Value(tag=name, histo=hp)
end

## Backward compatibility
log_histogram(logger, name, value, step) =
    log_histogram(logger, name, value, step=step)


text:
using TrainRuns
using Documenter

DocMeta.setdocmeta!(TrainRuns, :DocTestSetup, :(using TrainRuns); recursive=true)

makedocs(;
    modules=[TrainRuns],
    authors="Max Kannenberg, Martin Scheidt, and contributors",
    repo="https://github.com/railtoolkit/TrainRuns.jl/blob/{commit}{path}#{line}",
    sitename="TrainRuns.jl",
    format=Documenter.HTML(;
        prettyurls=get(ENV, "CI", "false") == "true",
        canonical="https://railtoolkit.github.io/TrainRuns.jl",
        assets=String[],
    ),
    pages=[
        "Home" => "index.md",
    ],
)

deploydocs(;
    repo="github.com/railtoolkit/TrainRuns.jl",
    devbranch="development",
)


text:
module PortAudio

using alsa_plugins_jll
using libportaudio_jll, SampledSignals

import Base: eltype, show
import Base: close, isopen
import Base: read, read!, write, flush

using Base.Threads: @spawn
using Base.Sys: islinux, iswindows

import LinearAlgebra
import LinearAlgebra: transpose!

import SampledSignals: nchannels, samplerate, unsafe_read!, unsafe_write

export PortAudioStream

include("libportaudio.jl")

# This size is in frames

# data is passed to and from portaudio in chunks with this many frames, because
# we need to interleave the samples
const CHUNK_FRAMES = 128

function versioninfo(io::IO = stdout)
    println(io, Pa_GetVersionText())
    println(io, "Version: ", Pa_GetVersion())
end

struct IOBounds
    max_channels::Int
    low_latency::Float64
    high_latency::Float64
end

mutable struct PortAudioDevice
    name::String
    host_api::String
    default_sample_rate::Float64
    index::PaDeviceIndex
    input::IOBounds
    output::IOBounds
end

PortAudioDevice(info::PaDeviceInfo, index) = PortAudioDevice(
    unsafe_string(info.name),
    unsafe_string(Pa_GetHostApiInfo(info.host_api).name),
    info.default_sample_rate,
    index,
    IOBounds(
        info.max_input_channels,
        info.default_low_input_latency,
        info.default_high_input_latency,
    ),
    IOBounds(
        info.max_output_channels,
        info.default_low_output_latency,
        info.default_high_output_latency,
    ),
)

function name(device::PortAudioDevice)
    device.name
end

function get_device(device::PortAudioDevice)
    device
end

function get_device(device_name::AbstractString)
    devices[device_name]
end

function get_device(index::Integer)
    PortAudioDevice(Pa_GetDeviceInfo(index), index)
end

function show(io::IO, device::PortAudioDevice)
    print(io,
        device.index,
        ": ",
        name(device)
    )
    max_input_channels = device.input.max_channels
    has_inputs = max_input_channels > 0
    max_output_channels = device.output.max_channels
    has_outputs = max_output_channels > 0
    if has_inputs || has_outputs
        print(io, " (")
        if has_inputs
            print(io, "in: ", max_input_channels)
        end
        if has_inputs && has_outputs
            print(io, ", ")
        end
        if has_outputs
            print(io, "out: ", max_output_channels)
        end
        print(io, ")")
    end
end

function devices()
    Dict(Iterators.map(
        function (index)
            device = PortAudioDevice(Pa_GetDeviceInfo(index), index)
            name(device) => device
        end,
        (1:Pa_GetDeviceCount()) .- 1
    ))
end

struct Portal{Sample}
    device::PortAudioDevice
    number_of_channels::Int
    chunk_buffer::Array{Sample,2}
end

function Portal(device, IO_bounds;
    number_of_channels = 2,
    Sample = Float32
)
    number_of_channels_filled =
        if number_of_channels === max
            IO_bounds.max_channels
        else
            number_of_channels
        end
    Portal{Sample}(device, number_of_channels_filled, zeros(Sample, number_of_channels_filled, CHUNK_FRAMES))
end

function input(;
    device_id = Pa_GetDefaultInputDevice(),
    keyword_arguments...
)
    device = get_device(device_id)
    Portal(device, device.input; keyword_arguments...)
end

function output(;
    device_id = Pa_GetDefaultOutputDevice(),
    keyword_arguments...
)
    device = get_device(device_id)
    Portal(device, device.output; keyword_arguments...)
end

function eltype(::Type{Portal{Sample}}) where {Sample}
    Sample
end

function nchannels(portal::Portal)
    portal.number_of_channels
end
function name(portal::Portal)
    name(portal.device)
end

mutable struct PortAudioStream{Sample}
    input_portal::Portal{Sample}
    output_portal::Portal{Sample}
    stream_pointer::PaStream
    the_sample_rate::Float64
    latency::Float64
end

function make_parameters(portal, latency)
    if portal === nothing
        Ptr{Pa_StreamParameters}(0)
    else
        Ref(
            Pa_StreamParameters(
                portal.device.index,
                portal.number_of_channels,
                TYPE_TO_FORMAT[eltype(portal)],
                latency,
                C_NULL,
            ),
        )
    end
end

function get_default_latency(input_portal, output_portal)
    if input_portal === nothing
        if output_portal === nothing
            error("No devices!")
        else
            output_device.output.high_latency
        end
    elseif output_portal === nothing
        input_device.input.high_latency
    else
        max(input_device.input.high_latency, output_device.output.high_latency)
    end
end

function get_default_sample_rate(
    input_portal,
    output_portal
)
    if input_portal === nothing
        if output_portal === nothing
            error("No devices!")
        else
            output_portal.device.default_sample_rate
        end
    elseif output_portal === nothing
        input_portal.device.default_sample_rate
    else
        sample_rate_input = input_portal.device.default_sample_rate
        sample_rate_output = output_portal.device.default_sample_rate
        if sample_rate_input != sample_rate_output
            error("Input and output default sample rates do not match. Plese specify a sample rate.")
        else
            sample_rate_input
        end
    end
end

# this is the top-level outer constructor that all the other outer constructors end up calling
"""
    PortAudioStream(inchannels=2, outchannels=2; options...)
    PortAudioStream(duplexdevice, inchannels=2, outchannels=2; options...)
    PortAudioStream(indevice, outdevice, inchannels=2, outchannels=2; options...)

Audio devices can either be `PortAudioDevice` instances as returned
by `PortAudio.devices()`, or strings with the device name as reported by the
operating system. If a single `duplexdevice` is given it will be used for both
input and output. If no devices are given the system default devices will be
used.

Options:

* `eltype`:         Sample type of the audio stream (defaults to Float32)
* `the_sample_rate`:     Sample rate (defaults to device sample rate)
* `latency`:        Requested latency. Stream could underrun when too low, consider
                    using provided device defaults
* `warn_xruns`:     Display a warning if there is a stream overrun or underrun, which
                    often happens when Julia is compiling, or with a particularly large
                    GC run. This can be quite verbose so is false by default.
* `recover_xruns`:  Attempt to recover from overruns and underruns by emptying and
                    filling the input and output buffers, respectively. Should result in
                    fewer xruns but could make each xrun more audible. True by default.
                    Only effects duplex streams.
"""
function PortAudioStream(
    input_portal,
    output_portal;
    the_sample_rate = get_default_sample_rate(input_portal, output_portal),
    latency = get_default_latency(input_portal, output_portal)
)
    # finalizer(close, this)
    input_parameters = make_parameters(input_portal, latency)
    output_parameters = make_parameters(output_portal, latency)
    stream_pointer = Pa_OpenStream(
        input_parameters,
        output_parameters,
        the_sample_rate,
        0,
        PA_NO_FLAG,
        nothing,
        nothing,
    )
    Pa_StartStream(stream_pointer)
    # pre-fill the output stream so we're less likely to underrun
    stream = PortAudioStream(
        input_portal,
        output_portal,
        stream_pointer,
        the_sample_rate,
        latency
    )
end

# handle do-syntax
function PortAudioStream(do_function::Function, arguments...; keyword_arguments...)
    stream = PortAudioStream(arguments...; keyword_arguments...)
    try
        do_function(stream)
    finally
        close(stream)
    end
end

function close(stream::PortAudioStream)
    stream_pointer = stream.stream_pointer
    if stream_pointer != C_NULL
        Pa_StopStream(stream_pointer)
        Pa_CloseStream(stream_pointer)
        stream.stream_pointer = C_NULL
    end
    nothing
end

isopen(stream::PortAudioStream) = stream.stream_pointer != C_NULL

samplerate(stream::PortAudioStream) = stream.the_sample_rate
eltype(::PortAudioStream{Sample}) where {Sample} = Sample

read(stream::PortAudioStream, arguments...) = read(PortAudioSource(stream), arguments...)
read!(stream::PortAudioStream, arguments...) = read!(PortAudioSource(stream), arguments...)
write(stream::PortAudioStream, arguments...) = write(PortAudioSink(stream), arguments...)
write(sink_stream::PortAudioStream, source_stream::PortAudioStream, arguments...) =
    write(PortAudioSink(sink_stream), PortAudioSource(source_stream), arguments...)
flush(stream::PortAudioStream) = flush(PortAudioSink(stream))

function show(io::IO, stream::PortAudioStream)
    println(io, typeof(stream))
    println(io, "  Samplerate: ", samplerate(stream), "Hz")
    sink_portal = stream.sink_portal
    sink_channels = nchannels(sink_portal)
    if sink_channels > 0
        print(io, "\n  ", sink_channels, " channel sink: \"", name(sink_portal), "\"")
    end
    source_portal = stream.source_portal
    source_channels = nchannels(source_portal)
    if source_channels > 0
        print(io, "\n  ", source_channels, " channel source: \"", name(source_portal), "\"")
    end
end

#
# PortAudioSink & PortAudioSource
#

# Define our source and sink types
struct PortAudioSink{Sample} <: SampleSink
    stream::PortAudioStream{Sample}
end

struct PortAudioSource{Sample} <: SampleSource
    stream::PortAudioStream{Sample}
end

nchannels(sink::PortAudioSink) = nchannels(sink.stream.sink_portal)
nchannels(source::PortAudioSource) = nchannels(source.stream.source_portal)
samplerate(sink_or_source::Union{PortAudioSink,PortAudioSource}) =
    samplerate(sink_or_source.stream)
eltype(::Union{Type{PortAudioSink{Sample}},Type{PortAudioSource{Sample}}}) where {Sample} = Sample
function close(sink_or_source::Union{PortAudioSink,PortAudioSource})
    close(sink_or_source.stream)
end
isopen(sink_or_source::Union{PortAudioSink,PortAudioSource}) = isopen(sink_or_source.stream)
name(sink::PortAudioSink) = name(sink.stream.sink_portal)
name(source::PortAudioSource) = name(source.stream.source_portal)

function show(io::IO, ::Type{PortAudioSink{Sample}}) where {Sample}
    print(io, "PortAudioSink{", Sample, "}")
end

function show(io::IO, ::Type{PortAudioSource{Sample}}) where {Sample}
    print(io, "PortAudioSource{", Sample, "{")
end

function show(
    io::IO,
    sink_or_source::SinkOrSource,
) where {SinkOrSource<:Union{PortAudioSink,PortAudioSource}}
    print(
        io,
        nchannels(sink_or_source),
        "-channel ",
        SinkOrSource,
        "(\"",
        name(sink_or_source),
        "\")",
    )
end

function unsafe_write(sink::PortAudioSink, buf::Array, frameoffset, framecount)
    stream = sink.stream
    stream_pointer = stream.stream_pointer
    chunk_buffer = stream.sink_portal.chunk_buffer
    warn_xruns = stream.warn_xruns
    recover_xruns = stream.recover_xruns
    number_written = 0
    while number_written < framecount
        number = min(framecount - number_written, CHUNK_FRAMES)
        # make a buffer of interleaved samples
        transpose!(
            view(chunk_buffer, :, 1:number),
            view(buf, (1:number) .+ number_written .+ frameoffset, :),
        )
        # TODO: if the stream is closed we just want to return a
        # shorter-than-requested frame count instead of throwing an error
        if Pa_WriteStream(stream_pointer, chunk_buffer, number, warn_xruns) ∈
           (PA_OUTPUT_UNDERFLOWED, PA_INPUT_OVERFLOWED) && recover_xruns
            recover_xrun(stream)
        end
        number_written += number
    end

    number_written
end

function unsafe_read!(source::PortAudioSource, buf::Array, frameoffset, framecount)
    stream = source.stream
    stream_pointer = stream.stream_pointer
    chunk_buffer = stream.source_portal.chunk_buffer
    warn_xruns = stream.warn_xruns
    recover_xruns = stream.recover_xruns
    number_read = 0
    while number_read < framecount
        number = min(framecount - number_read, CHUNK_FRAMES)
        # TODO: if the stream is closed we just want to return a
        # shorter-than-requested frame count instead of throwing an error
        if Pa_ReadStream(stream_pointer, chunk_buffer, number, warn_xruns) ∈
           (PA_OUTPUT_UNDERFLOWED, PA_INPUT_OVERFLOWED) && recover_xruns
            recover_xrun(stream)
        end
        # de-interleave the samples
        transpose!(
            view(buf, (1:number) .+ number_read .+ frameoffset, :),
            view(chunk_buffer, :, 1:number),
        )

        number_read += number
    end

    number_read
end

"""
    prefill_output(stream::PortAudioStream)

Fill the playback buffer of the given sink.
"""
function prefill_output(stream::PortAudioStream)
    stream_pointer = stream.stream_pointer
    chunk_buffer = stream.sink_portal.chunk_buffer
    a_zero = zero(eltype(chunk_buffer))
    to_write = Pa_GetStreamWriteAvailable(stream_pointer)
    while to_write > 0
        number = min(to_write, CHUNK_FRAMES)
        fill!(chunk_buffer, a_zero)
        Pa_WriteStream(stream_pointer, chunk_buffer, number, false)
        to_write -= number
    end
end

"""
    discard_input(source::PortAudioSource)

Read and discard data from the capture buffer.
"""
function discard_input(stream::PortAudioStream)
    stream_pointer = stream.stream_pointer
    chunk_buffer = stream.source_portal.chunk_buffer
    to_read = Pa_GetStreamReadAvailable(stream_pointer)
    while to_read > 0
        number = min(to_read, CHUNK_FRAMES)
        Pa_ReadStream(stream_pointer, chunk_buffer, number, false)
        to_read -= number
    end
end

function seek_config(folders)
    for folder in folders
        if isfile(joinpath(folder, "alsa.conf"))
            return folder
        end
    end
    throw(
        ErrorException(
            """
            Could not find ALSA config directory. Searched:
            $(join(folders, "\n"))

            if ALSA is installed, set the "ALSA_CONFIG_DIR" environment
            variable. The given directory should have a file "alsa.conf".

            If it would be useful to others, please file an issue at
            https://github.com/JuliaAudio/PortAudio.jl/issues
            with your alsa config directory so we can add it to the search
            paths.
            """,
        ),
    )
end

const ALSA_CONFIG_DIRS = ["/usr/share/alsa", "/usr/local/share/alsa", "/etc/alsa"]

function __init__()
    if islinux()
        get!(ENV, "ALSA_CONFIG_DIR") do
            seek_config(ALSA_CONFIG_DIRS)
        end
        if alsa_plugins_jll.is_available()
            get!(ENV, "ALSA_PLUGIN_DIR") do
                joinpath(alsa_plugins_jll.artifact_dir, "lib", "alsa-lib")
            end
        end
    end
    # initialize PortAudio on module load. libportaudio prints a bunch of
    # junk to STDOUT on initialization, so we swallow it.
    # TODO: actually check the junk to make sure there's nothing in there we
    # don't expect
    Pa_Initialize()

    atexit() do
        Pa_Terminate()
    end
end

end # module PortAudio


text:
# The purpose of this file is to test
# the basic (i.e. non-mathematical) functions
# of values with type Q<:AbstractQuArray.

###########
# QuArray #
###########
qa_coeffs = rand(Complex{Float64},4,8)
qa_basis = tuple(FiniteBasis(2,2), FiniteBasis(4,2))
qa = QuArray(qa_coeffs, qa_basis)

@assert coeffs(qa) == qa_coeffs
@assert rawcoeffs(qa) == qa_coeffs
@assert coefftype(qa) == typeof(qa_coeffs)
@assert QuBase.similar_type(qa) == QuArray
@assert length(qa) == length(qa_coeffs)
@assert size(qa) == size(qa_coeffs)
@assert size(qa,1) == size(qa_coeffs,1)
@assert ndims(qa) == ndims(qa_coeffs)

rand_i = rand(1:size(qa,1))
rand_j = rand(1:size(qa,2))

@assert qa[rand_i, rand_j] == qa_coeffs[rand_i, rand_j]

c = rand(Complex{Float64})
qa[rand_i, rand_j] = c

@assert qa[rand_i, rand_j] == qa_coeffs[rand_i, rand_j] == c
@assert in(qa[rand_i, rand_j], qa)
@assert qa == copy(qa)

@assert bases(qa) == qa_basis
@assert bases(qa,1) == qa_basis[1]
@assert rawbases(qa) == qa_basis
@assert rawbases(qa,2) == qa_basis[2]

mock_type = QuArray{FiniteBasis{Orthonormal}, BigFloat, 2, Matrix{BigFloat}}
prom_T = promote_type(eltype(qa), eltype(mock_type))
prom_A = promote_type(coefftype(qa), coefftype(mock_type))
prom_result = QuArray{FiniteBasis{Orthonormal}, prom_T, 2, prom_A}

@assert promote_type(typeof(qa), mock_type) == prom_result

##############
# CTranspose #
##############
qac = qa'

@assert coeffs(qac) == qa_coeffs'
@assert rawcoeffs(qac) == qa_coeffs
@assert coefftype(qac) == coefftype(qa)
@assert QuBase.similar_type(qac) == CTranspose
@assert length(qac) == length(qa)
@assert size(qac) == reverse(size(qa))
@assert size(qac,1) == size(qa,2)
@assert ndims(qac) == ndims(qa)
@assert qac[rand_j, rand_i] == qa[rand_i, rand_j]'

c = rand(Complex{Float64})
qac[rand_j, rand_i] = c

@assert qac[rand_j, rand_i] == qa[rand_i, rand_j]' == c
@assert in(qac[rand_j, rand_i], qac)

@assert qac == copy(qac)

@assert bases(qac) == reverse(bases(qa))
@assert bases(qac,1) == bases(qa,2)
@assert rawbases(qac) == bases(qa)
@assert rawbases(qac,2) == bases(qa,2)

qac_eager = QuBase.eager_ctranspose(qa)

@assert convert(QuBase.qarr_type(qac), qac) == qac_eager
@assert qac == qac_eager
@assert promote_type(typeof(qac), mock_type) == prom_result


text:
"""
    zero_rate(curve, maturity)

Returns the *zero-coupon* rate to `maturity`.
"""
function zero_rate end

"""
    forward_rate(curve, forward_date, maturity)

Returns the *forward rate* from a future date
`forward_date` to `maturity`.
"""
function forward_rate end

"""
    discountfactor(curve, maturity)
    discountfactor(curve, forward_date, maturity)

Returns the discount factor to `maturity`.
Or a forward discount factor from `forward_date` to `maturity`.
"""
function discountfactor end