text:
using SIMD

"""
    chipsort_medium!(data, Val(V), Val(J), Val(K))

Sort a medium array in-place. The data is divided between `K` blocks of `J` SIMD vectors of size `V`. The array should typically fit inside lower levels of cache memory (L1 or L2), for instance 8192 Int32 values on an conventional desktop computer. Our recipe is:
 - Generate small sorted vectors.
 - Use vectorized Comb sort.
 - Transpose in-place.
 - Finish with insertion sort over the nearly sorted array.

# Examples
```jldoctest
julia> chipsort_medium!(Int32[1:2^13...] .* 0xf0ca .%0x10000, Val(8), Val(8), Val(64))'
1×8192 LinearAlgebra.Adjoint{Int32,Array{Int32,1}}:
 30  32  34  36  38  70  72  74  76  108  110  112  114  116  …  65488  65490  65492  65494  65496  65528  65530  65532  65534
```
"""
function chipsort_medium!(input::AbstractVector{T}, ::Val{V}, ::Val{J}, ::Val{K}) where {T,V,J,K}

    sort_vecs!(input, Val(J), Val(V), Val(true))

    vectorized_combsort!(input, Val(J))

    transpose_blocks!(input, Val(V), Val(J))

    transpose!(input, Val(V), Val(K), Val(J))

    vectorized_combsort!(input, Val(J))

    sort_vecs!(input, Val(V), Val(J), Val(false))

    insertion_sort!(input)

    input
end

function vectorized_combsort!(input::AbstractArray{T,1}, ::Val{V}) where {T,V}
    la = length(input)

    interval = 3 * div(la, 32) * 8

    logV = mylog(V)
    interval = (la >> (2+logV)) * (3 * V)

    while interval > 0
        ap = pointer(input,1)
        finalp = pointer(input, la-interval)
        while ap < finalp
            a1 = vloada(Vec{V,T}, ap)
            a2 = vloada(Vec{V,T}, ap + interval*sizeof(T))
            b1 = min(a1, a2)
            b2 = max(a1, a2)
            vstorea(b1, ap)
            vstorea(b2, ap + interval*sizeof(T))
            ap = ap + V * sizeof(T)
        end
        interval = if interval==V 0 else max(V, (interval >> (2+logV)) * (3 * V)) end
    end

    input
end

"""Insertion sort, adapted from the Julia codebase."""
@inline function insertion_sort!(v::AbstractVector)
    lo, hi = 1, length(v)
    @inbounds for i = lo+1:hi
        j = i
        x = v[i]
        while j > lo
            if x <= v[j-1]
                v[j] = v[j-1]
                j -= 1
                continue
            end
            break
        end
        v[j] = x
    end
    return v
end

"""Regular version of Comb sort."""
function combsort!(input::AbstractArray{T,1}, initial_interval=nothing::Union{Nothing,Int}) where T

    la = length(input)
    interval = if (initial_interval == nothing)
        (3 * la) >> 2
    else
        initial_interval
    end

    @inbounds while interval > 1
        for i in 1:la-interval
            a1 = input[i]
            a2 = input[i+interval]
            input[i] = min(a1, a2)
            input[i+interval] = max(a1, a2)
        end
        interval = (3 * interval) >> 2
    end

    change = true
    interval=1
    @inbounds while change
        change = false
        for i in 1:la-1
            if input[i] > input[i+1]
                change = true
                a1 = input[i]
                a2 = input[i+interval]
                input[i] = min(a1, a2)
                input[i+interval] = max(a1, a2)
            end
        end
    end

    input
end

"""Regular Comb sort followed by Insertion sort when the interval becomes 1."""
function chipsort!(input::AbstractArray{T,1}, initial_interval=nothing::Union{Nothing,Int}) where T

    la = length(input)
    interval = if (initial_interval == nothing)
        (3 * la) >> 2
    else
        initial_interval
    end

    @inbounds while interval > 1
        for i in 1:la-interval
            a1 = input[i]
            a2 = input[i+interval]
            input[i] = min(a1, a2)
            input[i+interval] = max(a1, a2)
        end
        interval = (3 * interval) >> 2
    end

    insertion_sort!(input)

    input
end


text:
# for PLEXON file version 105 and greater.

mutable struct PLX_FileHeader
    MagicNumber::Cuint ;  # // == 0x58454c50
    Version::Cint ;

    Comment::String ;  # char[128]
    ADFrequency::Cint ;  # // TimeStamp frequency in hertz
    NumSpikeChannels::Cint ;
    NumEventChannels::Cint ;
    NumContinuousChannels::Cint ;

    NumPointsWave::Cint ;
    NumPointsPreThr::Cint ;

    Year::Cint ;
    Month::Cint ;
    Day::Cint ;
    Hour::Cint ;
    Minute::Cint ;
    Second::Cint ;

    FastRead::Cint ;  # // reserved
    WaveformFreq::Cint ;  # // waveform sampling rate;
    LastTimeStamp::Cdouble ;  # // duration of the experimental session, in ticks

    Trodalness::UInt8;
    DataTrodalness::UInt8;
    BitsPerSpikeSample::UInt8;
    BitsPerContinuousSample::UInt8;

    SpikeMaxMagnitudeMV::UInt16;
    ContinuousMaxMagnitudeMV::UInt16;
    SpikePreAmpGain::UInt16;

    #AcquiringSoftware::String  # char[18]
    #ProcessingSoftware::String  # char[18]
    Padding::String;  # // char[10]

    TSCounts::Array{Int32, 2};
    WFCounts::Array{Int32, 2};
    EVCounts::Vector{Int32};

    PLX_FileHeader(file::IOStream) = begin
        _x = new()

        _x.MagicNumber, _x.Version = read!(file, Vector{Cuint}(undef, 2))
        _x.Comment = read!(file, Vector{UInt8}(undef, 128)) |> String
        _x.ADFrequency, _x.NumSpikeChannels, _x.NumEventChannels, _x.NumContinuousChannels = read!(file, Vector{Cint}(undef, 4))
        _x.NumPointsWave, _x.NumPointsPreThr = read!(file, Vector{Cint}(undef, 2))
        _x.Year, _x.Month, _x.Day, _x.Hour, _x.Minute, _x.Second = read!(file, Vector{Cint}(undef, 6))
        _x.FastRead, _x.WaveformFreq = read!(file, Vector{Cint}(undef, 2))

        _x.LastTimeStamp = read!(file, Vector{Cdouble}(undef, 1))[1]

        _x.Trodalness, _x.DataTrodalness, _x.BitsPerSpikeSample, _x.BitsPerContinuousSample = read!(file, Vector{UInt8}(undef, 4))
        _x.SpikeMaxMagnitudeMV, _x.ContinuousMaxMagnitudeMV, _x.SpikePreAmpGain = read!(file, Vector{UInt16}(undef, 3))

        _x.Padding = read!(file, Vector{UInt8}(undef, 46)) |> String

        _x.TSCounts = read!(file, Array{Cint,2}(undef, 130, 5))
        _x.WFCounts = read!(file, Array{Cint,2}(undef, 130, 5))
        _x.EVCounts = read!(file, Vector{Cint}(undef, 512))

        _x
    end
end

mutable struct PLX_SpikeChannelHeader
    Name::String;  # char[32]
    SIGName::String;  # char[32]
    Channel::Cint;
    WFRate::Cint;

    SIG::Cint;
    Ref::Cint;
    Gain::Cint;

    Filter::Cint;
    Threashold::Cint;
    Method::Cint;
    NUnits::Cint;

    Template::Array{Cshort, 2};  # [5][64]
    Fit::Vector{Cint};  # [5]
    SortWidth::Cint;

    Boxes::Array{Cshort, 3};  # [5][2][4]
    SortBeg::Cint;

    Comment::String;  # char[128]

    # SrcId::Cuchar;
    # reserved::Cuchar;
    # ChanId::Cushort;
    Padding::Vector{Cint};  # char[10]

    PLX_SpikeChannelHeader(file::IOStream) = begin
        _x = new()

        _x.Name = read!(file, Vector{UInt8}(undef, 32)) |> String
        _x.SIGName = read!(file, Vector{UInt8}(undef, 32)) |> String

        _x.Channel, _x.WFRate, _x.SIG, _x.Ref, _x.Gain, _x.Filter, _x.Threashold, _x.Method, _x.NUnits = read!(file, Vector{Cint}(undef, 9))

        _x.Template = read!(file, Array{Cshort,2}(undef, 5,64))
        _x.Fit = read!(file, Vector{Cint}(undef, 5))
        _x.SortWidth = read!(file, Vector{Cint}(undef, 1))[1]

        _x.Boxes = read!(file, Array{Cshort, 3}(undef, 5,2,4))
        _x.SortBeg = read!(file, Vector{Cint}(undef, 1))[1]

        _x.Comment = read!(file, Vector{UInt8}(undef, 128)) |> String
        _x.Padding = read!(file, Vector{Cint}(undef, 11))

        _x
    end
end

mutable struct PLX_EventChannelHeader
    Name::String;  # char[32]
    Channel::Cint;
    Comment::String;  # char[128]

    # SrcId::Cuchar;
    # reserved::Cuchar;
    # ChanId::Cushort;
    Padding::Vector{Cint};  # char[32]

    PLX_EventChannelHeader(file::IOStream) = begin
        _x = new()

        _x.Name = read!(file, Vector{UInt8}(undef, 32)) |> String
        _x.Channel = read!(file, Vector{Cint}(undef, 1))[1]
        _x.Comment = read!(file, Vector{UInt8}(undef, 128)) |> String
        _x.Padding = read!(file, Vector{Cint}(undef, 33))

        _x
    end
end

mutable struct PLX_ContinuousChannelHeader
    Name::String;
    Channel::Cint;
    ADFreq::Cint;
    Gain::Cint;
    Enabled::Cint;
    PreAmpGain::Cint;
    SpikeChannel::Cint;
    Comment::String;
    Padding::Vector{Cint};

    PLX_ContinuousChannelHeader(file::IOStream) = begin
        _x = new()

        _x.Name = read!(file, Vector{UInt8}(undef, 32)) |> String
        _x.Channel, _x.ADFreq, _x.Gain, _x.Enabled, _x.PreAmpGain, _x.SpikeChannel = read!(file, Vector{Cint}(undef, 6))
        _x.Comment = read!(file, Vector{UInt8}(undef, 128)) |> String
        _x.Padding = read!(file, Vector{Cint}(undef, 28))

        _x
    end
end

PLX_DATATYPE_SPIKE = 1
PLX_DATATYPE_EVENT = 4
PLX_DATATYPE_CONTINUOUS = 5

mutable struct PLX_DataBlock_Header  # 16 bytes
    Type::Cshort;  # Data type; 1=spike, 4=Event, 5=continuous
    UpperByteOf5ByteTimestamp::Cushort;
    TimeStamp::UInt32;  # 40-bits timestamp
    Channel::Cshort;
    Unit::Cshort;
    NumberOfWaveforms::Cshort;
    NumberOfWordsInWaveform::Cshort;

    PLX_DataBlock_Header(file::IOStream) = begin
        _x = new()
        _x.Type, _x.UpperByteOf5ByteTimestamp = read!(file, Vector{Cshort}(undef, 2))
        _x.TimeStamp = read!(file, Vector{UInt32}(undef, 1))[1]
        _x.Channel, _x.Unit, _x.NumberOfWaveforms, _x.NumberOfWordsInWaveform = read!(file, Vector{Cshort}(undef, 4))
        _x
    end
end

mutable struct PLXData
    FILE_HEADER::PLX_FileHeader
    SPIKE_CHANNEL_HEADER::Vector{PLX_SpikeChannelHeader}
    EVENT_CHANNEL_HEADER::Vector{PLX_EventChannelHeader}
    CONTINUOUS_CHANNEL_HEADER::Vector{PLX_ContinuousChannelHeader}

    EVENTS::Dict{Int, Vector{UInt64}}
    SPIKE_DATA::Dict{Int, Dict{Int, Vector{Int16}}}
    SPIKE_TS::Dict{Int, Dict{Int, Vector{UInt64}}}
    CONTINUOUS_DATA::Dict{Int, Vector{Int16}}
    CONTINUOUS_TS::Dict{Int, Vector{UInt64}}

    PLXData(filename::String) = begin
        _f = open(filename, "r")
        _x = new()

        _x.FILE_HEADER = PLX_FileHeader(_f)
        if _x.FILE_HEADER.Version < 105
            @error "plexon files version < 105 are not supported. "
        end

        _x.SPIKE_CHANNEL_HEADER = [PLX_SpikeChannelHeader(_f) for i = 1:_x.FILE_HEADER.NumSpikeChannels]
        _x.EVENT_CHANNEL_HEADER = [PLX_EventChannelHeader(_f) for i = 1:_x.FILE_HEADER.NumEventChannels]
        _x.CONTINUOUS_CHANNEL_HEADER = [PLX_ContinuousChannelHeader(_f) for i = 1:_x.FILE_HEADER.NumContinuousChannels]

        _x.EVENTS = Dict{Int, Vector{UInt64}}()
        _x.SPIKE_DATA = Dict{Int, Dict{Int, Vector{Int16}}}()
        _x.SPIKE_TS = Dict{Int, Dict{Int, Vector{Int16}}}()
        _x.CONTINUOUS_DATA = Dict{Int, Vector{Int16}}()
        _x.CONTINUOUS_TS = Dict{Int, Vector{UInt64}}()

        _flag = 0
        while !eof(_f)
            _data_header = PLX_DataBlock_Header(_f)
            _channel = _data_header.Channel
            _unit = _data_header.Unit

            if _data_header.Type == PLX_DATATYPE_SPIKE
                if _data_header.NumberOfWaveforms == 1
                    _raw = read!(_f, Vector{Int16}(undef, _data_header.NumberOfWordsInWaveform))
                end

                if !haskey(_x.SPIKE_DATA, _channel)
                    _x.SPIKE_DATA[_channel] = Dict{Int, Vector{Int16}}()
                    _x.SPIKE_TS[_channel] = Dict{Int, Vector{UInt64}}()
                end
                if !haskey(_x.SPIKE_DATA[_channel], _unit)
                    _x.SPIKE_DATA[_channel][_unit] = _raw
                    _x.SPIKE_TS[_channel][_unit] = [_data_header.TimeStamp]
                else
                    append!(_x.SPIKE_DATA[_channel][_unit], _raw)
                    push!(_x.SPIKE_TS[_channel][_unit], _data_header.TimeStamp)
                end

            elseif _data_header.Type == PLX_DATATYPE_EVENT
                if !haskey(_x.EVENTS, _channel)
                    _x.EVENTS[_channel] = Vector{UInt64}([_data_header.TimeStamp])
                else
                    push!(_x.EVENTS[_channel], _data_header.TimeStamp)
                end

            elseif _data_header.Type == PLX_DATATYPE_CONTINUOUS
                if _data_header.NumberOfWaveforms == 1
                    _raw = read!(_f, Vector{Int16}(undef, _data_header.NumberOfWordsInWaveform))
                end
                if !haskey(_x.CONTINUOUS_DATA, _channel)
                    _x.CONTINUOUS_DATA[_channel] = Vector{Int16}(_raw)
                    _x.CONTINUOUS_TS[_channel] = Vector{UInt64}([_data_header.TimeStamp])
                else
                    append!(_x.CONTINUOUS_DATA[_channel], _raw)
                    push!(_x.CONTINUOUS_TS[_channel], _data_header.TimeStamp)
                end

            else
                @error "$(_flag) - unknow data type $(_data_header.Type)"
            end

            _flag += 1

        end
        close(_f)
        _x
    end
end

ext: jl


text:
#####################################################
# Surya
# NREL
# October 2021
# Script to generate csv's from EGRET System JSON
# Generate component CSV SIIP tabular parser expects from EGRET System JSON
#####################################################################################
# Auxilary Function
# Convert DataFrames.DataFrame to JSON
# Convert timeseries_pointers.csv to timeseries_pointers.JSON
#####################################################################################
function df_to_json(df::DataFrames.DataFrame,dir_name::String)
    df[!,"normalization_factor"] .= "max"
    pointers_dict = []
    for row in eachrow(df)
        push!(pointers_dict,Dict(fn=>get(row, fn,nothing) for fn ∈ DataFrames.names(df)))
    end

    ts_json_location = joinpath(dir_name,"timeseries_pointers.json")

    open(ts_json_location,"w") do f
        JSON.print(f, pointers_dict, 4)
    end
end
# Helper Functions
# Function to get p_max and p_min of Generator
# p_max and p_min for Hydro and Renewable. For these types, p_max is a
# time series and not an Int64. Currently, assigning max of time series values
# as for these types of Generator
#####################################################################################
function parse_p_minmax!(comp_values::Base.ValueIterator, comp_dict::Dict{Any, Any})
    p_max_values = []
    p_min_values = []
    ts_flag = 0
    for val in comp_values
        p_max_val = get(val,"p_max","None")
        if p_max_val isa Number
            push!(p_max_values,p_max_val)
        else
            push!(p_max_values,maximum(get(p_max_val,"values","None")))
            ts_flag +=1
        end

        p_min_val = get(val,"p_min","None")
        if p_min_val isa Number
            push!(p_min_values,p_min_val)
        else
            if (get(val,"fuel","None") =="Hydro")
                push!(p_min_values,maximum(get(p_min_val ,"values","None")))
            else
                push!(p_min_values,0)
            end
        end
    end

    push!(comp_dict,"p_max" => p_max_values)
    push!(comp_dict,"p_min" => p_min_values)

    # Time series availability
    flag = false
    if (ts_flag >0)
        flag = true
    else
        flag = false
    end

    return flag
end
#####################################################################################
# Functions to parse the EGRET Dict 'fuel_type'
# EGRET Code (https://github.com/grid-parity-exchange/Egret/blob/main/egret/parsers/rts_gmlc/parser.py#L560-L590)
# f[0] = (float(row['HR_avg_0'])*1000./ 1000000.)*x[0]
# for HR_0 : float(row['HR_avg_0'] = (f[0]/x[0])*1000
# f[i] = (((x[i]-x[i-1])*(float(row[f'HR_incr_{i}'])*1000. / 1000000.))) + f[i-1]
# Others: float(row[f'HR_incr_{i}'] = ((f[i] - f[i-1])/(x[i] - x[i-1]))*1000
#####################################################################################
function parse_fuel_dict!(comp_values::Base.ValueIterator, comp_dict::Dict{Any, Any},num_data_points::Int64)
    for i in 1:num_data_points
        push!(comp_dict,"output_pct_$(i-1)" =>[])
        push!(comp_dict,"HR_avg_$(i-1)" =>[])
    end

    for (comp_p_max,fuel_dict) in zip(get(comp_dict,"p_max","None"),get.(comp_values,"p_fuel","None"))
        if (fuel_dict != "None")
            if (length(fuel_dict["values"]) < num_data_points)
                for i in 1:length(fuel_dict["values"])
                    push!(comp_dict["output_pct_$(i-1)"],fuel_dict["values"][i][1]/comp_p_max)
                    if (i==1)
                        HR_temp= (fuel_dict["values"][i][2]/fuel_dict["values"][i][1])*1000
                        push!(comp_dict["HR_avg_$(i-1)"],HR_temp)
                    else
                        HR_temp= ((fuel_dict["values"][i][2] - fuel_dict["values"][i-1][2])/(fuel_dict["values"][i][1] - fuel_dict["values"][i-1][1]))*1000
                        push!(comp_dict["HR_avg_$(i-1)"],HR_temp)
                    end
                end
                for i in 1+length(fuel_dict["values"]):num_data_points
                    push!(comp_dict["output_pct_$(i-1)"],0)
                    push!(comp_dict["HR_avg_$(i-1)"],0)
                end
            else
                for i in 1:num_data_points
                    push!(comp_dict["output_pct_$(i-1)"],fuel_dict["values"][i][1]/comp_p_max)
                    if (i==1)
                        HR_temp= (fuel_dict["values"][i][2]/fuel_dict["values"][i][1])*1000
                        push!(comp_dict["HR_avg_$(i-1)"],HR_temp)
                    else
                        HR_temp= ((fuel_dict["values"][i][2] - fuel_dict["values"][i-1][2])/(fuel_dict["values"][i][1] - fuel_dict["values"][i-1][1]))*1000
                        push!(comp_dict["HR_avg_$(i-1)"],HR_temp)
                    end
                end
            end
        else
            for i in 1:num_data_points
                push!(comp_dict["output_pct_$(i-1)"],0)
                push!(comp_dict["HR_avg_$(i-1)"],0)
            end
        end
    end
end
#####################################################################################
# Functions to parse the EGRET Dict 'startup_fuel'
# EGRET Code (https://github.com/grid-parity-exchange/Egret/blob/main/egret/parsers/rts_gmlc/parser.py#L593-L624)
#=
startup_heat = (float(row['Start Heat Hot MBTU']),
                float(row['Start Heat Warm MBTU']),
                float(row['Start Heat Cold MBTU']))
startup_time = (float(row['Start Time Hot Hr']),
                float(row['Start Time Warm Hr']),
                float(row['Start Time Cold Hr']))
=#
#####################################################################################
function parse_startup_fuel_dict!(comp_values::Base.ValueIterator, comp_dict::Dict{Any, Any})

    lookup_dict = Dict([(1, ("Start Time Cold Hr","Start Heat Cold MBTU")), (2, ("Start Time Warm Hr","Start Heat Warm MBTU")),
                       (3, ("Start Time Hot Hr","Start Heat Hot MBTU"))]);

    push!(comp_dict,"Start Heat Hot MBTU" =>[])
    push!(comp_dict,"Start Heat Warm MBTU" =>[])
    push!(comp_dict,"Start Heat Cold MBTU" =>[])
    push!(comp_dict,"Start Time Hot Hr" =>[])
    push!(comp_dict,"Start Time Warm Hr" =>[])
    push!(comp_dict,"Start Time Cold Hr" =>[])

    for startup_dict in get.(comp_values,"startup_fuel","None")
        if (startup_dict != "None")
            for i in 1:3
                try
                    push!(comp_dict[lookup_dict[i][1]],startup_dict[i][1])
                    push!(comp_dict[lookup_dict[i][2]],startup_dict[i][2])
                catch ex
                    push!(comp_dict[lookup_dict[i][1]],0)
                    if (startup_dict[1][2] !=0)
                        push!(comp_dict[lookup_dict[i][2]],startup_dict[1][2])
                    else
                        push!(comp_dict[lookup_dict[i][2]],0)
                    end
                end
            end
        else
            for i in 1:3
                push!(comp_dict[lookup_dict[i][1]],0)
                push!(comp_dict[lookup_dict[i][2]],0)
            end
        end
    end
end
#####################################################################################
# Common function to make CSV's related to generator time series data
#####################################################################################
function make_gen_time_series!(time_stamps_DA::Vector{Dates.DateTime},dir_name::String,components_DA::Vector{Any},df_ts_pointer::DataFrames.DataFrame,gen_type::String;
                                time_stamps_RT::Union{Nothing, Vector{Dates.DateTime}} = nothing,components_RT::Union{Nothing, Vector{Any}} = nothing)

    ts_resolution_DA = (Dates.Second(last(time_stamps_DA) - first(time_stamps_DA)))/(length(time_stamps_DA)-1)
    if (time_stamps_RT !== nothing)
        ts_resolution_RT = (Dates.Second(last(time_stamps_RT) - first(time_stamps_RT)))/(length(time_stamps_RT)-1)
    end

    # Check if RT components are passed
    comps_dict = Dict("DAY_AHEAD" => (components_DA,time_stamps_DA,ts_resolution_DA.value))
    if (components_RT !== nothing)
        push!(comps_dict,"REAL_TIME" => (components_RT,time_stamps_RT,ts_resolution_RT.value))
    end
    for comp_key in keys(comps_dict)
        df = DataFrames.DataFrame()

        df[!,"DateTime"] = comps_dict[comp_key][2]
        comp_names = []
        comp_p_max = []
        for idx in 1:length(comps_dict[comp_key][1])
            df[!,comps_dict[comp_key][1][idx][1]] =  get(comps_dict[comp_key][1][idx][2],"p_max","None")["values"]
            push!(comp_names,comps_dict[comp_key][1][idx][1])
            push!(comp_p_max,maximum(df[!,comps_dict[comp_key][1][idx][1]]))
        end
        # Export CSV
        csv_path = joinpath(dir_name,comp_key*"_"*gen_type*".csv")
        CSV.write(csv_path, df,writeheader = true)

        # Pointers Dict
        num_components = length(comp_names)
        pointers_dict = Dict()
        push!(pointers_dict,"simulation" =>fill(comp_key,num_components))
        push!(pointers_dict,"resolution" =>fill(comps_dict[comp_key][3],num_components))
        push!(pointers_dict,"category" =>"Generator")
        push!(pointers_dict,"component_name" =>comp_names)
        push!(pointers_dict,"module" =>"PowerSystems")
        push!(pointers_dict,"type" =>"SingleTimeSeries")
        push!(pointers_dict,"name" =>"max_active_power")
        push!(pointers_dict,"scaling_factor_multiplier" =>"get_max_active_power")
        push!(pointers_dict,"scaling_factor_multiplier_module" =>"PowerSystems")
        push!(pointers_dict,"normalization_factor" =>comp_p_max)
        push!(pointers_dict,"data_file" =>fill(csv_path,num_components))

        append!(df_ts_pointer,pointers_dict)
    end
end
#####################################################################################
# Parse time series data
# **TODO: Need to be generaized to handle RT Systems and reserves
#####################################################################################
function time_series_processing(dir_name::String,areas_DA::Dict{String, Any},system_DA::Dict{String, Any};loads_DA::Union{Nothing, Dict{String, Any}} = nothing,
                                area_bus_mapping_dict::Union{Nothing, Dict{Any, Any}} = nothing,gen_components_DA::Union{Nothing, Dict{String, Any}} = nothing,
                                areas_RT::Union{Nothing, Dict{String, Any}} = nothing,system_RT::Union{Nothing, Dict{String, Any}} = nothing,
                                loads_RT::Union{Nothing, Dict{String, Any}} = nothing,gen_components_RT::Union{Nothing, Dict{String, Any}} = nothing)
    #Time stamp processing
    # Day-Ahead
    rt_flag = false

    date_format = Dates.DateFormat("Y-m-d H:M")
    time_stamps_DA = Dates.DateTime.(system_DA["time_keys"],date_format)
    ts_resolution_DA = (Dates.Second(last(time_stamps_DA) - first(time_stamps_DA)))/(length(time_stamps_DA)-1)

    # Real-Time
    if (system_RT !== nothing)
        time_stamps_RT = Dates.DateTime.(system_RT["time_keys"],date_format)
        ts_resolution_RT = (Dates.Second(last(time_stamps_RT) - first(time_stamps_RT)))/(length(time_stamps_RT)-1)
    end

    # Make time series data folder

    ts_dir_name = joinpath(dir_name,"timeseries_data_files")

    if (~isdir(ts_dir_name))
        mkpath(ts_dir_name)
    end

    # Time series pointer CSV
    df_ts_pointer = DataFrames.DataFrame()

    # Reserves Metadata CSV
    df_reserves_metadata = DataFrames.DataFrame()

    # Reserves Metadata CSV
    df_simulation_objects = DataFrames.DataFrame()

    #Reserves
    #**TODO - This currently assumes every EGRET JSON passed has all the reserve products RTS_GMLC has. There is an easy fix for this.
    gen_fuel_unit_types = [u_t for u_t in unique(get.(values(gen_components_DA),"fuel","None").*" ".*get.(values(gen_components_DA),"unit_type","None"))
                                if ~(u_t in ["Sync_Cond SYNC_COND","Nuclear NUCLEAR", "Solar RTPV"])]
    gen_fuel_unit_types = "("*join(gen_fuel_unit_types,",")*")"

    all_areas_DA = "("*join(collect(keys(areas_DA)),",")*")"

    # Spinning Reserves
    folder_name = joinpath(ts_dir_name,"RESERVES")
    mkpath(folder_name)
    # Check if RT System is passed
    areas_dict = Dict("DAY_AHEAD" => (areas_DA,time_stamps_DA,ts_resolution_DA.value))
    if (areas_RT !== nothing)
        push!(areas_dict,"REAL_TIME" => (areas_RT,time_stamps_RT,ts_resolution_RT.value))
    end

    for area_key in keys(areas_dict)
        object_names = []
        data_files = []
        region_max_values = []
        for (idx,key) in enumerate(keys(areas_dict[area_key][1]))
            df = DataFrames.DataFrame()
            df[!,"DateTime"] = areas_dict[area_key][2]
            column_name = "Spin_Up_R"*"$idx"
            push!(object_names,column_name)
            df[!,column_name] = areas_dict[area_key][1][key]["spinning_reserve_requirement"]["values"]
            push!(region_max_values,maximum(areas_dict[area_key][1][key]["spinning_reserve_requirement"]["values"]))

            # Export CSV
            csv_name = area_key*"_regional_Spin_Up_R"* "$idx" *".csv"
            csv_path = joinpath(folder_name,csv_name)
            push!(data_files,csv_path)
            CSV.write(csv_path, df,writeheader = true)
        end
        # Pointers Dict
        num_areas = length(keys(area_bus_mapping_dict))
        # Timeseries pointers Dict to build the necessary CSV
        pointers_dict = Dict()
        push!(pointers_dict,"simulation" =>fill(area_key,num_areas))
        push!(pointers_dict,"resolution" =>fill(areas_dict[area_key][3],num_areas))
        push!(pointers_dict,"category" =>"Reserve")
        push!(pointers_dict,"component_name" =>object_names)
        push!(pointers_dict,"module" =>"PowerSystems")
        push!(pointers_dict,"type" =>"SingleTimeSeries")
        push!(pointers_dict,"name" =>"requirement")
        push!(pointers_dict,"scaling_factor_multiplier" =>"get_requirement")
        push!(pointers_dict,"scaling_factor_multiplier_module" =>"PowerSystems")
        push!(pointers_dict,"normalization_factor" =>region_max_values)
        push!(pointers_dict,"data_file" =>data_files)

        append!(df_ts_pointer,pointers_dict)

        # Reserves Metadata Dict
        if (area_key =="DAY_AHEAD")
            # Reserves Metadata Dict to build the necessary CSV
            reserves_metadata_dict = Dict()

            push!(reserves_metadata_dict, "Reserve Product"=>object_names)
            push!(reserves_metadata_dict, "Timeframe (sec)"=>fill(600,num_areas))
            push!(reserves_metadata_dict, "Requirement (MW)"=>region_max_values)
            push!(reserves_metadata_dict, "Eligible Regions"=>collect(keys(areas_DA)))
            push!(reserves_metadata_dict, "Eligible Device Categories"=>fill("Generator",num_areas))
            push!(reserves_metadata_dict, "Eligible Device SubCategories"=>fill(gen_fuel_unit_types,num_areas))
            push!(reserves_metadata_dict, "Direction"=>fill("Up",num_areas))

            append!(df_reserves_metadata,reserves_metadata_dict)
        end
    end

    # Regulation Up & Down
    # Up & Down
    # Check if RT System is passed
    reg_dir_dict = Dict([("Up", ("regulation_up_requirement","_regional_Reg_Up.csv","Reg_Up")), ("Down", ("regulation_down_requirement","_regional_Reg_Down.csv","Reg_Down"))]);

    regulation_dict = Dict("DAY_AHEAD" => (system_DA,time_stamps_DA,24,ts_resolution_DA.value))
    if (system_RT !== nothing)
        push!(regulation_dict,"REAL_TIME" => (system_RT,time_stamps_RT,288,ts_resolution_RT.value))
    end
    for dir in keys(reg_dir_dict)
        for reg_up_key in keys(regulation_dict)
            df = DataFrames.DataFrame()
            max_reserve_vals = []
            for i in 1:length(regulation_dict[reg_up_key][2])÷ regulation_dict[reg_up_key][3]
                reserve_dict = Dict()
                start_data_range = ((i-1)*regulation_dict[reg_up_key][3] +1)
                data_range = range(start_data_range,length=regulation_dict[reg_up_key][3])
                start_ts = regulation_dict[reg_up_key][2][start_data_range]
                for (idx,val) in enumerate(regulation_dict[reg_up_key][1][reg_dir_dict[dir][1]]["values"][data_range])
                    push!(reserve_dict, string(idx) => val)
                end
                max_reserve_val = maximum(values(reserve_dict))
                push!(max_reserve_vals,max_reserve_val)
                year_value = Dates.Year(start_ts).value
                month_value = Dates.Month(start_ts).value
                day_value = Dates.Day(start_ts).value
                push!(reserve_dict,"Year" =>year_value)
                push!(reserve_dict,"Month" => month_value)
                push!(reserve_dict,"Day" =>day_value)

                append!(df,reserve_dict)
            end
            csv_path = joinpath(folder_name,reg_up_key*reg_dir_dict[dir][2])
            CSV.write(csv_path, df,writeheader = true)

            # Pointers Dict
            pointers_dict = Dict()
            push!(pointers_dict,"simulation" =>reg_up_key)
            push!(pointers_dict,"resolution" =>regulation_dict[reg_up_key][4])
            push!(pointers_dict,"category" =>"Reserve")
            push!(pointers_dict,"component_name" =>reg_dir_dict[dir][3])
            push!(pointers_dict,"module" =>"PowerSystems")
            push!(pointers_dict,"type" =>"SingleTimeSeries")
            push!(pointers_dict,"name" =>"requirement")
            push!(pointers_dict,"scaling_factor_multiplier" =>"get_requirement")
            push!(pointers_dict,"scaling_factor_multiplier_module" =>"PowerSystems")
            push!(pointers_dict,"normalization_factor" =>maximum(max_reserve_vals))
            push!(pointers_dict,"data_file" =>csv_path)

            append!(df_ts_pointer,pointers_dict)
            if (reg_up_key =="DAY_AHEAD")
                # Reserves Metadata Dict
                reserves_metadata_dict = Dict()
                push!(reserves_metadata_dict, "Reserve Product"=>reg_dir_dict[dir][3])
                push!(reserves_metadata_dict, "Timeframe (sec)"=>300)
                push!(reserves_metadata_dict, "Requirement (MW)"=>maximum(max_reserve_vals))
                push!(reserves_metadata_dict, "Eligible Regions"=>all_areas_DA)
                push!(reserves_metadata_dict, "Eligible Device Categories"=>"Generator")
                push!(reserves_metadata_dict, "Eligible Device SubCategories"=>gen_fuel_unit_types)
                push!(reserves_metadata_dict, "Direction"=>dir)

                append!(df_reserves_metadata,reserves_metadata_dict)
            end
        end
    end

    # Flexible Ramp Up & Down
    # Not available for RT System (must be handled)
    # Up
    flex_dir_dict = Dict([("Up", ("flexible_ramp_up_requirement","_regional_Flex_Up.csv","Flex_Up")), ("Down", ("flexible_ramp_down_requirement","_regional_Flex_Down.csv","Flex_Down"))]);

    for dir in keys(flex_dir_dict)
        df = DataFrames.DataFrame()
        max_reserve_vals = []
        for i in 1:length(time_stamps_DA)÷ 24
            reserve_dict = Dict()
            start_data_range = ((i-1)*24 +1)
            data_range = range(start_data_range,length=24)
            start_ts = time_stamps_DA[start_data_range]
            for (idx,val) in enumerate(system_DA[flex_dir_dict[dir][1]]["values"][data_range])
                push!(reserve_dict, string(idx) => val)
            end
            max_reserve_val = maximum(values(reserve_dict))
            push!(max_reserve_vals,max_reserve_val)
            year_value = Dates.Year(start_ts).value
            month_value = Dates.Month(start_ts).value
            day_value = Dates.Day(start_ts).value
            push!(reserve_dict,"Year" =>year_value)
            push!(reserve_dict,"Month" => month_value)
            push!(reserve_dict,"Day" =>day_value)

            append!(df,reserve_dict)
        end
        csv_path = joinpath(folder_name,"DAY_AHEAD"*flex_dir_dict[dir][2])
        CSV.write(csv_path, df,writeheader = true)

        # Pointers Dict
        pointers_dict = Dict()
        push!(pointers_dict,"simulation" =>"DAY_AHEAD")
        push!(pointers_dict,"resolution" =>ts_resolution_DA.value)
        push!(pointers_dict,"category" =>"Reserve")
        push!(pointers_dict,"component_name" =>flex_dir_dict[dir][3])
        push!(pointers_dict,"module" =>"PowerSystems")
        push!(pointers_dict,"type" =>"SingleTimeSeries")
        push!(pointers_dict,"name" =>"requirement")
        push!(pointers_dict,"scaling_factor_multiplier" =>"get_requirement")
        push!(pointers_dict,"scaling_factor_multiplier_module" =>"PowerSystems")
        push!(pointers_dict,"normalization_factor" =>maximum(max_reserve_vals))
        push!(pointers_dict,"data_file" =>csv_path)

        append!(df_ts_pointer,pointers_dict)

        # Reserves Metadata Dict
        reserves_metadata_dict = Dict()
        push!(reserves_metadata_dict, "Reserve Product"=>flex_dir_dict[dir][3])
        push!(reserves_metadata_dict, "Timeframe (sec)"=>1200)
        push!(reserves_metadata_dict, "Requirement (MW)"=>maximum(max_reserve_vals))
        push!(reserves_metadata_dict, "Eligible Regions"=>all_areas_DA)
        push!(reserves_metadata_dict, "Eligible Device Categories"=>"Generator")
        push!(reserves_metadata_dict, "Eligible Device SubCategories"=>gen_fuel_unit_types)
        push!(reserves_metadata_dict, "Direction"=>dir)

        append!(df_reserves_metadata,reserves_metadata_dict)
    end

    # Generator
    # Filtering components with time series data
    if (gen_components_DA !== nothing)
        #DA
        hydro_components_DA = []
        pv_components_DA = []
        rtpv_components_DA =[]
        wind_components_DA =[]
        #RT
        hydro_components_RT = []
        pv_components_RT = []
        rtpv_components_RT =[]
        wind_components_RT =[]
        for gen_key in keys(gen_components_DA)
            gen_unit_type = get(gen_components_DA[gen_key],"unit_type","None")

            if (gen_unit_type == "HYDRO")
                push!(hydro_components_DA,gen_key => gen_components_DA[gen_key])
            end
            if (gen_unit_type == "PV")
                push!(pv_components_DA,gen_key => gen_components_DA[gen_key])
            end
            if (gen_unit_type == "RTPV")
                push!(rtpv_components_DA,gen_key => gen_components_DA[gen_key])
            end
            if (gen_unit_type == "WIND")
                push!(wind_components_DA,gen_key => gen_components_DA[gen_key])
            end
        end
        if (gen_components_RT!== nothing)
            for gen_key in keys(gen_components_RT)
                gen_unit_type = get(gen_components_RT[gen_key],"unit_type","None")

                if (gen_unit_type == "HYDRO")
                    push!(hydro_components_RT,gen_key => gen_components_RT[gen_key])
                end
                if (gen_unit_type == "PV")
                    push!(pv_components_RT,gen_key => gen_components_RT[gen_key])
                end
                if (gen_unit_type == "RTPV")
                    push!(rtpv_components_RT,gen_key => gen_components_RT[gen_key])
                end
                if (gen_unit_type == "WIND")
                    push!(wind_components_RT,gen_key => gen_components_RT[gen_key])
                end
            end
        end
        # Call functions for different types of Generator
        gen_unit_dict = Dict([("HYDRO", (hydro_components_DA,hydro_components_RT)), ("PV", (pv_components_DA,pv_components_RT)),
                             ("RTPV", (rtpv_components_DA,rtpv_components_RT)),("WIND", (wind_components_DA,wind_components_RT))]);

        for u_t_key in keys(gen_unit_dict) # in.(keys(gen_unit_dict), Ref(get.(values(gen_components),"unit_type","None")))
            u_t_avail = in(u_t_key, get.(values(gen_components_DA),"unit_type","None"))
            if (u_t_avail)
                folder_name = joinpath(ts_dir_name,u_t_key)
                mkpath(folder_name)
                if (length(gen_unit_dict[u_t_key][2]) > 0)
                    make_gen_time_series!(time_stamps_DA,folder_name,gen_unit_dict[u_t_key][1],df_ts_pointer,u_t_key,time_stamps_RT= time_stamps_RT,components_RT = gen_unit_dict[u_t_key][2])
                else
                    make_gen_time_series!(time_stamps_DA,folder_name,gen_unit_dict[u_t_key][1],df_ts_pointer,u_t_key)
                end
            end
        end
    end
    # Loads
    if (loads_DA !== nothing)
        folder_name = joinpath(ts_dir_name,"LOAD")
        mkpath(folder_name)

        loads_dict = Dict("DAY_AHEAD" => (loads_DA,time_stamps_DA,ts_resolution_DA.value))
        if (loads_RT !== nothing)
            push!(loads_dict,"REAL_TIME" => (loads_RT,time_stamps_RT,ts_resolution_RT.value))
        end
        for load_key in keys(loads_dict)
            df = DataFrames.DataFrame()
            df[!,"DateTime"] = loads_dict[load_key][2]
            reg_load_max_vals = []
            for key in keys(area_bus_mapping_dict)
                area_loads = filter(!isnothing,get.(Ref(loads_dict[load_key][1]),area_bus_mapping_dict[key],nothing)) # cannot directly broacast without filter because not all buses have loads
                filtered_area_load_vals  = get.(get.(area_loads,"p_load",0),"values",0)
                sum_area_load = sum(filtered_area_load_vals)
                push!(reg_load_max_vals, maximum(sum_area_load))

                df[!,key] = sum_area_load
            end

            # Export CSV
            csv_path = joinpath(folder_name,load_key*"_regional_Load.csv")
            CSV.write(csv_path, df,writeheader = true)

            # Pointers Dict
            num_areas = length(keys(area_bus_mapping_dict))
            pointers_dict = Dict()
            push!(pointers_dict,"simulation" =>fill(load_key,num_areas))
            push!(pointers_dict,"resolution" =>fill(loads_dict[load_key][3],num_areas))
            push!(pointers_dict,"category" =>"Area")
            push!(pointers_dict,"component_name" =>collect(keys(area_bus_mapping_dict)))
            push!(pointers_dict,"module" =>"PowerSystems")
            push!(pointers_dict,"type" =>"SingleTimeSeries")
            push!(pointers_dict,"name" =>"max_active_power")
            push!(pointers_dict,"scaling_factor_multiplier" =>"get_max_active_power")
            push!(pointers_dict,"scaling_factor_multiplier_module" =>"PowerSystems")
            push!(pointers_dict,"normalization_factor" =>reg_load_max_vals)
            push!(pointers_dict,"data_file" =>fill(csv_path,num_areas))

            append!(df_ts_pointer,pointers_dict)
        end
    end

    # Export timeseries_pointers CSV
    csv_path = joinpath(dir_name,"timeseries_pointers.csv")
    CSV.write(csv_path, df_ts_pointer,writeheader = true)

    # Export timeseries_pointers JSON
    df_to_json(df_ts_pointer,dir_name)

    # Export reserves metadata CSV
    csv_path = joinpath(dir_name,"reserves.csv")
    CSV.write(csv_path, df_reserves_metadata,writeheader = true)

    # Export simulation objects CSV
    simulation_objects_dict = Dict()
    push!(simulation_objects_dict, "Simulation_Parameters" => ["Periods_per_Step","Period_Resolution","Date_From","Date_To","Look_Ahead_Periods_per_Step",
                                   "Look_Ahead_Resolution","Reserve_Products"])
    reserve_types_DA = ["Flex_Up", "Flex_Down", "Spin_Up", "Reg_Up", "Reg_Down"]
    reserve_types_DA = "("*join(reserve_types_DA,",")*")"

    reserve_types_RT = ["Spin_Up", "Reg_Up", "Reg_Down"]
    reserve_types_RT = "("*join(reserve_types_RT,",")*")"

    push!(simulation_objects_dict,"DAY_AHEAD" => [24, ts_resolution_DA.value,first(time_stamps_DA),last(time_stamps_DA),24, ts_resolution_DA.value,reserve_types_DA])
    if (system_RT !== nothing)
        push!(simulation_objects_dict,"REAL_TIME" => [1, ts_resolution_RT.value,first(time_stamps_DA),last(time_stamps_DA),2, ts_resolution_RT.value,reserve_types_RT])
        rt_flag = true
    end

    append!(df_simulation_objects,simulation_objects_dict)

    csv_path = joinpath(dir_name,"simulation_objects.csv")
    CSV.write(csv_path, df_simulation_objects,writeheader = true)

    if(rt_flag == true)
        @info "Successfully parsed DA and RT Systems time series data."
    else
        @info "Successfully parsed DA System time series data."
    end
    return rt_flag
end
#####################################################################################
# Functions to parse EGRET Bus
# Note: Load MW and Load MVAR assigned as max of the time series data.
#####################################################################################
function parse_EGRET_bus(components::Dict{String,Any},loads::Dict{String, Any},dir_name::String;shunt::Union{Nothing, Dict{String, Any}} = nothing)
    comp_dict = Dict()
    comp_names = collect(keys(components))
    push!(comp_dict, "Name" => comp_names)
    comp_dict_values = values(components)

    for comp_field in keys(first(comp_dict_values))
        push!(comp_dict, comp_field => get.(comp_dict_values,comp_field,"None"))
    end

    # Parse Shunt elements
    if (shunt !== nothing)
        branch_shunt_dicts = get.(Ref(shunt),comp_names,0)
        branch_shunt_vals = []
        for shunt_dict in branch_shunt_dicts
            if (shunt_dict != 0)
                push!(branch_shunt_vals, get(shunt_dict,"bs","None"))
            else
                push!(branch_shunt_vals, 0)
            end
        end
        push!(comp_dict,"MVAR Shunt" => branch_shunt_vals)
    end

    # Include MW Shunt G column
    push!(comp_dict,"MW Shunt G" => zeros(Int64, length(comp_dict_values)))

    # Parse loads
    ts_flag = 0
    load_dicts = get.(Ref(loads),comp_names,0)
    mw_load_vals = []
    mvar_load_vals = []

    for load_dict in load_dicts
        if (load_dict != 0)
            push!(mw_load_vals, maximum(get(get(load_dict,"p_load","None"),"values","None")))
            push!(mvar_load_vals, maximum(get(get(load_dict,"q_load","None"),"values","None")))
            ts_flag +=1
        else
            push!(mw_load_vals, 0)
            push!(mvar_load_vals, 0)
        end
    end
    push!(comp_dict,"MW Load" => mw_load_vals)
    push!(comp_dict,"MVAR Load" => mvar_load_vals)

    df = DataFrames.DataFrame(comp_dict)

    # Export CSV
    csv_path = joinpath(dir_name,"bus.csv")
    CSV.write(csv_path, df,writeheader = true,transform = (col, val) -> something(val, missing))

    # Make a mapping Dict from Area => Bus Name
    area_names = unique(get.(comp_dict_values,"area","None"))
    area_bus_mapping_dict = Dict()
    for area_name in area_names
        push!(area_bus_mapping_dict,area_name =>String[])
    end
    for name in comp_names
        push!(area_bus_mapping_dict[get(components[name],"area","None")],name)
    end

    # Make a mapping Dict from Bus Name => Bus ID.
    bus_name_id_mapping_dict = Dict()
    for (name,id) in zip(get(comp_dict,"Name","None"),get(comp_dict,"id","None"))
        push!(bus_name_id_mapping_dict, name => id)
    end
    # Time series availability
    flag = false
    if (ts_flag >0)
        flag = true
    else
        flag = false
    end

    @info "Successfully parsed buses in the JSON."
    return bus_name_id_mapping_dict,area_bus_mapping_dict,flag
end
#####################################################################################
# Functions to parse EGRET Branch
# Add IDs for from and to and check with original source data
#####################################################################################
function parse_EGRET_branch(components::Dict{String,Any},mapping_dict::Dict{Any,Any},dir_name::String)
    comp_dict = Dict()
    comp_names = collect(keys(components))
    push!(comp_dict, "Name" => comp_names)
    comp_dict_values = values(components)

    for comp_field in keys(first(comp_dict_values))
        push!(comp_dict, comp_field => get.(comp_dict_values,comp_field,"None"))
    end

    # Replace bus names with Bus IDs using mapping dict
    from_bus_ids = getindex.(Ref(mapping_dict),comp_dict["from_bus"])
    to_bus_ids = getindex.(Ref(mapping_dict),comp_dict["to_bus"])
    delete!(comp_dict,"from_bus")
    delete!(comp_dict,"to_bus")
    push!(comp_dict,"from_bus" => from_bus_ids)
    push!(comp_dict,"to_bus" => to_bus_ids)

    # Handle Transformers
    Tr_Ratio_vals = []
    for comp_dict_value in comp_dict_values
        tap = get(comp_dict_value,"transformer_tap_ratio","None")
        if tap isa Number
            push!(Tr_Ratio_vals,tap)
        else
            push!(Tr_Ratio_vals,0)
        end
    end

    if ("transformer_tap_ratio" in keys(comp_dict))
        delete!(comp_dict,"transformer_tap_ratio")
    end
    push!(comp_dict,"Tr Ratio" => Tr_Ratio_vals)

    df = DataFrames.DataFrame(comp_dict)

    # Export CSV
    csv_path = joinpath(dir_name,"branch.csv")
    CSV.write(csv_path, df,writeheader = true,transform = (col, val) -> something(val, missing))

    @info "Successfully parsed branches in the JSON."
end
#####################################################################################
# Functions to parse EGRET Generator
# **Note EGRET doesn't handle CSP units. SO, if there are any CSP units in the root dataset,
# they will not show up in the converted PSY System!
#####################################################################################
function parse_EGRET_generator(components::Dict{String,Any},mapping_dict::Dict{Any,Any},dir_name::String)
    comp_dict = Dict()
    comp_names = collect(keys(components))
    push!(comp_dict, "Name" => comp_names)
    comp_dict_values = values(components)

    for comp_field in keys(first(comp_dict_values))

        if(comp_field in ["agc_capable","area","bus","fuel","generator_type","in_service","mbase","ramp_q","unit_type","zone"]) # These should return 'nothing' if not available
            push!(comp_dict, comp_field => get.(comp_dict_values,comp_field,nothing))
        end

        if(comp_field in ["fuel_cost", "initial_p_output","initial_q_output","initial_status","min_down_time","min_up_time", "non_fuel_startup_cost","p_max_agc","p_min_agc",
                         "pg", "qg","ramp_agc","ramp_down_60min","ramp_up_60min","shutdown_capacity","shutdown_cost","startup_capacity"]) # These should return '0' if not available
            push!(comp_dict, comp_field => get.(comp_dict_values,comp_field,0))
        end
    end

    # Change the "initial_status" and "initial_p_output" columns so tabular data can understand
    comp_dict["initial_status"] = floor.(Int64,abs.(comp_dict["initial_status"]))
    gen_initial_p = Bool[]
    for p in comp_dict["initial_p_output"]
        p>0 ? push!(gen_initial_p,true) : push!(gen_initial_p,false)
    end
    delete!(comp_dict,"initial_p_output")
    push!(comp_dict,"initial_p_output" =>gen_initial_p)

    # Include Generator Category
    gen_categories = get.(comp_dict_values,"fuel","None").*" ".*get.(comp_dict_values,"unit_type","None")
    push!(comp_dict,"category" => gen_categories)

    # Replace bus names with Bus IDs using mapping dict
    bus_ids = getindex.(Ref(mapping_dict),comp_dict["bus"])
    delete!(comp_dict,"bus")
    push!(comp_dict,"bus" => bus_ids)

    # Parse p_max and p_min
    gen_ts_flag = parse_p_minmax!(comp_dict_values,comp_dict)

    # Parse fuel_dict
    fuel_dicts = get.(comp_dict_values,"p_fuel","None");
    num_data_points = maximum([length(fuel_dict["values"]) for fuel_dict in fuel_dicts if fuel_dict !="None"])
    parse_fuel_dict!(comp_dict_values,comp_dict,num_data_points)

    # Parse startupfuel_dict
    parse_startup_fuel_dict!(comp_dict_values,comp_dict)

    df = DataFrames.DataFrame(comp_dict)

    # Export CSV
    csv_path = joinpath(dir_name,"gen.csv")
    CSV.write(csv_path, df,writeheader = true,transform = (col, val) -> something(val, missing))

    @info "Successfully parsed generators in the JSON."
    return gen_ts_flag
end
#####################################################################################
# Main Function to parse EGRET JSON
#####################################################################################
function parse_EGRET_JSON(EGRET_json_DA::Dict{String, Any};EGRET_json_RT::Union{Nothing, Dict{String, Any}} = nothing,location::Union{Nothing, String} = nothing)
    # Initial Checks

    if (~(haskey(EGRET_json_DA, "elements")) || ~(haskey(EGRET_json_DA, "system")))
        error("Please check the EGRET DA System JSON")
    end
    if (EGRET_json_RT !== nothing)
        if (~(haskey(EGRET_json_RT, "elements")) || ~(haskey(EGRET_json_RT, "system")))
            error("Please check the EGRET RT System JSON")
        end
    end

    #kwargs handling
    if (location === nothing)
        location =dirname(dirname(@__DIR__))
        @warn  "Location to save the exported tabular data not specified. Using the Converted_CSV_Files folder of the module."
    end

    dt_now = Dates.format(Dates.now(),"dd-u-yy-H-M-S");
    dir_name = joinpath(location,"data","Converted_CSV_Files",dt_now,EGRET_json_DA["system"]["name"])

    if (~isdir(dir_name))
        mkpath(dir_name)
    end

    # Parsing different elements in EGRET System
    # Bus
    bus_mapping_dict, area_mapping_dict, load_ts_flag =
    if haskey(EGRET_json_DA["elements"], "bus")
        @info "Parsing buses in EGRET JSON..."
        if (haskey(EGRET_json_DA["elements"], "shunt"))
            parse_EGRET_bus(EGRET_json_DA["elements"]["bus"],EGRET_json_DA["elements"]["load"],dir_name,shunt = EGRET_json_DA["elements"]["shunt"])
        else
            parse_EGRET_bus(EGRET_json_DA["elements"]["bus"],EGRET_json_DA["elements"]["load"],dir_name)
        end
    else
        error("No buses in the EGRET DA System JSON")
    end

    # Branch
    if haskey(EGRET_json_DA["elements"], "branch")
        @info "Parsing branches in EGRET JSON..."
        parse_EGRET_branch(EGRET_json_DA["elements"]["branch"],bus_mapping_dict,dir_name)
    else
        error("No branches in the EGRET DA System JSON")
    end

    # Generator
    gen_ts_flag =
    if haskey(EGRET_json_DA["elements"], "generator")
        @info "Parsing generators in EGRET JSON..."
        parse_EGRET_generator(EGRET_json_DA["elements"]["generator"],bus_mapping_dict,dir_name)
    else
        error("No generators in the EGRET DA System JSON")
    end

    # Calling time series processing functions
    rt_flag =
    if (load_ts_flag && gen_ts_flag)
        if (EGRET_json_RT !== nothing)
            @info "Parsing time series of loads and generators and generating time series metadata for DA and RT Systems..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],loads_DA = EGRET_json_DA["elements"]["load"],
                                   gen_components_DA=EGRET_json_DA["elements"]["generator"],area_bus_mapping_dict=area_mapping_dict,
                                   areas_RT = EGRET_json_RT["elements"]["area"],system_RT=EGRET_json_RT["system"],loads_RT = EGRET_json_RT["elements"]["load"],
                                   gen_components_RT=EGRET_json_RT["elements"]["generator"])
        else
            @info "Parsing time series of loads and generators and generating time series metadata for DA System..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],loads_DA = EGRET_json_DA["elements"]["load"],
                                   gen_components_DA=EGRET_json_DA["elements"]["generator"],area_bus_mapping_dict=area_mapping_dict)
        end
    elseif load_ts_flag
        if (EGRET_json_RT !== nothing)
            @info "Parsing time series of loads and generating time series metadata for DA and RT Systems..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],loads_DA = EGRET_json_DA["elements"]["load"],
                                   area_bus_mapping_dict=area_mapping_dict,areas_RT = EGRET_json_RT["elements"]["area"],
                                   system_RT=EGRET_json_RT["system"],loads_RT = EGRET_json_RT["elements"]["load"])
        else
            @info "Parsing time series of loads and generating time series metadata for DA System..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],loads_DA = EGRET_json_DA["elements"]["load"],
                                   area_bus_mapping_dict=area_mapping_dict)
        end
    elseif gen_ts_flag
        if (EGRET_json_RT !== nothing)
            @info "Parsing time series of generators and generating time series metadata for DA and RT Systems..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],gen_components_DA=EGRET_json_DA["elements"]["generator"],
                                   areas_RT = EGRET_json_RT["elements"]["area"],system_RT=EGRET_json_RT["system"],gen_components_RT=EGRET_json_RT["elements"]["generator"])
        else
            @info "Parsing time series of generators and generating time series metadata for DA System..."
            time_series_processing(dir_name,EGRET_json_DA["elements"]["area"],EGRET_json_DA["system"],gen_components_DA=EGRET_json_DA["elements"]["generator"])
        end
    else
        @warn "No generator and load time series data available in the EGRET DA JSON"
    end

    @info "Successfully generated CSV files compatible with SIIP PSY tabular data parser here : $(dir_name)."

    return dir_name, EGRET_json_DA["system"]["baseMVA"],rt_flag
end


text:
"""
## Parameteric types
* `T`: should be a symbol to identify the parameter.
* `R`: the value type of the parameter.
"""
abstract type AbstractParameter{T,R} end

"""
    PolymerParameter{T, R} <: AbstractParameter

Type represents phyiscal parameters of a polymer system. Each type with a specific T (normally a symbol) defines a parameter. E.g.

    ```julia
    PolymerParameter{:ϕ} # define the ϕ parameter
    ```

Pre-defined parameter symbols:
* χ: Flory-Huggins interaction parameters.
* N: chain length in number of monomers or Kuhn segments.
* f: volume fraction of a block in a block copolymer.
* ϕ: volume fraction of a type of chain in a polymer system.
* Rg: radius of gyration of a polymer chain.
* C: chain density C = ρ₀Rg^3/N.
* b: length of a monomer or a Kuhn segment.
* α: chain length of a chain with respect to a reference chain.
* τ: ratio of length of two or more blocks.

Fields
* `description`: a description of the parameter.
* `variable_name`: a string representation of the parameter which can be used as Julia variable name.
* `ascii_label`: a label with pure ascii characters which shall be used for file names, directory names, etc.
* `plot_label`: a LaTeXStrings used as label for plotting.
* `value_type`: the number type of the parameter, e.g. integer, float, complex, etc.
* `allowed_min`: minimum value allowed. If negative infinity, using `typemin(Type)`.
* `allowed_max`: maximum value allowed. If positive infinity, using `typemax(Type)`.
* `allowed_values`: only used for discrete finite set. If the allowed values are infinite, use `allowed_min` and `allowed_max` instead.
* `disallowed_values`: an array of values which are invalid.

User can define their own parameter symbols as long as they also define following two field values. Other fields are deduced automatically from parametric type `T`.
* `description`
* `plot_label`
"""
struct PolymerParameter{T, R<:Real} <: AbstractParameter{T,R}
    description::String
    variable_name::String
    ascii_label::String
    plot_label::LaTeXString
    value_type::Type{R}
    allowed_min::Union{R, typeof(Inf)}
    allowed_max::Union{R, typeof(Inf)}
    allowed_values::AbstractVector{R}
    disallowed_values::AbstractVector{R}
    function PolymerParameter{T}(desc, plot_label, value_type::Type{R}; allowed_min=-Inf, allowed_max=Inf, allowed_values=R[], disallowed_values=R[]) where {T, R<:Real}
        varname = String(T)
        ascii_label = ""
        for c in varname
            ascii_label *= unicodesymbol2string(c)
        end
        new{T, R}(desc, varname, ascii_label, plot_label, value_type, allowed_min, allowed_max, allowed_values, disallowed_values)
    end
end

"""
A list of pre-defined polymer parameters.
"""
const χParam = PolymerParameter{:χ}(
    "Flory-Huggins Interaction Parameter.",
    L"\chi", Float64)
const NParam = PolymerParameter{:N}(
    "Chain length in number of monomers or Kuhn segments.",
    L"N", Int; allowed_min=zero(Int))
const χNParam = PolymerParameter{:χN}(
    "Flory-Huggins Interaction Parameter times N.",
    L"\chi N", Float64)
const fParam = PolymerParameter{:f}(
    "Volume fraction of a block in a block copolymer.",
    L"f", Float64; allowed_min=0, allowed_max=1)
const ϕParam = PolymerParameter{:ϕ}(
    "volume fraction of a type of chain in a polymer system.",
    L"\phi", Float64; allowed_min=0, allowed_max=1)
const RgParam = PolymerParameter{:Rg}(
    "Radius of gyration of a polymer chain.",
    L"R_g", Float64; allowed_min=0)
const CParam = PolymerParameter{:C}(
    "Chain density C = ρ₀Rg^3/N.",
    L"C", Float64; allowed_min=0)
const bParam = PolymerParameter{:b}(
    "Length of a monomer or a Kuhn segment.",
    L"b", Float64; allowed_min=0)
const αParam = PolymerParameter{:α}(
    "Chain length of a chain with respect to a reference chain.",
    L"\alpha", Float64; allowed_min=0)
const τParam = PolymerParameter{:τ}(
    "Ratio of length of two or more blocks.",
    L"\tau", Float64; allowed_min=0)

const DEFAULT_PARAMETERS = Dict(
    :χ => χParam,
    :N => NParam,
    :χN => χNParam,
    :f => fParam,
    :ϕ => ϕParam,
    :Rg => RgParam,
    :C => CParam,
    :b => bParam,
    :α => αParam,
    :τ => τParam,
)

"""
Accessors for the `PolymerParameter` type.
"""
description(p::AbstractParameter) = p.description
value_type(p::AbstractParameter) = p.value_type
variable_symbol(::AbstractParameter{T,R}) where {T,R} = T
as_variable_name(p::AbstractParameter) = p.variable_name
as_ascii_label(p::AbstractParameter) = p.ascii_label
as_plot_label(p::AbstractParameter) = p.plot_label


text:
options = Dict(
	"tagsfile" => "MagnaTagATune/annotations_final.csv",
	"samplespath" => "MagnaTagATune/mp3/",
	"pathcolumn" => "mp3_path",
	"excludedcolumns" => ["clip_id"],
)

max_stars_repo_name: gabriel-ss/music-autotagger

max_issues_repo_name: gabriel-ss/music-autotagger

max_forks_repo_name: gabriel-ss/music-autotagger


text:
using AWSCore
using AWSSDK.S3

struct S3Store <: AbstractStore
    bucket::String
    store::String
    listversion::Int
    aws::Dict{Symbol, Any}
end

function S3Store(bucket::String, store::String;
  listversion = 2,
  aws = nothing,
  region = get(ENV, "AWS_DEFAULT_REGION", "us-east-1"),
  creds = nothing,
  )
  if aws === nothing
    aws = aws_config(creds=creds,region=region)
  end
  S3Store(bucket, store, listversion, aws)
end

Base.show(io::IO,s::S3Store) = print(io,"S3 Object Storage")

function error_is_ignorable(e)
  isa(e,AWSCore.AWSException) && (e.code=="NoSuchKey" || e.code=="404")
end

function Base.getindex(s::S3Store, i::String)
  try
    return S3.get_object(s.aws,Bucket=s.bucket,Key=joinpath(s.store,i))
  catch e
    if error_is_ignorable(e)
      return nothing
    else
      throw(e)
    end
  end
end
getsub(s::S3Store, d::String) = S3Store(s.bucket, joinpath(s.store,d), s.listversion, s.aws)

function storagesize(s::S3Store)
  r = cloud_list_objects(s)
  haskey(r,"Contents") || return 0
  contents = r["Contents"]
  datafiles = filter(entry -> !any(filename -> endswith(entry["Key"], filename), [".zattrs",".zarray",".zgroup"]), contents)
  if isempty(datafiles)
    0
  else
    sum(datafiles) do f
      parse(Int, f["Size"])
    end
  end
end

function zname(s::S3Store)
  d = split(s.store,"/")
  i = findlast(!isempty,d)
  d[i]
end

function isinitialized(s::S3Store, i::String)
  try
    S3.head_object(s.aws,Bucket=s.bucket,Key=joinpath(s.store,i))
    return true
  catch e
    if error_is_ignorable(e)
      return false
    else
      println(joinpath(s.store,i))
      throw(e)
    end
  end
end

function cloud_list_objects(s::S3Store)
  prefix = (isempty(s.store) || endswith(s.store,"/")) ? s.store : string(s.store,"/")
  listfun = s.listversion==2 ? S3.list_objects_v2 : S3.list_objects
  listfun(s.aws, Bucket=s.bucket, prefix=prefix, delimiter = "/")
end
function subdirs(s::S3Store)
  s3_resp = cloud_list_objects(s)
  !haskey(s3_resp,"CommonPrefixes") && return String[]
  allstrings(s3_resp["CommonPrefixes"],"Prefix")
end
function Base.keys(s::S3Store)
  s3_resp = cloud_list_objects(s)
  !haskey(s3_resp,"Contents") && return String[]
  r = allstrings(s3_resp["Contents"],"Key")
  map(i->splitdir(i)[2],r)
end
allstrings(v::AbstractArray,prefixkey) = map(i -> String(i[prefixkey]), v)
allstrings(v,prefixkey) = [String(v[prefixkey])]

path(s::S3Store) = s.store


text:
"""
    BinaryWhiteRBM(a, b, w, affine_v, affine_h)
    BinaryWhiteRBM(a, b, w)

Construct a whitened RBM with binary visible and hidden units.
"""
function BinaryWhiteRBM(
    a::AbstractArray, b::AbstractArray, w::AbstractArray,
    affine_v::Affine, affine_h::Affine
)
    rbm = RBMs.BinaryRBM(a, b, w)
    return WhiteRBM(rbm, affine_v, affine_h)
end

function BinaryWhiteRBM(a::AbstractArray, b::AbstractArray, w::AbstractArray)
    rbm = RBMs.BinaryRBM(a, b, w)
    return WhiteRBM(rbm)
end


text:
# using Optim

@doc raw"""
    sumBassSB(p,q,m,c0,alpha,F,T,Pi)

This function implements Srivastava and Bullo's algorithm to maximize a sum of
Bass functions and returns the maximizer and maximum.
"""
function sumBassSB(p,q,m,c0,alpha,F,T,Pi::Real)
    # max FLP
    ub = maximum(Bass_inflection.(p,q,m,c0))
    # res = optimize(lambda -> -FLP(lambda)[2], 0.0, ub)
    # There are some issues with the optimizer because the objective is discontinuous with the maximizer at one of the discontinuities
    # return FLP(Optim.minimizer(res))
    # Instead make a grid search
    lambdas = collect(range(0,ub,length=1000))
    FLPs = [FLP(lambdas[i],p,q,m,c0,alpha,F,T,Pi)[2] for i in 1:length(lambdas)]
    lambdastar = lambdas[partialsortperm(FLPs,1:50,rev=true)]
    lambdas .= range(minimum(lambdastar),maximum(lambdastar),length=1000)
    FLPs .= [FLP(lambdas[i],p,q,m,c0,alpha,F,T,Pi)[2] for i in 1:length(lambdas)]
    # maximizer = FLP(lambdas[argmax(FLPs)],p,q,m,c0,alpha,F,T,Pi)[1]
    # maximum = sum(alpha .* Bass.(maximizer,p,q,m,c0))
    return FLP(lambdas[argmax(FLPs)],p,q,m,c0,alpha,F,T,Pi)
end

@doc raw"""
    FLP(lambda,p,q,m,c0,alpha,F,T,Pi)

This function evaluates the linear program relaxation of the maximization of the
sum of Bass functions.
"""
function FLP(lambda,p,q,m,c0,alpha,F,T,Pi::Real)
    n = length(p)
    Pi_eff = Pi - sum(F)
    f = Vector{Function}(undef,n)
    # df = Vector{Function}(undef,n)
    # df_inv = Vector{Function}(undef,n)
    # z = Vector{Float64}(undef,n)
    for i in 1:n
        f[i] = t -> alpha[i]*Bass(t,p[i],q[i],m[i],c0[i])
        # df_inv[i] = lambda -> Bass_derivative_inverse(lambda/alpha[i],p[i],q[i],m[i],c0[i])[1]
    end

    df_invs = [Bass_derivative_inverse(lambda/alpha[i],p[i],q[i],m[i],c0[i])[1] for i in 1:n]
    df_invs .= [F[i] < df_invs[i] < T[i] ? df_invs[i] : 0.0 for i in 1:n]

    LPindex = Vector{Float64}(undef,n)
    xLP = Vector{Float64}(undef,n)
    gain_rate = Vector{Float64}(undef,3)
    for i in 1:n
        gain_rate .= [df_invs[i] > 0 ? f[i](df_invs[i])/df_invs[i] : 0, F[i]>0 ? f[i](F[i])/F[i] : 0, T[i] > 0 ? f[i](T[i])/T[i] : 0]
        xLPind = argmax(gain_rate)
        xLP[i] = [df_invs[i],F[i],T[i]][xLPind]
        LPindex[i] = gain_rate[xLPind]
    end
    LPperm = sortperm(LPindex,rev = true)
    cum_capacity = 0.0
    k = n
    for i in 1:n
        cum_capacity += xLP[LPperm[i]] - F[LPperm[i]]
        if cum_capacity >= Pi_eff
            k = i
            break
        end
    end
    if k < n
        xLP[LPperm[(k+1):n]] .= F[(k+1):n]
    end
    xLP[LPperm[k]] -= max(cum_capacity - Pi_eff,0.0) # if the capacity is not
    return xLP, sum(alpha[i].*Bass(xLP[i],p[i],q[i],m[i],c0[i]) for i in 1:n)
end

max_stars_repo_name: andres-alban/res-alloc-bass-demands

max_issues_repo_name: andres-alban/res-alloc-bass-demands

max_forks_repo_name: andres-alban/res-alloc-bass-demands


text:
# This is to replicate the language modeling experiments from Zaremba,
# Wojciech, Ilya Sutskever, and Oriol Vinyals. "Recurrent neural
# network regularization." arXiv preprint arXiv:1409.2329 (2014).
#
# Usage: julia rnnlm.jl ptb.train.txt ptb.valid.txt ptb.test.txt
# Type julia rnnlm.jl --help for more options

using Knet
module RNNLM
using Knet, ArgParse
using Knet: regs, getp, setp, stack_length, stack_empty!, params

function main(args=ARGS)
    info("RNN language model example from Zaremba et al. 2014.")
    isa(args, AbstractString) && (args=split(args))
    opts = parse_commandline(args)
    println(opts)
    opts["seed"] > 0 && setseed(opts["seed"])
    opts["dropout"] > 0 && error("TODO: implement dropout")
    dict = Dict{Any,Int32}()
    global data = Any[]
    for f in opts["datafiles"]
        push!(data, LMData(f;  dict=dict,
                           batch=opts["batch_size"],
                           seqlen=opts["seq_length"],
                           ftype=opts["float64"]?Float64:Float32,
                           dense=opts["dense"]))
    end

    vocab_size = length(dict)
    global net = compile(:rnnlm;  layers = opts["layers"], rnn_size = opts["rnn_size"], vocab_size = vocab_size)
    lr = opts["lr"]
    setp(net; lr=lr)
    init = Uniform(-opts["init_weight"], opts["init_weight"])
    for r in params(net); r.op.init=init; end
    if opts["nosharing"]
        setp(net, :forwoverwrite, false)
        setp(net, :backoverwrite, false)
    end
    perp = zeros(length(data)); l=zeros(2); m=zeros(2)

    for ep=1:opts["max_max_epoch"]
        ep > opts["max_epoch"] && (lr /= opts["decay"]; setp(net; lr=lr))
        train(net, data[1], softloss; gclip=opts["max_grad_norm"], losscnt=fill!(l,0), maxnorm=fill!(m,0))
        perp[1] = exp(l[1]/l[2])
        opts["gcheck"]>0 && gradcheck(net,
                                      f->train(f,data[1],softloss;gcheck=true),
                                      f->test(f,data[1],softloss;gcheck=true);
                                      gcheck=opts["gcheck"])
        for idata = 2:length(data)
            ldev = test(net, data[idata], softloss)
            perp[idata] = exp(ldev)
        end
        @show (ep, perp..., m..., lr)
    end
    return (perp..., m...)
end

@knet function rnnlm(word; layers=0, rnn_size=0, vocab_size=0, rnn_type=:lstm1, o...)
    wvec = wdot(word; o..., out=rnn_size) # 1-3
    yrnn = repeat(wvec; o..., frepeat=rnn_type, nrepeat=layers, out=rnn_size) # 4-40 with 41 copy for return
    return wbf(yrnn; o..., out=vocab_size, f=:soft) # 42-46
end

@knet function lstm1(x; o...)   # need version without fbias
    input  = wbf2(x,h; o..., f=:sigm)
    forget = wbf2(x,h; o..., f=:sigm)
    output = wbf2(x,h; o..., f=:sigm)
    newmem = wbf2(x,h; o..., f=:tanh)
    cell = input .* newmem + cell .* forget
    h  = tanh(cell) .* output
    return h
end

reset_trn!(f;o...)=reset!(f, keepstate=true)

function reset_trn_old!(f; o...)
    if stack_length(f) != length(f)
        info("Stack length: $(stack_length(f)) Regs: $(length(f))")
    end
    stack_empty!(f)
    reset_tst!(f; o...)
    for p in regs(f)            # This is necessary so when back looks at time < 0 the stack gives something back.
        p.dif = nothing
        getp(p,:incr) && fill!(p.dif0, 0)
        setp(p, :forw, true)    # I am not sure why this is necessary
        push!(f,p)
    end
end

function reset_tst!(f; keepstate=false)
    for p in regs(f)
        p.out = keepstate && isdefined(p,:out0) ? p.out0 : nothing
    end
end

# _update_dbg = 0

function train(f, data, loss; gcheck=false, gclip=0, maxnorm=nothing, losscnt=nothing)
    reset_trn!(f)
    ystack = Any[]
    for item in data
        if item != nothing
            (x,ygold) = item
            ypred = sforw(f, x)
            # Knet.netprint(f); error(:ok)
            losscnt != nothing && (losscnt[1] += loss(ypred, ygold); losscnt[2] += 1)
            push!(ystack, copy(ygold))
        else                    # end of sequence
            while !isempty(ystack)
                ygold = pop!(ystack)
                sback(f, ygold, loss)
            end
            #error(:ok)
            gcheck && break # return losscnt[1] leave the loss calculation to test # the parameter gradients are cumulative over the whole sequence
            g = (gclip > 0 || maxnorm!=nothing ? gnorm(f) : 0)
            # global _update_dbg; _update_dbg +=1; _update_dbg > 1 && error(:ok)
            update!(f; gscale=(g > gclip > 0 ? gclip/g : 1))
            if maxnorm != nothing
                w=wnorm(f)
                w > maxnorm[1] && (maxnorm[1]=w)
                g > maxnorm[2] && (maxnorm[2]=g)
            end
            reset_trn!(f; keepstate=true)
        end
    end
    # losscnt[1]/losscnt[2]       # this will give per-token loss, should we do per-sequence instead?
end

function test(f, data, loss; gcheck=false)
    #info("testing")
    sumloss = numloss = 0.0
    reset_tst!(f)
    for item in data
        if item != nothing
            (x,ygold) = item
            ypred = forw(f, x)
            # @show (hash(x),hash(ygold),vecnorm0(ypred))
            sumloss += loss(ypred, ygold)
            numloss += 1
        else
            gcheck && return sumloss
            reset_tst!(f; keepstate=true)
        end
    end
    return sumloss/numloss
end

import Base: start, next, done

type LMData; data; dict; batchsize; seqlength; ftype; dense; x; y; end

function LMData(fname::AbstractString; batch=20, seqlen=20, dict=Dict{Any,Int32}(), ftype=Float32, dense=false)
    data = Int32[]
    open(fname) do f
        for l in eachline(f)
            for w in split(l)
                push!(data, get!(dict, w, 1+length(dict)))
            end
            push!(data, get!(dict, "<eos>", 1+length(dict))) # end-of-sentence
        end
    end
    info("Read $fname: $(length(data)) words, $(length(dict)) vocab.")
    LMData(data, dict, batch, seqlen, ftype, dense, nothing, nothing)
end

function next(d::LMData,state)                              	# d.data is the whole corpus represented as a sequence of Int32's
    (nword, eos) = state                                        # eos is true if last output was nothing
    if !eos && (nword % (d.batchsize * d.seqlength) == 0)
        return (nothing, (nword, true))                         # output nothing indicating end of sequence
    end
    segsize = div(length(d.data), d.batchsize)                  # we split it into d.batchsize roughly equal sized segments
    offset = div(nword, d.batchsize)                            # this is how many words have been served so far from each segment
    if issparse(d.x)
        for b = 1:d.batchsize
            idata = (b-1)*segsize + offset
            d.x.rowval[b] = d.data[idata+1]
            d.y.rowval[b] = d.data[idata+2]
        end
    else
        fill!(d.x, 0)
        fill!(d.y, 0)
        for b = 1:d.batchsize
            idata = (b-1)*segsize + offset
            d.x[d.data[idata+1], b] = 1
            d.y[d.data[idata+2], b] = 1
        end
    end
    ((d.x, d.y), (nword + d.batchsize, false))	# each call to next will deliver a single word from each of the d.batchsize segments
end

# The state indicates number of words served, and whether the last output was end-of-sequence nothing
function start(d::LMData)
    ndict = length(d.dict)
    if d.x == nothing || size(d.x) != (ndict, d.batchsize)
        if d.dense
            d.x = zeros(d.ftype, ndict, d.batchsize)
            d.y = zeros(d.ftype, ndict, d.batchsize)
        else
            d.x = sponehot(d.ftype, ndict, d.batchsize)
            d.y = sponehot(d.ftype, ndict, d.batchsize)
        end
    end
    return (0, true)
end

function done(d::LMData,state)
    # nword is the number of sequences served
    # stop if there is not enough data for another full sequence
    (nword, eos) = state
    eos && nword + d.batchsize * d.seqlength > length(d.data)
end

function parse_commandline(args)
    s = ArgParseSettings()
    @add_arg_table s begin
        "--batch_size"
        help = "minibatch size"
        arg_type = Int
        default = 20
        "--seq_length"
        help = "length of the input sequence"
        arg_type = Int
        default = 20
        "--layers"
        help = "number of lstm layers"
        arg_type = Int
        default = 1
        "--decay"
        help = "divide learning rate by this every epoch after max_epoch"
        arg_type = Float64
        default = 2.0
        "--rnn_size"
        help = "size of the lstm"
        arg_type = Int
        default = 200
        "--dropout"
        help = "dropout units with this probability"
        arg_type = Float64
        default =  0.0
        "--init_weight"
        help = "initialize weights uniformly in [-init_weight,init_weight]"
        arg_type = Float64
        default =  0.1
        "--lr"
        help = "learning rate"
        arg_type = Float64
        default = 1.0
        "--vocab_size"
        help = "size of the vocabulary"
        arg_type = Int
        default = 10000
        "--max_epoch"
        help = "number of epochs at initial lr"
        arg_type = Int
        default = 1 # 4
        "--max_max_epoch"
        help = "number of epochs to train"
        arg_type = Int
        default = 1 # 13
        "--max_grad_norm"
        help = "gradient clip"
        arg_type = Float64
        default = 5.0
        "--gcheck"
        help = "gradient check this many units per layer"
        arg_type = Int
        default = 0
        "--seed"
        help = "random seed, use 0 to turn off"
        arg_type = Int
        default = 42
        "--float64"
        help = "Use Float64 (Float32 default)"
        action = :store_true
        "--dense"
        help = "Use dense matrix ops (sparse ops default)"
        action = :store_true
        "--nosharing"
        help = "Do not share register arrays (this is for debugging, should not change results)"
        action = :store_true
        "--preset"
        help = "load one of the preset option combinations"
        arg_type = Int
        default = 0
        "datafiles"
        help = "corpus files: first one will be used for training"
        nargs = '+'
        required = true
    end
    opts = parse_args(args,s)
    # Parameters from the Zaremba implementation:
    if opts["preset"] == 1      # Trains 1h and gives test 115 perplexity.
        opts["batch_size"]=20
        opts["seq_length"]=20
        opts["layers"]=2
        opts["decay"]=2
        opts["rnn_size"]=200
        opts["dropout"]=0
        opts["init_weight"]=0.1
        opts["lr"]=1
        opts["vocab_size"]=10000
        opts["max_epoch"]=4
        opts["max_max_epoch"]=13
        opts["max_grad_norm"]=5
    elseif opts["preset"] == 2  # Train 1 day and gives 82 perplexity.
        opts["batch_size"]=20
        opts["seq_length"]=35
        opts["layers"]=2
        opts["decay"]=1.15
        opts["rnn_size"]=1500
        opts["dropout"]=0.65
        opts["init_weight"]=0.04
        opts["lr"]=1
        opts["vocab_size"]=10000
        opts["max_epoch"]=14
        opts["max_max_epoch"]=55
        opts["max_grad_norm"]=10
    end
    return opts
end

!isinteractive() && !isdefined(Main,:load_only) && main(ARGS)

end # module

### SAMPLE RUN: results may vary because of sparse matrix multiplications.
#
# julia rnnlm.jl --preset 1 ptb.valid.txt ptb.test.txt
# WARNING: requiring "Options" did not define a corresponding module.
# Dict{AbstractString,Any}("rnn_size"=>200,"lr"=>1,"decay"=>2,"batch_size"=>20,"dropout"=>0,"max_grad_norm"=>5,"preset"=>1,"max_epoch"=>4,"init_weight"=>0.1,"gcheck"=>0,"layers"=>2,"vocab_size"=>10000,"seq_length"=>20,"seed"=>42,"max_max_epoch"=>13,"datafiles"=>Any["data/ptb.valid.txt","data/ptb.test.txt"])
# INFO: Read data/ptb.valid.txt: 73760 words, 6022 vocab.
# INFO: Read data/ptb.test.txt: 82430 words, 7596 vocab.
#  37.057684 seconds (29.17 M allocations: 2.530 GB, 1.31% gc time)
#   9.723167 seconds (11.06 M allocations: 496.906 MB, 1.30% gc time)
# (ep,perp...,wmax,gmax,lr) = (1,964.7574812581911,680.8404134165187,359.66156655473776,182.60932042613925,1)
#  32.492913 seconds (24.07 M allocations: 2.306 GB, 1.39% gc time)
#   9.682416 seconds (11.04 M allocations: 496.187 MB, 1.23% gc time)
# (ep,perp...,wmax,gmax,lr) = (2,533.9658674987758,500.7701607049704,376.4021748281246,29.527137764770796,1)
#  32.433211 seconds (24.07 M allocations: 2.306 GB, 1.39% gc time)
#   9.759406 seconds (11.05 M allocations: 496.439 MB, 1.30% gc time)
# (ep,perp...,wmax,gmax,lr) = (3,390.1504382596744,435.37182169855157,390.84524322483344,30.692745370555468,1)

### Compilation results:
# op(46): input,par,dot; 4xadd2(8); mul,mul,add,tanh,mul; par,dot,par,add,soft,softloss
# tosave(11): 1,3,11,19,27,35,38,39,40,45,46
# toincr(18): 15 par + 3 multi
# toback(45): all except 1
# tmp(6/18): 18 toincr, 6 unique sizes
# out(29/46): 15 par + 11 tosave + 3 tmp
# dif(26/45):

# julia> rnnlm("ptb.test.txt")
# (ep,perp...,wmax,gmax,lr) = (1,694.3535880341253,256.35040516045433,159.35646292641206,1.0)

# function rnnlmModel2(;
#                     layers = 0,
#                     rnn_size = 0,
#                     vocab_size = 0,
#                     )
#     prog = quote                # do we need the prefix?  lstm will do a dot?  yes if you want the same embedding going into each gate.
#         i0 = input()
#         x0 = wdot(i0; out=$rnn_size)
#     end
#     s0 = s1 = :x0
#     for n=1:layers
#         s0 = s1
#         s1 = symbol("x$n")
#         op = :($s1 = lstm($s0; out=$rnn_size))
#         push!(prog.args, op)
#     end
#     prog2 = quote
#         ou = wbf($s1; out=$vocab_size, f=soft)
#     end
#     append!(prog.args, prog2.args)
#     return prog
# end


text:
module fns

abstract type FunctionArgument end

struct Arg <: FunctionArgument end
struct KwArg <: FunctionArgument end
struct OptionalArg <: FunctionArgument end

# Code to parse argument Expr into
# `FunctionArgument`s
# ...

#=
Function for Extracting Parameters
Accepts lambda functions only. In the form:
```
@get_args (a,b) -> a + b
```
or
```
@get_args function(a,b)
    a + b
end
```

(TODO) Not yet:
```
@get_args add(a,b) = a + b
```
(TODO) or:
```
function add(a,b)
    a + b
end

@get_args add
```
=#
function get_args(fn::Expr)
  # Check that this function was defined inline
  @assert fn.head in (:->, :function), "Function should be defined inline."
  arg_exprs = fn.args[1].args
  # ...
end

function is_arg(e::Expr)

end

function is_optional(e::Expr)

end

function is_kwargs(e::Expr)

end

function is_kwarg(e::Expr)

end

end # module

# a = Argument
# o = Optional Argument
# k1 = Keyword Argument
# k2 = Keyword Argument
fn = :((a::Int, o::Float64 = 1. ; k1::Int = 1, k2::Float64 = 2.) -> 42)

all_args = fn.args[1].args
#=
3-element Vector{Any}:
 :($(Expr(:parameters, :($(Expr(:kw, :(k1::Int), 1))), :($(Expr(:kw, :(k2::Float64), 2.0))))))
 :(a::Int)
 :(o::Float64 = 1.0)
=#

arg = all_args[2] # :(a::Int)

opt_arg = all_args

kwargs = all_args[1]

#=

Recursively deconstruct function argument `Expr`.

* Args are the simplest :(argname::ArgType)
* Optional Args have head `:(=)` and args `[:(name::Type),"Default Value"]`
* All Kwargs are grouped together into a single expression with head `:parameters`
* Individual kwargs have head `:kw` and args `[:(kwarg_name::Type),"Default Value"]`
=#


text:
using SnoopCompile

### Log the compiles
# This only needs to be run once (to generate "/tmp/colortypes_compiles.csv")

SnoopCompile.@snoopc "snoop.csv" begin
    using Pkg
    include("src/JuliaChem.jl")
end

### Parse the compiles and generate precompilation scripts
# This can be run repeatedly to tweak the scripts

data = SnoopCompile.read("snoop.csv")

pc = SnoopCompile.parcel(reverse!(data[2]))
SnoopCompile.write("snoop", pc)


text:
# Julia wrapper for header: /usr/local/include/libavresample/avresample.h
# Automatically generated using Clang.jl wrap_c, version 0.0.0

export
    #avresample_version,
    avresample_configuration,
    avresample_license,
    avresample_get_class,
    avresample_alloc_context,
    avresample_open,
    avresample_close,
    avresample_free,
    avresample_build_matrix,
    avresample_get_matrix,
    avresample_set_matrix,
    avresample_set_channel_mapping,
    avresample_set_compensation,
    avresample_convert,
    avresample_get_delay,
    avresample_available,
    avresample_read

function avresample_version()
    ccall((:avresample_version,libavresample),UInt32,())
end

function avresample_configuration()
    ccall((:avresample_configuration,libavresample),Ptr{UInt8},())
end

function avresample_license()
    ccall((:avresample_license,libavresample),Ptr{UInt8},())
end

function avresample_get_class()
    ccall((:avresample_get_class,libavresample),Ptr{AVClass},())
end

function avresample_alloc_context()
    ccall((:avresample_alloc_context,libavresample),Ptr{AVAudioResampleContext},())
end

function avresample_open(avr)
    ccall((:avresample_open,libavresample),Cint,(Ptr{AVAudioResampleContext},),avr)
end

function avresample_close(avr)
    ccall((:avresample_close,libavresample),Void,(Ptr{AVAudioResampleContext},),avr)
end

function avresample_free(avr)
    ccall((:avresample_free,libavresample),Void,(Ptr{Ptr{AVAudioResampleContext}},),avr)
end

function avresample_build_matrix(in_layout::UInt64,out_layout::UInt64,center_mix_level::Cdouble,surround_mix_level::Cdouble,lfe_mix_level::Cdouble,normalize::Integer,matrix,stride::Integer,matrix_encoding::AVMatrixEncoding)
    ccall((:avresample_build_matrix,libavresample),Cint,(UInt64,UInt64,Cdouble,Cdouble,Cdouble,Cint,Ptr{Cdouble},Cint,AVMatrixEncoding),in_layout,out_layout,center_mix_level,surround_mix_level,lfe_mix_level,normalize,matrix,stride,matrix_encoding)
end

function avresample_get_matrix(avr,matrix,stride::Integer)
    ccall((:avresample_get_matrix,libavresample),Cint,(Ptr{AVAudioResampleContext},Ptr{Cdouble},Cint),avr,matrix,stride)
end

function avresample_set_matrix(avr,matrix,stride::Integer)
    ccall((:avresample_set_matrix,libavresample),Cint,(Ptr{AVAudioResampleContext},Ptr{Cdouble},Cint),avr,matrix,stride)
end

function avresample_set_channel_mapping(avr,channel_map)
    ccall((:avresample_set_channel_mapping,libavresample),Cint,(Ptr{AVAudioResampleContext},Ptr{Cint}),avr,channel_map)
end

function avresample_set_compensation(avr,sample_delta::Integer,compensation_distance::Integer)
    ccall((:avresample_set_compensation,libavresample),Cint,(Ptr{AVAudioResampleContext},Cint,Cint),avr,sample_delta,compensation_distance)
end

function avresample_convert(avr,output,out_plane_size::Integer,out_samples::Integer,input,in_plane_size::Integer,in_samples::Integer)
    ccall((:avresample_convert,libavresample),Cint,(Ptr{AVAudioResampleContext},Ptr{Ptr{UInt8}},Cint,Cint,Ptr{Ptr{UInt8}},Cint,Cint),avr,output,out_plane_size,out_samples,input,in_plane_size,in_samples)
end

function avresample_get_delay(avr)
    ccall((:avresample_get_delay,libavresample),Cint,(Ptr{AVAudioResampleContext},),avr)
end

function avresample_available(avr)
    ccall((:avresample_available,libavresample),Cint,(Ptr{AVAudioResampleContext},),avr)
end

function avresample_read(avr,output,nb_samples::Integer)
    ccall((:avresample_read,libavresample),Cint,(Ptr{AVAudioResampleContext},Ptr{Ptr{UInt8}},Cint),avr,output,nb_samples)
end

max_stars_repo_name: asafasban/AAVideoIO

max_issues_repo_name: asafasban/AAVideoIO

max_forks_repo_name: asafasban/AAVideoIO


text:
export MultirunReport
mutable struct MultirunReport
    rawData::Array{Real,3}
    timeGrid::Vector{Float64}
    dataCols::Vector{Symbol}

    MultirunReport() = new( zeros( Real, 0, 0, 0 ), Vector{Float64}(),
        Vector{Symbol}() )

    MultirunReport( rawData::Array{T,3}, timeGrid::Vector{Float64},
        dataCols::Vector{Symbol} ) where T <: Real =
        new( deepcopy(rawData), deepcopy(timeGrid), deepcopy(dataCols) )

    MultirunReport( rawData::Array{T,3}, timeGrid::Vector{Float64},
        dataCols::Vector{String} ) where T <: Real =
        MultirunReport( rawData, timeGrid, Symbol.(dataCols) )
end  # mutable struct MultirunReport


text:
module TestDoctest

import ConcurrentCollections
using Documenter: doctest
using Test

function test_doctest()
    doctest(ConcurrentCollections)
end

end  # module


text:
push!(LOAD_PATH,"../src/")

using Documenter, DynamicBounds, DynamicBoundsBase, DynamicBoundspODEsIneq, DynamicBoundspODEsDiscrete

import DynamicBoundsBase: AbstractPathConstraint, Bound, Relaxation, Gradient, Subgradient, get,
                          ODERelaxProb, getall, UnsupportedError, NotAllowedError, UnsupportedRelaxAttribute,
                          SetRelaxAttributeNotAllowed, AbstractODERelaxProblem

import DynamicBoundspODEsIneq: LocalProblemStorage, DifferentialInequalityCond, DifferentialInequalityAffect,
                               DifferentialInequalityAffectNeg, DifferentialInequalityf, DifferentialInequality

import DynamicBoundspODEsDiscrete: jetcoeffs!, TaylorFunctor!, JacTaylorFunctor!, jacobian_taylor_coeffs!,
                                   set_JxJp!, LohnersFunctor, HermiteObreschkoffFunctor, QRDenseStorage,
                                   calculateQ!, calculateQinv!, μ!, ρ!, AbstractStateContractorName,
                                   reinitialize, qr_stack, StepParams, StepResult, ExistStorage,
                                   ContractorStorage, reinitialize!, existence_uniqueness!, improvement_condition,
                                   single_step!, set_xX!, state_contractor_steps, state_contractor_γ,
                                   state_contractor_k, excess_error, set_Δ!, compute_X0!, set_P!, excess_error,
                                   contains, calc_alpha, mul_split!

makedocs(modules = [DynamicBounds, DynamicBoundsBase, DynamicBoundspODEsIneq, DynamicBoundspODEsDiscrete],
         doctest = false,
         format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true",
                                  collapselevel = 1,),
         authors = "Matthew Wilhelm",
         sitename = "DynamicBounds.jl: Nonlinear Interval Bounds and State Relaxations of Differential Equations",
         pages = ["Introduction" => "index.md",
                  "Getting Started" => "getting_started.md",
                  "DynamicsBoundsBase, An extendable abstraction layer" => "Base/Base.md",
                  "Discrete-Time Differential Relaxations" => "pODEsDiscrete/pODEsDiscrete.md",
                  "Continuous-Time Differential Relaxations" => "pODEsIneq/pODEsIneq.md",
                  "Contributing to DynamicBounds" => Any["Contributing/contributing.md",
                                                         "Contributing/add_problem.md",
                                                         "Contributing/add_integrator.md"],
                  "References" => "ref.md"]
)
@info "Deploying documentation..."
deploydocs(repo = "github.com/PSORLab/DynamicBounds.jl.git",
           push_preview  = true)


text:
# This script holds measurement error, which figures
# out what EE and MM matrices to return based on the subspec of the model
function measurement_error(m::DSGEVAR{T}) where {T<:Real}
    ss = subspec(m)
    if ss == "no subspecs with measurement error yet"
    else
        EE = zeros(T, n_observables(m), n_observables(m))
        MM = zeros(T, n_observables(m), n_shocks(m))
    end

    return EE, MM
end


text:
using Test

@testset "advection" begin
  include("wave_1d.jl")
  include("wave_2d.jl")
  include("wave_3d.jl")
end


text:
module ReverseDiffOverload
using Calculus

export reversediff, testdiff, rectlin, logΓ

type Call{f, T, As <: Tuple}
    deps::Int
    val::T
    dval::T
    args::As
    Call(val) = new(0, val, zero(val))
    Call(val, args) = new(0, val, zero(val), args)
end
Call(f::Function, args...) = begin
    val = float(f(map(init_value, args)...))
    Call{Base.function_name(f), typeof(val), typeof(args)}(val, args)
end
init_value(v::Call) = begin
    v.deps += 1
    v.val
end
init_value(x) = x
Call(val) = begin
    fval = float(val)
    Call{Nothing, typeof(fval), ()}(fval)
end

value(v::Call) = v.val
value(x) = x

diff(dval, c::Call{Nothing}) = begin
    c.dval += dval
end
diff(dval, val) = nothing

diff(c::Call) = begin
    c.deps += 1
    diff(ones(size(value(c))), c)
end
diff(d, c::Call) = begin
    @assert c.deps>0
    c.deps -= 1
    c.dval += d
    if c.deps == 0
        dxs = diff_con(c, c.dval, map(value, c.args)...)
        map(diff, dxs, c.args)
    end
    nothing
end

macro d1(f, ds...)
    eval(parse("import Base.($f)"))
    esc(d1(f, ds...))
end
macro d2(f, ds...)
    eval(parse("import Base.($f)"))
    esc(d2(f, ds...))
end
d1(f::Symbol, dx, tx=:Any) =
    quote
        $f(x::Call) = Call($f, x)
        diff_con(c::Call{Base.function_name($f)}, d, x::$tx) = ($dx,)
    end
d2(f::Symbol, dx, dy, tx=:Any, ty=:Any) =
    quote
        $f(x::Call, y::Call) = Call($f, x, y)
        $f(x::Call, y) = Call($f, x, y)
        $f(x, y::Call) = Call($f, x, y)
        diff_con(c::Call{Base.function_name($f)}, d, x::$tx, y::$ty) = ($dx, $dy)
    end

#Wrapper
reversediff(f::Function, args...) = begin
    cargs = map(Call, args)
    res = f(cargs...)
    diff(res)
    (res.val, map(x -> x.dval, cargs))
end

#Functions to pull and vectorize definitions from Calculus.jl
vectorize_dict = {:* => :.*, :/ => :./, :\ => :.\, :^ => :.^, :xp => :d}
vectorize_expr(exp::Expr) = Expr(exp.head, map(vectorize_expr, exp.args)...)
vectorize_expr(sym::Symbol) =
    if haskey(vectorize_dict, sym)
        vectorize_dict[sym]
    else
        sym
    end
vectorize_expr(any) = any
for (funsym, exp) in Calculus.derivative_rules
    @eval @d1($(funsym), $(vectorize_expr(exp)))
end

#Differentiation rules.
#Operators
@d2(+, d, d)
@d2(+, sum(d), d, Number, AbstractArray)
@d2(+, d, sum(d), AbstractArray, Number)

@d2(.+, d, d)
@d2(.+, sum(d), d, Number, AbstractArray)
@d2(.+, d, sum(d), AbstractArray, Number)

@d2(-, d, -d)
@d2(-, sum(d), -d, Number, AbstractArray)
@d2(-, d, -sum(d), AbstractArray, Number)
@d1(-, -d)

import Base.sum
sum(x::Call, y::Int) = Call(sum, x, y)
diff_con(c::Call{:sum}, d, x::AbstractArray, y::Int) = (broadcast(*, ones(size(x)...), d), 0.)

@d2(.-, d, -d)
@d2(.-, sum(d), -d, Number, AbstractArray)
@d2(.-, d, -sum(d), AbstractArray, Number)

@d2(*, d*y', x'*d)
@d2(*, y*d, dot(x, d), AbstractArray, Number)
@d2(*, dot(y, d), x*d, Number, AbstractArray)

@d2(.*, d.*y, d.*x)

@d2(/, d/y', -(y'\x')*(d/y'))
@d2(\, -(x'\d)*(y'/x'), x'\d)

@d2(./, d./y, -d.*x./y./y)

@d2(.^, d.*y.*(x.^(y-1)), d.*log(x).*(x.^y))

@d2(dot, d*y, d*x)
@d1(det, d*det(x)*inv(x)')
@d1(trace, d*eye(size(x)...))
@d1(inv, -(x'\d)/x')
#@d1(exp, d.*exp(x))
#@d1(sin, d.*cos(x))
#@d1(cos, -d.*sin(x))
@d1(ctranspose, ctranspose(d))
@d1(first, (tmp = zeros(size(x)); tmp[1] = d; tmp))
@d1(vec, reshape(d, size(x)...))
@d1(sum, d*ones(size(x)))

import Base.getindex
getindex(x::Call, y::Int) = Call(getindex, x, y)
diff_con(c::Call{:getindex}, d, x::Vector, y::Int) = ((res=zeros(size(x)...); res[y] = d; res), 0.)

#Special functions
rectlin(x) = max(0, x)
@d1(rectlin, d.*(x.>0))
logΓ(x) = log(gamma(x))
@d1(logΓ, d.*digamma(x))

#Testing code
import Base.isapprox
isapprox(x::AbstractArray, y::AbstractArray; rtol=1E-6, atol=1E-10) =
    all(map((x, y) -> isapprox(x, y; rtol=rtol, atol=atol), x, y))
import Base.ones
ones(arg::()) = 1.

testdiff(f::Function, args...) = begin
    #Reverse diff
    calls = map(Call, args)
    res   = f(calls...)
    diff(res)
    dargs_rdo = map(x -> x.dval, calls)
    #Finite gradients
    (vf, devec) = vectorize(f, args...)
    dargs_fg = devec(gradient(vf, vectorize(args)[1]))
    @assert all(map((x, y) -> isapprox(x, y; rtol=1E-6, atol=1E-10), dargs_rdo, dargs_fg))
end

vectorize(a::Real) =
    (Float64[a], a::Vector{Float64} -> a[1])
vectorize(a::Vector) =
    (a, identity)
vectorize(a::Matrix) =
    (reshape(a, size(a, 1)*size(a, 2)), v -> reshape(v, size(a, 1), size(a, 2)))
vectorize(args::Tuple) = begin
    vdvs = map(vectorize, args)
    vector = vcat(map(x -> x[1], vdvs)...)
    #Construct devectorize function
    sizes = map(x -> length(x[1]), vdvs)
    ranges = Range1{Int}[]
    push!(ranges, 1:sizes[1])
    for i = 2:length(sizes)
        push!(ranges, (last(ranges[end])+1):(last(ranges[end])+sizes[i]))
    end
    f = vs::Vector{Float64} -> map((vdv, r) -> vdv[2](vs[r]), vdvs, ranges)
    (vector, f)
end
vectorize(f::Function, args...) = begin
    (vargs, devec) = vectorize(args)
    (v::Vector{Float64} -> f(devec(v)...), devec)
end
end


text:
function test()
    para = Parameter(0.6, 15.0)
    x = collect(linspace(0,1,1000))
    y = sin(10π*x) + x
    prob = Problem(x,y)
    ppara = get_pointer(para)
    pprob = get_pointer(prob)
    pscore = compute_score(pprob, ppara)
    @show pscore
    mic(pscore)
    free_score(pscore)
end

function get_pointer(obj::Parameter)
    convert(Ptr{Parameter}, pointer_from_objref(obj))
end

function get_pointer(obj::Problem)
    convert(Ptr{Problem}, pointer_from_objref(obj))
end

function get_pointer(obj::Score)
    convert(Ptr{Score}, pointer_from_objref(obj))
end