text:
module GasChromatographyTools

#using Reexport
using GasChromatographySimulator
using Interpolations
using Intervals
using Dierckx
using HypertextLiteral
using PlutoUI
using Plots

#----RT-locking----------------------------------------------------
include("./RTlocking.jl")
#---end-RT-locking-------------------------------------------------

#----begin-notebooks-functions----------------------------------------------------------------------
## UI-functions
include("./UI.jl")

##---begin-Plot-functions---------------------------------------------------------------------------
"""
	local_plots(xx, yy, sol, par)

Show additional 'local' plots of selected `yy` quantities over selected `xx`
quantities.

# Arguments
* `xx`: Selected quantity shown on the x-axis. Possible values: "z", "t", "T",
  "τ", "σ" and "u".
* `yy`: Selected quantity shown on the y-axis. Possible values: "z", "t", "T",
  "τ", "σ" and "u".
* `sol`: The solution of the simulation.
* `par`: The parameters of the simulated GC-system.
"""
function local_plots(xx, yy, sol, par)
	n = size(sol)[1]

	df_sol = GasChromatographySimulator.sol_extraction(sol, par)
	xvalues = Array{Array{Float64,1}}(undef, n)
	yvalues = Array{Array{Float64,1}}(undef, n)

	p_add = plot(legend=false)
	for i=1:n
		if xx=="z"
			xvalues[i] = df_sol.z[i]
			xlabel = "position z in m"
		elseif xx=="t"
			xvalues[i] = df_sol.t[i]
			xlabel = "time t in s"
		elseif xx=="T"
			xvalues[i] = par.prog.T_itp.(df_sol.z[i], df_sol.t[i]).-273.15
			xlabel = "temperature T in °C"
		elseif xx=="τ"
			xvalues[i] = sqrt.(df_sol.τ²[i])
			xlabel = "peak width τ in s"
		elseif xx=="σ"
			xvalues[i] = velocity(df_sol, i, par).*sqrt.(df_sol.τ²[i])
			xlabel = "band width in m"
		elseif xx=="u"
			xvalues[i] = velocity(df_sol, i, par)
			xlabel = "solute velocity in m/s"
		end
		if yy=="z"
			yvalues[i] = df_sol.z[i]
			ylabel = "position z in m"
		elseif yy=="t"
			yvalues[i] = df_sol.t[i]
			ylabel = "time t in s"
		elseif yy=="T"
			yvalues[i] = par.prog.T_itp.(df_sol.z[i], df_sol.t[i]).-273.15
			ylabel = "temperature T in °C"
		elseif yy=="τ"
			yvalues[i] = sqrt.(df_sol.τ²[i])
			ylabel = "peak width in s"
		elseif yy=="σ"
			yvalues[i] = velocity(df_sol, i, par).*sqrt.(df_sol.τ²[i])
			ylabel = "band width in m"
		elseif yy=="u"
			yvalues[i] = velocity(df_sol, i, par)
			ylabel = "solute velocity in m/s"
		end
		plot!(p_add, xvalues[i], yvalues[i], xlabel=xlabel, ylabel=ylabel, label=par.sub[i].name, m=:o)
	end
	return p_add
end

"""
	velocity(df_sol, i, par)

Calculate the velocity (in m/s) coressponding to solution of the `i-th` sunstance of a
GC-system defined by `par`.
"""
function velocity(df_sol, i, par)
	x = df_sol.z[i]
	t = df_sol.t[i]
	T_itp = par.prog.T_itp
	pin_itp = par.prog.pin_itp
	pout_itp = par.prog.pout_itp
	L = par.sys.L
	d = par.sys.d
	df = par.sys.df
	gas = par.sys.gas
	ΔCp = par.sub[i].ΔCp
	Tchar = par.sub[i].Tchar
	θchar = par.sub[i].θchar
	φ₀ = par.sub[i].φ₀
	u = Array{Float64}(undef, length(x))
	for j=1:length(x)
		u[j] = 1/GasChromatographySimulator.residency(x[j], t[j], T_itp, pin_itp, pout_itp, L, d, df, gas, ΔCp, Tchar, θchar, φ₀)
	end
	return u
end

"""
	common(s_1, s_1)

Compare two arrays and return the common elements.
"""
function common(s_1, s_2)
	common_s = String[]
	if length(s_1) >= length(s_2)
		for i=1:length(s_1)
			if s_1[i] in s_2
				push!(common_s, s_1[i])
			end
		end
	else
		for i=1:length(s_2)
			if s_2[i] in s_1
				push!(common_s, s_2[i])
			end
		end
	end
	return common_s
end

"""
	compare_peaklist(pl_1, pl_2)

Compare two peaklists (results of GasChromatographySimulator.jl) and calculate
absolute and relative differences of retention times and peak widths.
"""
function compare_peaklist(pl_1, pl_2)
	name = pl_1.Name
	tR1 = pl_1.tR
	τR1 = pl_1.τR
	tR2 = Array{Float64}(undef, size(pl_1)[1])
	τR2 = Array{Float64}(undef, size(pl_1)[1])
	for i=1:size(pl_1)[1]
		i2 = findfirst(name[i].==pl_2.Name)
		tR2[i] = pl_2.tR[i2]
		τR2[i] = pl_2.τR[i2]
	end
	ΔtR = tR1 .- tR2
	ΔτR = τR1 .- τR2
	rel_tR = ΔtR.*100.0./tR1
	rel_τR = ΔτR.*100.0./τR1
	compare_pl = DataFrame(Name=name, tR1=tR1, tR2=tR2, ΔtR=ΔtR, rel_tR=rel_tR, τR1=τR1, τR2=τR2, ΔτR=ΔτR, rel_τR=rel_τR)
	return compare_pl
end

"""
	compare_measurement_simulation(meas, peaklist)

Compare the retention times of measured and simulated substances.

# Arguments
* `meas`: DataFrame consisting at least of the columns `:Name` and `:RT`
  (measured retention time in s)
* `peaklist`: DataFrame as result from GasChromatographySimulator.jl with the
  columns `:Name` and `:tR` (simulated retention time in s).

The comparison is done by searching the same `Name` of the substance in both
DataFrames and calculating the absolute difference of the retention times (in s)
and the relative difference (in %).
"""
function compare_measurement_simulation(meas, peaklist)
	name = meas.Name
	tRm = meas.RT
	tRs = Array{Float64}(undef, size(meas)[1])
	for i=1:size(meas)[1]
		i2 = findfirst(name[i].==peaklist.Name)
		if typeof(i2) == Nothing
			tRs[i] = NaN
		else
			tRs[i] = peaklist.tR[i2]
		end
	end
	compare_df = DataFrame(Name=name, measured_tR=tRm, simulated_tR=tRs, ΔtR=tRm.-tRs, rel_tR=(tRm.-tRs)./tRm.*100.0)
	return compare_df
end

#----notebooks-functions----------------------------------------------------------------------------

#---misc-functions-----------------------------------------------------------------------------------
"""
	change_initial(par, init_t, init_τ)

Change the inital time and peak widths for the substances in a defined GC-system
`par` to the values `init_t` and `, init_τ`.
"""
function change_initial(par::GasChromatographySimulator.Parameters, init_t, init_τ)
	# copys the parameters `par` and changes the values of par.sub[i].τ₀ and par.sub[i].t₀ to init_τ[i] resp. init_t[i]
	newsub = Array{GasChromatographySimulator.Substance}(undef, length(par.sub))
	for i=1:length(par.sub)
		newsub[i] = GasChromatographySimulator.Substance(par.sub[i].name, par.sub[i].CAS, par.sub[i].Tchar, par.sub[i].θchar, par.sub[i].ΔCp, par.sub[i].φ₀, par.sub[i].ann, par.sub[i].Dag, init_t[i], init_τ[i])
	end
	newpar = GasChromatographySimulator.Parameters(par.sys, par.prog, newsub, par.opt)
	return newpar
end

"""
	general_step(x, L, a)

A generalized step function

# Arguments
* `x::Float64`: variable (e.g. x-position)
* `L::Array{Float64,1}`: Array with different length of the constant segments
* `a::Array{<:Any,1}`: Array with the values for the segments, can also be
  functions.
"""
function general_step(x::Float64, L::Array{Float64,1}, a::Array{<:Any,1})
    if length(L)!=length(a)
        error("Parameters `L` and `a` must have the same length.")
        return
    end
	intervals = Array{Intervals.Interval}(undef, length(L))
	cumL = round.(cumsum([0; L]), digits=4)
	for i=1:length(intervals)
		if i==length(intervals)
			intervals[i] = Interval{Closed, Closed}(cumL[i],cumL[i+1])
		else
			intervals[i]=Interval{Closed, Open}(cumL[i],cumL[i+1])
		end
		if x in intervals[i]
			return a[i]
		end
	end
end

#----end-misc-functions-----------------------------------------------------------------------------
end # module


text:
function _make_event_gui()

    popup_event = Menu()
    if VERSION > v"0.7-"
        event_handles=Array{MenuItemLeaf}(undef,0)
    else
        event_handles=Array{MenuItemLeaf}(0)
    end
    for i=1:8
        push!(event_handles,MenuItem(string("Analog ",i)))
        push!(popup_event,event_handles[i])
    end

    for i=1:16
        push!(event_handles,MenuItem(string("TTL ",i)))
        push!(popup_event,event_handles[8+i])
    end

    popup_event_none=MenuItem("None")
    push!(popup_event,popup_event_none)
    Gtk.showall(popup_event)

    (event_handles,popup_event,popup_event_none)
end

function add_event_callbacks(event_handles,popup_event_none,handles)

    for i=1:8
        signal_connect(popup_event_cb,event_handles[i],"activate",Void,(),false,(handles,i-1))
    end

    for i=9:24
        signal_connect(popup_event_cb,event_handles[i],"activate",Void,(),false,(handles,i-1))
    end

    signal_connect(popup_event_cb,popup_event_none,"activate",Void,(),false,(handles,-1))

    nothing
end

function popup_event_cb(widgetptr::Ptr,user_data::Tuple{Gui_Handles,Int64})

    han, event_id = user_data

    ctx=Gtk.getgc(han.c)
    myheight=height(ctx)

    chan_id=1
    if han.mim[2]<(myheight-250)
        chan_id=1
    elseif han.mim[2]<(myheight-200)
        chan_id=2
    elseif han.mim[2]<(myheight-150)
        chan_id=3
    elseif han.mim[2]<(myheight-100)
        chan_id=4
    elseif han.mim[2]<(myheight-50)
        chan_id=5
    else
        chan_id=6
    end

    han.events[chan_id]=event_id

    nothing
end

#Event plotting

function plot_events(fpga::Array{FPGA,1},han::Gui_Handles,myreads::Int64)

    @inbounds for i=1:6
	if han.events[i]>-1
	    if han.events[i]<8 #analog
		val=parse_analog(fpga[1].adc,han.events[i]+1)
		plot_analog(han,i,myreads,val)
	    else
		val=parse_ttl(fpga[1].ttlin,han.events[i]-7)
		plot_ttl(han,i,myreads,val)
	    end
	end
    end

    nothing
end

function plot_events(fpga::DArray{FPGA,1,Array{FPGA,1}},han::Gui_Handles,myreads::Int64)

    nothing
end

function parse_analog(adc::Array,chan::Int64)

    mysum=0
    for i=1:size(adc,1)
	mysum+=adc[i,chan]
    end

    round(Int64,mysum/size(adc,1)/0xffff*30)
end

function plot_analog(han::Gui_Handles,channel::Int64,myreads::Int64,val::Int64)

    ctx=Gtk.getgc(han.c)
    myheight=height(ctx)

    move_to(ctx,myreads-1,myheight-260 + (channel-1)*50-val)
    line_to(ctx,myreads,myheight-260 + (channel-1)*50-val)
    set_source_rgb(ctx,1.0,1.0,0.0)
    stroke(ctx)

    nothing
end

function parse_ttl(ttlin::Array,chan::Int64)

    y=0

    for i=1:length(ttlin)
        y=y|(ttlin[i]&(2^(chan-1)))
    end

    y>0
end

function plot_ttl(han::Gui_Handles,channel::Int64,myreads::Int64,val::Bool)

    ctx=Gtk.getgc(han.c)
    myheight=height(ctx)

    offset=0
    if val==true
	offset=30
    end

    move_to(ctx,myreads-1,myheight-260+(channel-1)*50-offset)
    line_to(ctx,myreads,myheight-260+(channel-1)*50-offset)
    set_source_rgb(ctx,1.0,1.0,0.0)
    stroke(ctx)

    nothing
end

ext: jl


text:
type EpsilonGreedy <: BanditAlgorithm
  epsilon::Float64
  counts::Vector{Int64}
  values::Vector{Float64}
end

function EpsilonGreedy(epsilon::Float64, n_arms::Int64)
  EpsilonGreedy(epsilon, zeros(Int64, n_arms), zeros(n_arms))
end

function initialize(algo::EpsilonGreedy, n_arms::Int64)
  algo.counts = zeros(Int64, n_arms)
  algo.values = zeros(n_arms)
end

function initialize(algo::EpsilonGreedy, arms::Vector)
  initialize(algo, length(arms))
end

function select_arm(algo::EpsilonGreedy)
  if rand() > algo.epsilon
    return ind_max(algo.values)
  else
    return randi(length(algo.values))
  end
end

function update(algo::EpsilonGreedy, chosen_arm::Int64, reward::Real)
  algo.counts[chosen_arm] += 1
  n = algo.counts[chosen_arm]

  value = algo.values[chosen_arm]
  algo.values[chosen_arm] = ((n - 1) / n) * value + (1 / n) * reward
end

max_stars_repo_name: rmminusrslash/BanditsBook

max_issues_repo_name: rmminusrslash/BanditsBook

max_forks_repo_name: rmminusrslash/BanditsBook


text:
# Finite state machine library for Julia
# @license MIT
# @author jack@tinybike.net (Jack Peterson)

module FiniteStateMachine

using Compat

export StateMachine, state_machine, fire, callback, before_event, before_this_event, before_any_event, after_event, after_this_event, after_any_event, change_state, enter_state, enter_this_state, enter_any_state, leave_state, leave_this_state, leave_any_state

@compat type StateMachine
    map::Dict
    actions::Dict
    current::AbstractString
    terminal::Union{AbstractString, Void}
    StateMachine() = new(Dict(), Dict(), "none", nothing)
end

include("state_machine.jl")

include("fire.jl")

include("transitions.jl")

end # module


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

mutable struct VirtualMachineExtensionInstanceView <: SwaggerModel
    name::Any # spec type: Union{ Nothing, String } # spec name: name
    type::Any # spec type: Union{ Nothing, String } # spec name: type
    typeHandlerVersion::Any # spec type: Union{ Nothing, String } # spec name: typeHandlerVersion
    substatuses::Any # spec type: Union{ Nothing, Vector{InstanceViewStatus} } # spec name: substatuses
    statuses::Any # spec type: Union{ Nothing, Vector{InstanceViewStatus} } # spec name: statuses

    function VirtualMachineExtensionInstanceView(;name=nothing, type=nothing, typeHandlerVersion=nothing, substatuses=nothing, statuses=nothing)
        o = new()
        validate_property(VirtualMachineExtensionInstanceView, Symbol("name"), name)
        setfield!(o, Symbol("name"), name)
        validate_property(VirtualMachineExtensionInstanceView, Symbol("type"), type)
        setfield!(o, Symbol("type"), type)
        validate_property(VirtualMachineExtensionInstanceView, Symbol("typeHandlerVersion"), typeHandlerVersion)
        setfield!(o, Symbol("typeHandlerVersion"), typeHandlerVersion)
        validate_property(VirtualMachineExtensionInstanceView, Symbol("substatuses"), substatuses)
        setfield!(o, Symbol("substatuses"), substatuses)
        validate_property(VirtualMachineExtensionInstanceView, Symbol("statuses"), statuses)
        setfield!(o, Symbol("statuses"), statuses)
        o
    end
end # type VirtualMachineExtensionInstanceView

const _property_map_VirtualMachineExtensionInstanceView = Dict{Symbol,Symbol}(Symbol("name")=>Symbol("name"), Symbol("type")=>Symbol("type"), Symbol("typeHandlerVersion")=>Symbol("typeHandlerVersion"), Symbol("substatuses")=>Symbol("substatuses"), Symbol("statuses")=>Symbol("statuses"))
const _property_types_VirtualMachineExtensionInstanceView = Dict{Symbol,String}(Symbol("name")=>"String", Symbol("type")=>"String", Symbol("typeHandlerVersion")=>"String", Symbol("substatuses")=>"Vector{InstanceViewStatus}", Symbol("statuses")=>"Vector{InstanceViewStatus}")
Base.propertynames(::Type{ VirtualMachineExtensionInstanceView }) = collect(keys(_property_map_VirtualMachineExtensionInstanceView))
Swagger.property_type(::Type{ VirtualMachineExtensionInstanceView }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_VirtualMachineExtensionInstanceView[name]))}
Swagger.field_name(::Type{ VirtualMachineExtensionInstanceView }, property_name::Symbol) =  _property_map_VirtualMachineExtensionInstanceView[property_name]

function check_required(o::VirtualMachineExtensionInstanceView)
    true
end

function validate_property(::Type{ VirtualMachineExtensionInstanceView }, name::Symbol, val)
end


text:
# This file is a part of PropDicts.jl, licensed under the MIT License (MIT).

function contains_vars(s::String)
    escaped = false
    for c in s
        if c == '\\'
            escaped = !escaped
        else
            if c == '$' && !escaped
                return true
            end
            escaped = false
        end
    end

    return false
end

_isalnum(c::Char) = isletter(c) || isnumeric(c)

function substitute_vars(input::AbstractString, var_values::Dict{String,String} = Dict{String,String}(); use_env::Bool = false, ignore_missing::Bool = false)
    if !contains_vars(input)
        return input
    end

    out = IOBuffer()
    idxs = eachindex(input)
    from = first(idxs)
    to = last(idxs)

    npos = from - 1
    no_brace = Char(0)

    open_brache_chars = ('{', '(')
    close_brache_chars = ('}', ')')

    escaped = false
    var_from = npos;
    var_until = npos;
    open_brace = no_brace
    close_brace = no_brace
    pos = from;

    while pos <= to
        c = input[pos];
        if (var_from == npos)
            if c == '\\'
                escaped = !escaped
                print(out, c)
            else
                if ((c == '$') && !escaped && (pos < to))
                    var_from = pos + 1;
                else
                    print(out, c)
                end
                escaped = false
            end
            pos += 1
        else
            if c in open_brache_chars
                if (pos == var_from)
                    var_from = pos + 1
                    open_brace = c
                    close_brace = close_brache_chars[something(findfirst(isequal(open_brace), open_brache_chars), 0)]
                else
                    throw(ArgumentError("Encountered extra \"$c\" during variable substitution in string \"$input\""))
                end
            else
                if !all(_isalnum, c) && (c != '_')
                    if open_brace != no_brace
                        if c in close_brache_chars
                            if c == close_brace
                                var_until = pos
                                pos += 1
                            else
                                throw(ArgumentError("Encountered closing \"$c\" for open \"$open_brace\" during variable substitution in string \"$input\""))
                            end
                        elseif (c == '\\')
                            throw(ArgumentError("Encountered illegal character \"\\\" in variable name during variable substitution in string \"$input\""))
                        end
                    else
                        var_until = pos;
                    end
                elseif isdigit(c) && (pos == var_from)
                    throw(ArgumentError("Illegal variable name, starting with a digit, during variable substitution in string \"$input\""))
                end
            end

            if ( (var_until == npos) && (pos + 1 > to) )
                if open_brace != no_brace
                    throw(ArgumentError("Missing \"$close_brace\" for \"\$$open_brace\" during variable substitution in string \"$input\""))
                else
                    pos += 1
                    var_until = pos
                end
            end

            if (var_until != npos)
                if (var_until > var_from)
                    var_name = input[var_from : var_until - 1];

                    var_expr_from, var_expr_to = (open_brace != no_brace) ? (var_from - 2, var_until) : (var_from - 1, var_until - 1)

                    subst_value = if haskey(var_values, var_name)
                        var_values[var_name]
                    elseif use_env && haskey(ENV, var_name)
                        ENV[var_name]
                    elseif ignore_missing
                        input[var_expr_from:var_expr_to]
                    else
                        throw(ArgumentError("Unknown variable \"$var_name\" during variable substitution in string \"$input\""))
                    end

                    if ((var_expr_from == from) && (var_expr_to == to))
                        return subst_value;
                    else
                        print(out, subst_value);
                    end
                else
                    if open_brace != no_brace
                        throw(ArgumentError("Encountered illegal \"\$$open_brace$close_brace\" during variable substitution in string \"$input\""))
                    else
                        print(out, input[pos-1], input[pos])
                        pos += 1
                    end
                end
                var_from = npos
                var_until = npos
                open_brace = no_brace
            else
                pos += 1
            end
        end
    end

    return String(take!(out))
end

function substitute_vars!(
    d::AbstractDict, var_values::Dict{String,String} = Dict{String,String}();
    use_env::Bool = false, ignore_missing::Bool = false, recursive::Bool = true
)
    for (k, v) in d
        if isa(v, AbstractDict)
            if recursive
                substitute_vars!(v, var_values, use_env = use_env, ignore_missing = ignore_missing, recursive = recursive)
            end
        elseif isa(v, AbstractString) && contains_vars(v)
            d[k] = substitute_vars(v, var_values, use_env = use_env, ignore_missing = ignore_missing)
        end
    end
    d
end


text:
function gen_confus(;
  filename = "confus.txt",
  n_rows = 50,
  col_max = 5000,
  dense = false,
)
  keys = sort(rand(100_000:999_999, n_rows))
  while !allunique(keys)
    keys = unique(keys)
    keys = sort([keys; rand(1:100_000, n_rows - length(keys))])
  end
  n_elem = 0
  open(filename, "w") do io
    for i = 1:n_rows
      n_cols = if dense
        rand(1:col_max)
      else
        ceil(Int, 10 ^ rand(range(0, log10(col_max), length=col_max)))
      end
      n_elem += n_cols
      cols = sort(rand(100_000:999_999, n_cols))
      line = string(keys[i]) * "#" * join(cols, ",")
      println(io, line)
    end
  end

  return n_elem
end


text:
using MAT

include("../../configs.jl")

"""
Precision = TP / (TP + FP)
Recall = TP / (TP + FN)

"""

function AP(gt, bboxes, confidences; iou_thold=0.5, thresh_num=1000, iou_vals=nothing, return_array=false)
    results = zeros(2, thresh_num+1) # precisions 1st idx, recalls in 2nd idx
    if iou_vals === nothing iou_vals = iou(gt[1:4,:], bboxes) end

    selected_boxes = zeros(size(bboxes, 2)); selected_gts = zeros(size(gt, 2)); assigned_gts = 0;

    pos_ious = findall(iou_vals .>= iou_thold)
    pos_priors = getindex.(pos_ious, [1 2])[:,1]; pos_gts = getindex.(pos_ious, [1 2])[:,2];
    sorted_priors = sortperm(confidences[2,pos_priors], rev=true)

    neg_ious = findall(iou_vals .< iou_thold); assigned_neg_gts = 0;
    neg_priors = getindex.(neg_ious, [1 2])[:,1]; sorted_neg_priors = sortperm(confidences[2,neg_priors], rev=true);
    neg_gts = getindex.(neg_ious, [1 2])[:,2]; num_neg_gts = length(unique(neg_gts));

    for t in 1:thresh_num+1
        thold = (t-1) / thresh_num
        TP = 0; FP = 0; FN = 0;
        for s in sorted_priors
            prior = pos_priors[s]; gt_idx = pos_gts[s];
            if confidences[2, prior] >= thold && selected_boxes[prior] == 0 && selected_gts[gt_idx] == 0
                # found TP example
                TP += 1; assigned_gts += 1;
                selected_boxes[prior] = 1; selected_gts[gt_idx] = 1;
            elseif confidences[2, prior] < thold
                # unassigned positive gt boxes are counted as FN
                FN = length(unique(pos_gts)) - assigned_gts
                break
            end
        end

        for ns in sorted_neg_priors
            prior = neg_priors[ns]; gt_idx = neg_gts[ns];
            if confidences[2, prior] >= thold && selected_boxes[prior] == 0 && selected_gts[gt_idx] == 0
                # found TP example
                FP += 1; assigned_neg_gts += 1;
                selected_boxes[prior] = -1; selected_gts[gt_idx] = -1;
            elseif confidences[2, prior] < thold || num_neg_gts == assigned_neg_gts
                break
            end
        end

        precision = TP == 0 ? 0 : TP / (TP + FP)
        recall = TP == 0 ? 0 : TP / (TP + FN)
        results[1, t] = precision; results[2, t] = recall;
    end

    main_results = zeros(thresh_num+1)
    for t in 1:thresh_num+1
        thold = (t-1) / thresh_num
        recall_indices = findall(results[2,:] .>= thold)
        if length(recall_indices) == 0
            main_results[t] = 0
        else
            main_results[t] = maximum(results[1,recall_indices])
        end
    end

    if return_array return main_results end

    AP = sum(main_results) / (thresh_num + 1)
    return AP
end

function mAP(gt, bboxes, confidences, iou_tholds; thresh_num=1000)
    APs = 0; cnt = 0; iou_vals = iou(gt, bboxes);
    for iou_thold in iou_tholds[1]:iou_tholds[2]:iou_tholds[3]
        APs += AP(gt, bboxes, confidences, iou_vals=iou_vals, thresh_num=1000, iou_thold=iou_thold)
        cnt += 1
    end
    mAP = APs / cnt
    return mAP
end

"""
!!! Each individual box in boxes has the format (x_min, y_min, x_max, y_max) !!!
Returns intersection of unions for combination of each boxes in both parameters.
A   : number of boxes in boxes1
B   : number of boxes in boxes2

Return: iou values with shape (A, B)
"""
function iou(boxes1, boxes2)
    C, A = size(boxes1); D, B = size(boxes2);
    area1 = max.(boxes1[4,:] .- boxes1[2,:], 0) .* max.(boxes1[3,:] .- boxes1[1,:], 0)
    area2 = max.(boxes2[4,:] .- boxes2[2,:], 0) .* max.(boxes2[3,:] .- boxes2[1,:], 0)
    intersections = _get_intersections(boxes1, boxes2)
    unions = reshape(area1, (1, A)) .+ reshape(area2, (B, 1)) .- intersections
    return intersections ./ unions
end

"""
!!! Each individual box in boxes has the format (x_min, y_min, x_max, y_max) !!!
We resize both boxes to [A,B,4]:
* [A,2] -> [A,1,2] -> [A,B,2]
* [B,2] -> [1,B,2] -> [A,B,2]
Then we compute the area of intersect between box_a and box_b.

Return: intersection area with shape (A, B)
"""
function _get_intersections(boxes1, boxes2)
    C, A = size(boxes1); D, B = size(boxes2);
    min_coords = max.(reshape(boxes1[1:2,:], (2, 1, A)), reshape(boxes2[1:2,:], (2, B, 1)))
    max_coords = min.(reshape(boxes1[3:4,:], (2, 1, A)), reshape(boxes2[3:4,:], (2, B, 1)))
    intersections = max.(max_coords .- min_coords, 0)
    return intersections[1,:,:] .* intersections[2,:,:]
end

max_stars_repo_name: barisbatuhan/retinaface

max_issues_repo_name: barisbatuhan/retinaface

max_forks_repo_name: barisbatuhan/retinaface


text:
using Test
using SCIP

@testset "direct library calls" begin
    include("direct_library_calls.jl")
end

@testset "managed memory" begin
    include("managed_scip.jl")
end

# new type definitions in module (needs top level)
include("conshdlr_support.jl")

@testset "constraint handlers" begin
    include("conshdlr.jl")
end

@testset "MathOptInterface tests (bridged)" begin
    include("MOI_wrapper_bridged.jl")
end

@testset "MathOptInterface tests (bridged & cached)" begin
    include("MOI_wrapper_cached.jl")
end

@testset "MathOptInterface tests (direct)" begin
    include("MOI_wrapper_direct.jl")
end

@testset "MathOptInterface additional tests" begin
    include("MOI_additional.jl")
end

@testset "MathOptInterface nonlinear expressions" begin
    include("MOI_nonlinear_exprs.jl")
end

@testset "constraint handlers (with MOI)" begin
    include("MOI_conshdlr.jl")
end


text:
# Holger Teichgraeber, Elias Kuepper, 2019

 ######################
 # ClustForOpt
 # Analyzing clustering techniques as input for energy systems optimization
 #
 #####################
module ClustForOpt
  using Reexport
  using StatsKit
  @reexport using FileIO
  using JuMP #QUESTION Part of ClustForOpt?

   #TODO how to make PyPlot, PyCall, and TimeWarp optional? -> only import when needed

   export run_opt,
          run_clust,
          run_darkperiod_clust,
          run_pure_clust,
          get_sup_kw_args,
          InputData,
          FullInputData,
          ClustData,
          ClustDataMerged,
          ClustResult,
          ClustResultAll,
          ClustResultBest,
          SimpleExtremeValueDescr,
          get_EUR_to_USD,
          load_input_data,
          plot_clusters,
          subplot_clusters,
          z_normalize,
          undo_z_normalize,
          sakoe_chiba_band,
          kmedoids_exact,
          plot_k_rev,
          plot_k_rev_subplot,
          plot_SSE_rev,
          sort_centers,
          calc_SSE,
          find_medoids,
          resize_medoids,
          load_timeseries_data,
          combine_timeseries_weather_data

  include(joinpath("utils","datastructs.jl"))
  include(joinpath("utils","utils.jl"))
  include(joinpath("utils","load_data.jl"))
  include(joinpath("optim_problems","run_opt.jl"))
  include(joinpath("clustering","run_clust.jl"))
  include(joinpath("clustering","exact_kmedoids.jl"))
  include(joinpath("clustering","extreme_vals.jl"))
  include(joinpath("clustering","attribute_weighting.jl"))
  include(joinpath("clustering","intraperiod_segmentation.jl"))
  include(joinpath("clustering","other_clust.jl"))
end # module ClustForOpt


text:
function bmm_test(a,b; transA = false, transB = false)
    bs = size(a,3)
    transA && (a = permutedims(a, [2,1,3]))
    transB && (b = permutedims(b, [2,1,3]))
    c = []
    for i = 1:bs
        push!(c, a[:,:,i]*b[:,:,i])
    end

    cat(c...; dims = 3)
end

function bmm_adjtest(a,b; adjA = false, adjB = false)
    bs = size(a,3)
    c = []
    for i = 1:bs
        ai = adjA ? adjoint(a[:,:,i]) : a[:,:,i]
        bi = adjB ? adjoint(b[:,:,i]) : b[:,:,i]
        push!(c, ai*bi)
    end

    cat(c...; dims = 3)
end

@testset "batched_mul: Float64 * $TB" for TB in [Float64, Float32]

    A = randn(7,5,3)
    B = randn(TB, 5,7,3)
    C = randn(7,6,3)

    @test batched_mul(A, B) ≈ bmm_test(A, B)
    @test batched_mul(batched_transpose(A), batched_transpose(B)) ≈ bmm_test(A, B; transA = true, transB = true)
    @test batched_mul(batched_transpose(A), C) ≈ bmm_test(A, C; transA = true)
    @test batched_mul(A, batched_transpose(A)) ≈ bmm_test(A, A; transB = true)

    cA = randn(Complex{Float64}, 7,5,3)
    cB = randn(Complex{TB}, 5,7,3)
    cC = randn(Complex{Float64}, 7,6,3)

    @test batched_mul(cA, cB) ≈ bmm_adjtest(cA, cB)
    @test batched_mul(batched_adjoint(cA), batched_adjoint(cB)) ≈ bmm_adjtest(cA, cB; adjA = true, adjB = true)
    @test batched_mul(batched_adjoint(cA), cC) ≈ bmm_adjtest(cA, cC; adjA = true)
    @test batched_mul(cA, batched_adjoint(cA)) ≈ bmm_adjtest(cA, cA; adjB = true)

    @test batched_transpose(batched_transpose(A)) === A
    @test batched_adjoint(batched_adjoint(cA)) === cA

    TBi = TB==Float64 ? Int64 : Int32
    iA = rand(1:99, 7,5,3)
    iB = TB.(rand(1:99, 5,7,3))
    iC = zeros(Int, 7,6,3)
    @test batched_mul(iA, iB) == bmm_adjtest(iA, iB)
    @test batched_mul(cA, iB) ≈ bmm_adjtest(cA, iB)

    @test_throws DimensionMismatch batched_mul(rand(2,2,2), rand(TB, 2,2,10))
    @test_throws DimensionMismatch batched_mul(rand(2,2,2), rand(TB, 10,2,2))
    @test_throws Exception batched_mul!(zeros(2,2,10), rand(2,2,2), rand(TB, 2,2,2))

end


text:
module Test_Composition

using Modia3D
using Modia3D.Test
using Modia3D.LinearAlgebra
using Modia3D.ModiaMath
using Modia3D.Unitful

r2 = [2.0,3.0,4.0]
@assembly TestObject3D begin
   world = Modia3D.Object3D()

   # Object3D fixed in world
   frame1 = Modia3D.Object3D(world, r=[1.0, 2.0, 3.0])

   # Object3D moving relatively to frame1

   frame2 = Modia3D.Object3D(frame1, r=r2, fixed=false)

   # Object3D fixed in frame2
   frame3 = Modia3D.Object3D(frame2, r=[-1,-2,-3])
end

@assembly TestGraphics begin
   world = Modia3D.Object3D()

   font  = Modia3D.Font(fontFamily="Arial", bold=true, charSize=0.2, color="LightBlue", transparency=0.5)
   text1 = Modia3D.TextShape("This is a box")
   text2 = Modia3D.TextShape("This is the xy plane"; font=font,
                              axisAlignment=Modia3D.XY_Plane, alignment=Modia3D.Left)

   textShape1 = Modia3D.Object3D(world, text1)
   textShape2 = Modia3D.Object3D(world, text2; r=[1.0, 1.0, 0.0])
end

@assembly TestVisualElements begin
   world = Modia3D.Object3D()

   vmat   = Modia3D.Material(color="Blue", transparency=0.5)
   box    = Modia3D.Box(1.0,2.0,3.0; material=vmat)

   shape1 = Modia3D.Object3D(world , box)
   shape2 = Modia3D.Object3D(world , Modia3D.Sphere(0.1))
   shape3 = Modia3D.Object3D(shape1, box; r=[1.0,2.0,3.0])
   shape4 = Modia3D.Object3D(shape3, box; R=ModiaMath.rot2(45u"°"))
end

@assembly TestSolids begin
   world = Modia3D.Object3D()

   sbox  = Modia3D.SolidBox(1.0,2.0,3.0)
   smat  = Modia3D.SolidMaterial(density = 2700)
   vmat  = Modia3D.Material(color="Blue", transparency=0.5)
   cmat  = "Steel"   # Modia3D.ContactMaterialElastic(c=1e5, d=100)
   mass  = Modia3D.MassProperties(m=0.1, Ixx=1.0, Iyy=2.0, Izz=3.0)

   solid1 = Modia3D.Object3D(world, Modia3D.Solid(sbox, "Aluminium", vmat))
   solid2 = Modia3D.Object3D(world, Modia3D.Solid(sbox, 2700       , vmat))
   solid3 = Modia3D.Object3D(world, Modia3D.Solid(sbox, smat       , vmat))
   solid4 = Modia3D.Object3D(solid1, Modia3D.Solid(sbox, nothing, contactMaterial=cmat); r=[10.0,10.0,20.0])
   solid5 = Modia3D.Object3D(world, Modia3D.Solid(Modia3D.SolidSphere(0.1), mass, vmat, contactMaterial=cmat); r=[1.0,2.0,3.0])
   solid6 = Modia3D.Object3D(world, Modia3D.Solid(nothing                 , mass); r=[1.0,2.0,3.0])
end

# Test global properties
grav1 = Modia3D.NoGravityField()
grav2 = Modia3D.UniformGravityField()
grav3 = Modia3D.PointGravityField()
r     = [Modia3D.EarthRadius, 0.0, 0.0]
g1    = norm( Modia3D.gravityAcceleration(grav1,r) )
g2    = norm( Modia3D.gravityAcceleration(grav2,r) )
g3    = norm( Modia3D.gravityAcceleration(grav3,r) )

prop = Modia3D.SceneOptions()

@testset "Modia3D.Composition: test global properties" begin
   @test g1 == 0.0
   @test isapprox(g2,9.81)
   @test isapprox(g3,9.8;atol=0.01)
end

# Test other objects
testObject3D = TestObject3D()
Modia3D.initAnalysis!(testObject3D)

for time = range(0.0, stop=2.0, length=101)
   r2 .+= 0.01*sin.([time, 2*time, 3*time])
   Modia3D.set_r!(testObject3D.frame2,r2)
   Modia3D.updatePosition!(testObject3D)  # absolute position of frame2 and frame3 updated
end
Modia3D.closeAnalysis!(testObject3D)

TestGraphics()
TestVisualElements()
TestSolids()

end


text:
abstract type HierarchicalPlan{T} <: AbstractMatrix{T} end

function *(P::HierarchicalPlan, x::AbstractVector)
    mul!(zero(x), P, x)
end

function *(P::HierarchicalPlan, X::AbstractMatrix)
    mul!(zero(X), P, X)
end

# mul!! mutates x while overwriting y. The generic fallback assumes it doesn't mutate x.
mul!!(y::AbstractVector, P::HierarchicalPlan, x::AbstractVector) = mul!(y, P, x)
mul!!(Y::AbstractMatrix, P::HierarchicalPlan, X::AbstractMatrix) = mul!(Y, P, X)
mul_col_J!!(Y::AbstractMatrix, P::HierarchicalPlan, X::AbstractMatrix, J::Int) = mul_col_J!(Y, P, X, J)

# mul! falls back to the mutating version with a copy.
mul!(y::AbstractVector, P::HierarchicalPlan, x::AbstractVector) = mul!!(y, P, copy(x))
mul!(Y::AbstractMatrix, P::HierarchicalPlan, X::AbstractMatrix) = mul!!(Y, P, copy(X))
mul_col_J!(Y::AbstractMatrix, P::HierarchicalPlan, X::AbstractMatrix) = mul_col_J!!(Y, P, copy(X), J)

function scale_col_J!(b::AbstractVector, A::AbstractVecOrMat, J::Int)
    m, n = size(A)
    COLSHIFT = m*(J-1)
    @inbounds @simd for i = 1:m
        A[i+COLSHIFT] *= b[i]
    end
    A
end

function scale_col_J!(b::Number, A::AbstractVecOrMat, J::Int)
    m, n = size(A)
    COLSHIFT = m*(J-1)
    @inbounds @simd for i = 1:m
        A[i+COLSHIFT] *= b
    end
    A
end

abstract type HierarchicalPlanWithParity{T} <: HierarchicalPlan{T} end

size(P::HierarchicalPlanWithParity) = (size(P.even, 1)+size(P.odd, 1), size(P.even, 2)+size(P.odd, 2))

evensize(v::AbstractVecOrMat, d) = (L = size(v, d); iseven(L) ? L÷2 : (L+1)÷2)
oddsize(v::AbstractVecOrMat, d) = (L = size(v, d); iseven(L) ? L÷2 : (L-1)÷2)

UpperTriangularHierarchicalMatrix(::Type{T}, f::Function, bd::Int) where {T} = UpperTriangularHierarchicalMatrix(T, f, bd, bd)
UpperTriangularHierarchicalMatrix(::Type{T}, f::Function, b::Int, d::Int) where {T} = UpperTriangularHierarchicalMatrix(T, f, 1, b, 1, d)

function UpperTriangularHierarchicalMatrix(::Type{T}, f::Function, a::Int, b::Int, c::Int, d::Int) where T
    if (b-a+1) < BLOCKSIZE(T) && (d-c+1) < BLOCKSIZE(T)
        i = (b-a)÷2
        j = (d-c)÷2
        H = HierarchicalMatrix(T, 2, 2)
        H[Block(1), Block(1)] = T[j ≥ i ? f(T,i,j) : zero(T) for i=a:a+i, j=c:c+j]
        H[Block(1), Block(2)] = T[f(T,i,j) for i=a:a+i, j=c+j+1:d]
        H[Block(2), Block(2)] = T[j ≥ i ? f(T,i,j) : zero(T) for i=a+i+1:b, j=c+j+1:d]

        H
    else
        i = (b-a)÷2
        j = (d-c)÷2
        H = HierarchicalMatrix(T, 2, 2)
        H[Block(1), Block(1)] = UpperTriangularHierarchicalMatrix(T, f, a, a+i, c, c+j)
        H[Block(1), Block(2)] = HierarchicalMatrix(T, f, a, a+i, c+j+1, d)
        H[Block(2), Block(2)] = UpperTriangularHierarchicalMatrix(T, f, a+i+1, b, c+j+1, d)

        H
    end
end

function HierarchicalMatrix(::Type{T}, f::Function, a::Int, b::Int, c::Int, d::Int) where T
    if (b-a+1) < BLOCKSIZE(T) && (d-c+1) < BLOCKSIZE(T)
        i = (b-a)÷2
        j = (d-c)÷2
        H = HierarchicalMatrix(T, 2, 2)
        H[Block(1), Block(1)] = barycentricmatrix(T, f, a, a+i, c, c+j)
        H[Block(1), Block(2)] = barycentricmatrix(T, f, a, a+i, c+j+1, d)
        H[Block(2), Block(1)] = T[f(T,i,j) for i=a+i+1:b, j=c:c+j]
        H[Block(2), Block(2)] = barycentricmatrix(T, f, a+i+1, b, c+j+1, d)

        H
    else
        i = (b-a)÷2
        j = (d-c)÷2
        H = HierarchicalMatrix(T, 2, 2)
        H[Block(1), Block(1)] = barycentricmatrix(T, f, a, a+i, c, c+j)
        H[Block(1), Block(2)] = barycentricmatrix(T, f, a, a+i, c+j+1, d)
        H[Block(2), Block(1)] = HierarchicalMatrix(T, f, a+i+1, b, c, c+j)
        H[Block(2), Block(2)] = barycentricmatrix(T, f, a+i+1, b, c+j+1, d)

        H
    end
end

function Meven(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x)).*Λ(1.0*(y+x-2)))
end

function Modd(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x)).*Λ(1.0*(y+x-1)))
end

function Leven(::Type{T}, x, y) where T
    if x == y
        if x == 1.0
            T(1.0)
        else
            T(sqrt(π)/2/Λ(2.0*(x-1.0)))
        end
    else
        T(-(y-1.0)*(2.0*x-1.5)/(2.0*x+2.0*y-3.0)/(y-x)*Λ(1.0*(y-x-1.0)).*Λ(1.0*(y+x-2.5)))
    end
end

function Lodd(::Type{T}, x, y) where T
    if x == y
        if x == 1.0
            T(1.0)
        else
            T(sqrt(π)/2/Λ(2.0*x-1.0))
        end
    else
        T(-(2.0*y-1.0)*(2.0*x-0.5)/(2.0*x+2.0*y-1.0)/(2.0*y-2.0*x)*Λ(1.0*(y-x-1)).*Λ(0.5*(2.0*y+2.0*x-3)))
    end
end

struct LegendreToChebyshevPlan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
end

function getindex(P::LegendreToChebyshevPlan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]*(2-δ(i,1))/π
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]*2/π
    else
        zero(eltype(P))
    end
end

struct ChebyshevToLegendrePlan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
end

function getindex(P::ChebyshevToLegendrePlan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]
    else
        zero(eltype(P))
    end
end

function mul!(y::Vector, P::LegendreToChebyshevPlan, x::AbstractVector)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
    lmul!(2/π, y)
    y[1] *= 0.5
    y
end

function mul!(y::Vector, P::ChebyshevToLegendrePlan, x::AbstractVector)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
end

function mul!(Y::Matrix, P::LegendreToChebyshevPlan, X::Matrix)
    m, n = size(X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    lmul!(2/π, Y)
    for j = 1:n
        Y[1+m*(j-1)] *= 0.5
    end
    Y
end

function mul!(Y::Matrix, P::ChebyshevToLegendrePlan, X::Matrix)
    m, n = size(X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    Y
end

################################################################################
# NORMLEG2CHEB

function Mevennorm(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x)).*Λ(1.0*(y+x-2)))
end

function Moddnorm(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x)).*Λ(1.0*(y+x-1)))
end

function Levennorm(::Type{T}, x, y) where T
    if x == y
        if x == 1.0
            two(T)
        else
            T(sqrt(π)/2/Λ(2.0*(x-1.0))/(2.0*x-1.5))
        end
    else
        T(-(y-1.0)/(2.0*x+2.0*y-3.0)/(y-x)*Λ(1.0*(y-x-1.0)).*Λ(1.0*(y+x-2.5)))
    end
end

function Loddnorm(::Type{T}, x, y) where T
    if x == y
        if x == 1.0
            two(T)/T(3)
        else
            T(sqrt(π)/2/Λ(2.0*x-1.0)/(2.0*x-0.5))
        end
    else
        T(-(2.0*y-1.0)/(2.0*x+2.0*y-1.0)/(2.0*y-2.0*x)*Λ(1.0*(y-x-1)).*Λ(0.5*(2.0*y+2.0*x-3)))
    end
end

struct NormalizedLegendreToChebyshevPlan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
    scl::Vector{T}
end

function getindex(P::NormalizedLegendreToChebyshevPlan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]*(2-δ(i,1))/π*P.scl[j]
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]*2/π*P.scl[j]
    else
        zero(eltype(P))
    end
end

struct ChebyshevToNormalizedLegendrePlan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
    scl::Vector{T}
end

function getindex(P::ChebyshevToNormalizedLegendrePlan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]*P.scl[i]
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]*P.scl[i]
    else
        zero(eltype(P))
    end
end

function mul!!(y::Vector, P::NormalizedLegendreToChebyshevPlan, x::AbstractVector)
    unsafe_broadcasttimes!(x, P.scl)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
    lmul!(2/π, y)
    y[1] *= 0.5
    y
end

function mul!(y::Vector, P::ChebyshevToNormalizedLegendrePlan, x::AbstractVector)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
    unsafe_broadcasttimes!(y, P.scl)
end

function mul!!(Y::Matrix, P::NormalizedLegendreToChebyshevPlan, X::Matrix)
    m, n = size(X)
    lmul!(P.scl, X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    lmul!(2/π, Y)
    @inbounds @simd for j = 1:n
        Y[1+m*(j-1)] *= 0.5
    end
    Y
end

function mul!(Y::Matrix, P::ChebyshevToNormalizedLegendrePlan, X::Matrix)
    m, n = size(X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    lmul!(P.scl, Y)
end

function mul_col_J!!(Y::Matrix, P::NormalizedLegendreToChebyshevPlan, X::Matrix, J::Int)
    m, n = size(X)
    COLSHIFT = m*(J-1)
    scale_col_J!(P.scl, X, J)
    mul!(Y, P.even, X, 1+COLSHIFT, 1+COLSHIFT, 2, 2)
    mul!(Y, P.odd, X, 2+COLSHIFT, 2+COLSHIFT, 2, 2)
    scale_col_J!(2/π, Y, J)
    @inbounds Y[1+COLSHIFT] *= 0.5
    Y
end

function mul_col_J!(Y::Matrix, P::ChebyshevToNormalizedLegendrePlan, X::Matrix, J::Int)
    m, n = size(X)
    COLSHIFT = m*(J-1)
    mul!(Y, P.even, X, 1+COLSHIFT, 1+COLSHIFT, 2, 2)
    mul!(Y, P.odd, X, 2+COLSHIFT, 2+COLSHIFT, 2, 2)
    scale_col_J!(P.scl, Y, J)
end

################################################################################
# NORMLEG12CHEB2

function Mnormeven(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x))*Λ(1.0*(y+x-1))*(4.0*x-2.0)/π)
end

function Mnormodd(::Type{T}, x, y) where T
    T(Λ(1.0*(y-x))*Λ(1.0*(y+x))*(4.0*x)/π)
end

function Lnormeven(::Type{T}, x, y) where T
    -T(Λ(1.0*(y-x))/(2.0*y-2.0*x-1.0)*Λ(1.0*(y+x-1)+0.5)/(2.0*y+2.0*x-2.0)*(2.0*x-0.5))
end

function Lnormodd(::Type{T}, x, y) where T
    -T(Λ(1.0*(y-x))/(2.0*y-2.0*x-1.0)*Λ(1.0*(y+x)+0.5)/(2.0*y+2.0*x)*(2.0*x+0.5))
end

struct NormalizedLegendre1ToChebyshev2Plan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
    scl::Vector{T}
end

function getindex(P::NormalizedLegendre1ToChebyshev2Plan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]*P.scl[j]
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]*P.scl[j]
    else
        zero(eltype(P))
    end
end

struct Chebyshev2ToNormalizedLegendre1Plan{T} <: HierarchicalPlanWithParity{T}
    even::HierarchicalMatrix{T}
    odd::HierarchicalMatrix{T}
    scl::Vector{T}
end

function getindex(P::Chebyshev2ToNormalizedLegendre1Plan, i::Int, j::Int)
    if isodd(i) && isodd(j)
        P.even[(i+1)÷2,(j+1)÷2]*P.scl[i]
    elseif iseven(i) && iseven(j)
        P.odd[i÷2,j÷2]*P.scl[i]
    else
        zero(eltype(P))
    end
end

function mul!!(y::Vector, P::NormalizedLegendre1ToChebyshev2Plan, x::AbstractVector)
    unsafe_broadcasttimes!(x, P.scl)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
end

function mul!(y::Vector, P::Chebyshev2ToNormalizedLegendre1Plan, x::AbstractVector)
    mul!(y, P.even, x, 1, 1, 2, 2)
    mul!(y, P.odd, x, 2, 2, 2, 2)
    unsafe_broadcasttimes!(y, P.scl)
end

function mul!!(Y::Matrix, P::NormalizedLegendre1ToChebyshev2Plan, X::Matrix)
    m, n = size(X)
    lmul!(P.scl, X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    Y
end

function mul!(Y::Matrix, P::Chebyshev2ToNormalizedLegendre1Plan, X::Matrix)
    m, n = size(X)
    for j = 1:n
        mul!(Y, P.even, X, 1+m*(j-1), 1+m*(j-1), 2, 2)
        mul!(Y, P.odd, X, 2+m*(j-1), 2+m*(j-1), 2, 2)
    end
    lmul!(P.scl, Y)
end

function mul_col_J!!(Y::Matrix, P::NormalizedLegendre1ToChebyshev2Plan, X::Matrix, J::Int)
    m, n = size(X)
    COLSHIFT = m*(J-1)
    scale_col_J!(P.scl, X, J)
    mul!(Y, P.even, X, 1+COLSHIFT, 1+COLSHIFT, 2, 2)
    mul!(Y, P.odd, X, 2+COLSHIFT, 2+COLSHIFT, 2, 2)
    Y
end

function mul_col_J!(Y::Matrix, P::Chebyshev2ToNormalizedLegendre1Plan, X::Matrix, J::Int)
    m, n = size(X)
    COLSHIFT = m*(J-1)
    mul!(Y, P.even, X, 1+COLSHIFT, 1+COLSHIFT, 2, 2)
    mul!(Y, P.odd, X, 2+COLSHIFT, 2+COLSHIFT, 2, 2)
    scale_col_J!(P.scl, Y, J)
end

### API

for (f, T, feven, fodd) in ((:leg2cheb, :LegendreToChebyshevPlan, :Meven, :Modd),
                          (:cheb2leg, :ChebyshevToLegendrePlan, :Leven, :Lodd),
                          (:normleg2cheb, :NormalizedLegendreToChebyshevPlan, :Mevennorm, :Moddnorm),
                          (:cheb2normleg, :ChebyshevToNormalizedLegendrePlan, :Levennorm, :Loddnorm),
                          (:normleg12cheb2, :NormalizedLegendre1ToChebyshev2Plan, :Mnormeven, :Mnormodd),
                          (:cheb22normleg1, :Chebyshev2ToNormalizedLegendre1Plan, :Lnormeven, :Lnormodd))
    plan_f = Meta.parse("plan_"*string(f))
    plan_even_f = Meta.parse("plan_even_"*string(f))
    plan_odd_f = Meta.parse("plan_odd_"*string(f))
    @eval begin
        $plan_f(v::VecOrMat) = $T(v)
        $f(v::VecOrMat) = $plan_f(v)*v
        $plan_even_f(v::VecOrMat) = UpperTriangularHierarchicalMatrix(eltype(v), $feven, evensize(v, 1))
        $plan_odd_f(v::VecOrMat) = UpperTriangularHierarchicalMatrix(eltype(v), $fodd, oddsize(v, 1))
    end
end

LegendreToChebyshevPlan(v::VecOrMat) = LegendreToChebyshevPlan(plan_even_leg2cheb(v), plan_odd_leg2cheb(v))
ChebyshevToLegendrePlan(v::VecOrMat) = ChebyshevToLegendrePlan(plan_even_cheb2leg(v), plan_odd_cheb2leg(v))

NormalizedLegendreToChebyshevPlan(v::VecOrMat) = NormalizedLegendreToChebyshevPlan(plan_even_normleg2cheb(v), plan_odd_normleg2cheb(v), eltype(v)[sqrt(j-0.5) for j in 1:size(v, 1)])
ChebyshevToNormalizedLegendrePlan(v::VecOrMat) = ChebyshevToNormalizedLegendrePlan(plan_even_cheb2normleg(v), plan_odd_cheb2normleg(v), eltype(v)[sqrt(i-0.5) for i in 1:size(v, 1)])

NormalizedLegendre1ToChebyshev2Plan(v::VecOrMat) = NormalizedLegendre1ToChebyshev2Plan(plan_even_normleg12cheb2(v), plan_odd_normleg12cheb2(v), eltype(v)[sqrt((j+0.5)/(j*(j+1))) for j in 1:size(v, 1)])
Chebyshev2ToNormalizedLegendre1Plan(v::VecOrMat) = Chebyshev2ToNormalizedLegendre1Plan(plan_even_cheb22normleg1(v), plan_odd_cheb22normleg1(v), eltype(v)[sqrt(i*(i+1)/(i+0.5)) for i in 1:size(v, 1)])

"""
Computes the Chebyshev expansion coefficients given the Legendre expansion coefficients:

```math
{\\rm CLT} : \\sum_{n=0}^N c_n^{\\rm leg}P_n(x) \\to \\sum_{n=0}^N c_n^{\\rm cheb}T_n(x).
```
"""
leg2cheb(::Vector)

"""
Computes the Legendre expansion coefficients given the Chebyshev expansion coefficients:

```math
{\\rm iCLT} : \\sum_{n=0}^N c_n^{\\rm cheb}T_n(x) \\to \\sum_{n=0}^N c_n^{\\rm leg}P_n(x).
```
"""
cheb2leg(::Vector)

"""
Pre-computes the Legendre--Chebyshev transform.
"""
plan_leg2cheb(::Vector)

"""
Pre-computes the Chebyshev--Legendre transform.
"""
plan_cheb2leg(::Vector)


text:
#================
Preprocessing
     Module
================#
@doc """
      |====== Lathe.preprocess =====\n
      |____________/ Generalized Processing ___________\n
      |_____preprocess.TrainTestSplit\n
      |_____preprocess.SortSplit\n
      |_____preprocess.UniformSplit\n
      |____________/ Feature Scaling ___________\n
      |_____preprocess.Rescalar\n
      |_____preprocess.ArbitraryRescale\n
      |_____preprocess.MeanNormalization\n
      |_____preprocess.StandardScalar\n
      |____________/ Categorical Encoding ___________\n
      |_____preprocess.OneHotEncoder\n
      |_____preprocess.OrdinalEncoder\n
      |_____preprocess.FloatEncoder\n

       """ ->
module preprocess
# [deps]
using Random
using DataFrames
using Lathe.stats
# [deps]
abstract type Preprocessor <: LatheObject end
abstract type Encoder <: Preprocessor end
abstract type Scaler <: Preprocessor end
abstract type Transformer <: Preprocessor end
abstract type Manager <: Preprocessor end
abstract type Booster <: Preprocessor end
#==
Data Processing
==#
include("preprocess/data_processing.jl")
#==
Encoding
==#
include("preprocess/encoding.jl")
#==
Feature Scaling
==#
include("preprocess/feature_scaling.jl")
#==
Macros
==#
include("preprocess/preprocess_macros.jl")
#==
Decompisition
==#
include("preprocess/decomp.jl")
#-----------------------------
end


text:
module EmonCMS

export EmonDataSet
export update
export energyperperiod
export feedlist
export getfeed
export yearlysummary
export energysummary
export exportcsv

using Unitful
import HTTP
using URIs
import JSON3
using JuliaDB
using Dates
using ProgressMeter
using Trapz
using DelimitedFiles
using Statistics

struct Connection
  serveraddress::URI
  apikey::String
end
Connection(address::String, apikey::String) = Connection(URI(address), String(apikey))

JSON3.StructTypes.StructType(::Type{Connection}) = JSON3.StructTypes.Struct()
JSON3.StructTypes.StructType(::Type{URI}) = JSON3.StructTypes.Struct()

headers(c::Connection) = Dict("Authorization" => c.apikey)

function getjson(c::Connection, page, parameters=Dict())
  uri = URI(joinpath(c.serveraddress, page); query=parameters)
  response = HTTP.get(uri, headers(c))
  if isempty(response.body)
    return []
  end
  result_json = JSON3.read(response.body)
  if "success" ∈ keys(result_json) && !result_json["success"]
    throw(ErrorException("Request $uri failed with error: $(result_json["message"])"))
  end
  return result_json
end

list(c::Connection) = getjson(c, "list.json")
aget(c::Connection, id) = getjson(c, "aget.json", Dict("id" => id))
getmeta(c::Connection, id) = getjson(c, "getmeta.json", Dict("id" => id))

emontimemilli(dt::DateTime) = Int(floor(datetime2unix(dt)*1000))

const maxnbjsonentries = 8928

maxtimestep(interval) = maxnbjsonentries*1000*interval

"""
Get data from a feed. first and last are in seconds here, and converted to the required miliseconds. Interval is also in seconds
"""
function data(c::Connection, id, first, last, interval)
  # Times in seconds
  parameters = Dict("id" => id, "start" => first*1000, "end" => last*1000, "interval" => interval)
  return getjson(c, "data.json", parameters)
end

connectionfile(base) = joinpath(base,"connection.json")
feedsfile(base) = joinpath(base,"feeds"*dbextension)

"""
  EmonDataSet(path)

Load the dataset at the given path

  EmonDataSet(path, serveraddress, apikey)

Construct a new dataset in the existing directory at `path`, connecting to `serveraddress` using read-only EmonCMS API key `apikey`
"""
struct EmonDataSet
  path::String
  connection::Connection

  function EmonDataSet(path)
    if !isfile(connectionfile(path))
      throw(ErrorException("No existing database at \"$path\". Please create a new one using EmonDataSet(path, serveraddress, apikey)"))
    end
    connjson = JSON3.read(read(connectionfile(path), String))
    return new(path, Connection(connjson["serveraddress"]["uri"],connjson["apikey"]))
  end

  function EmonDataSet(path, serveraddress, apikey)
    if !isdir(path)
      throw(ErrorException("\"$path\" is not a directory, please create it first."))
    end
    return new(path, Connection(serveraddress, apikey))
  end
end

const dbextension = ".juliadb"

createfeedtable(t::AbstractVector{Int64}=Int64[], values::AbstractVector{Union{Missing,Float64}}=Vector{Union{Missing,Float64}}()) = table((time=t, value=values), pkey=:time)

toseconds(n)::Int64 = n ÷ 1000

function appendblock(feedtable, block, interval, blockstart, blockend)
  if isempty(block)
    return feedtable
  end

  blockfirst = toseconds(block[1][1])
  blocklast = toseconds(block[end][1])
  if (blocklast - blockfirst) % interval != 0
    throw(ErrorException("Block with starttime $starttime and endtime $endtime does not match interval $interval"))
  end
  if !isempty(feedtable) && ((blockfirst - feedtable[end][:time]) % interval) != 0
    throw(ErrorException("Feed timing does not match stored table"))
  end

  starttime = isempty(feedtable) ? blockfirst : feedtable[end][:time]+interval
  times = starttime:interval:blocklast
  values = Array{Union{Missing, Float64}}(missing, length(times))
  function time_to_index(t)
    @assert (t-starttime) % interval == 0
    return (t-starttime) ÷ interval + 1
  end
  for (t,v) in block
    t = toseconds(t)
    if t < blockstart || t > blockend || t < starttime
      continue
    end
    i = time_to_index(t)
    values[i] = v
  end

  return merge(feedtable, createfeedtable(times,values))
end

function updatefeedtable(getter::Function, conn, feedtable, feedid, feedstart, feedend, interval, name)
  starttime = (isempty(feedtable) ? feedstart : (feedtable[end][:time] + interval))::Int64
  nsteps = (feedend - (starttime-interval)) ÷ interval
  nblocks = Int(ceil(nsteps / maxnbjsonentries))
  @showprogress 0.5 "Appending $nblocks blocks to feed $name..." for blockidx in 1:nblocks
    blockstart = starttime + (blockidx - 1)*maxnbjsonentries*interval
    blockend = blockstart + (maxnbjsonentries-1)*interval
    try
      block = getter(conn, feedid, blockstart, blockend, interval)
      feedtable = appendblock(feedtable, block, interval, blockstart, blockend)
    catch e
      @warn "Aborting feed update with error $e"
      break
    end
  end
  @info "Appended $nsteps entries to feed $name"

  return feedtable
end

function writeconnfile(connection, filename)
  @info "Creating new database at \"$(dirname(filename))\""

  open(filename, "w") do filestream
    JSON3.pretty(filestream, JSON3.write(connection))
    println(filestream)
  end
end
"""
  update(ds)

Update the EmonDataSet `ds` with the latest available data on the emonPi.

  update(ds; feeds=[id1, id2, ...])

First call to update on EmonDataSet `ds` , with a list of integer feed IDs to store in the dataset.
"""
function update(ds::EmonDataSet; feeds=[])
  connfile = connectionfile(ds.path)
  if !isfile(connfile)
    writeconnfile(ds.connection, connfile)
  end

  conn = ds.connection

  feedlistfile = feedsfile(ds.path)
  if !isfile(feedlistfile)
    if isempty(feeds)
      throw(ErrorException("No existing feeds list found, and no feeds given. Call update with keyword argument \"feeds=[id1, if2, ...]\""))
    end
    @info "Creating new feeds list"

    nfeeds = length(feeds)
    feedstuple = (id=feeds, unit=fill("W", nfeeds), name=String[], starttime=Int64[], interval=Int64[])
    for id in feeds
      feedinfo = aget(conn, id)
      feedmeta = getmeta(conn, id)
      push!(feedstuple.name, feedinfo["name"])
      push!(feedstuple.starttime, feedmeta["start_time"]) # start_time in seconds
      push!(feedstuple.interval, feedmeta["interval"])
    end

    feedstable = table(feedstuple, pkey=[:id, :name])
    save(feedstable, feedlistfile)
  else
    if !isempty(feeds)
      throw(ErrorException("Feeds table exists, can't add new feeds. Call update without feed list or create a new database"))
    end
    feedstable = load(feedlistfile)
  end

  for fd in feedstable
    feedinfo = aget(conn, fd.id)
    feedfile = joinpath(ds.path, fd.name*dbextension)
    feedtable = isfile(feedfile) ? load(feedfile) : createfeedtable()
    updatedfeedtable = updatefeedtable(data, conn, feedtable, fd.id, fd.starttime, feedinfo["time"], fd.interval, fd.name)
    save(updatedfeedtable, feedfile)
  end
  return
end

"""
  feedlist(ds)

Get the table of stored feeds in `ds`.
"""
feedlist(ds) = load(feedsfile(ds.path))

loadfeed(ds,name) = load(joinpath(ds.path, name*dbextension))

"""
  getfeed(ds, name)

Get the values of a given feed. The result is a table with columns `time` and `value`.
"""
function getfeed(ds::EmonDataSet, name)
  feedtable = loadfeed(ds, name)
  unit = filter(i -> i.name == name, feedlist(ds))[1][:unit]
  return table((time= unix2datetime.(select(feedtable, :time)), value=select(feedtable, :value) .* uparse(unit)), pkey=:time)
end

function _replace_missing_limited(values, allowedmissing)
  nbmissing = count(ismissing, values)
  if nbmissing != 0 && nbmissing/length(values) < allowedmissing
    return replace(values, missing => zero(eltype(values)))
  end

  return values
end

"""
  energyperperiod(feedtable, period)

For the power data in `feedtable`, calculate the energy per given periods, returning the result as a table with columns `date` and `energy`

Example:

To get the monthly totals of feed `HeatPump` do:
```
feedtable = getfeed(ds, "HeatPump")
energyperperiod(feedtable, Month(1))
```

`Month` comes from the `Dates` package.

* The keyword argument `energyunit` allows you to set the unit of the output (`kW hr` by default)
* The keyword argument `allowedmissing` indicates the fraction of missing values that is allowed to be replaced by zero. If this fraction is exceeded for a given period, the period value will be `missing`.

"""
function energyperperiod(feedtable, period; energyunit = u"kW*hr", allowedmissing = 0.1)
  pertype = typeof(period)

  times = datetime2unix.(select(feedtable, :time))
  values = ustrip.(u"W", select(feedtable, :value))

  starttime = floor(unix2datetime(times[1]), pertype)
  endtime = floor(unix2datetime(times[end]), pertype)
  daterange = starttime:period:endtime
  energyvalues = zeros(Union{Float64,Missing}, length(daterange))
  for (i,p) in enumerate(daterange)
    periodstart = (datetime2unix(p) |> Int64)
    periodend = (datetime2unix(p + period) |> Int64)
    firstidx = max(searchsortedlast(times, periodstart),1)
    lastidx = min(searchsortedfirst(times, periodend), length(times))
    energyvalues[i] = trapz(times[firstidx:lastidx], _replace_missing_limited(values[firstidx:lastidx], allowedmissing))
  end

  return table((date=daterange, energy=((energyvalues .* u"J") .|> energyunit)), pkey=:date)
end

"""
  yearlysummary(feed, period, years)

Return the average energy used for each period (e.g. monthly) over the given range of years, for the given feed.

The returned value is a tuple containing the averaged values and the number of times this period was counted.
"""
function yearlysummary(reduction, feed, period, years=[2018,2019,2020]; allowedmissing = 0.1)
  nbperiods = length(DateTime(2017):period:DateTime(2018))-1
  yearlytable = Array{Union{Missing,typeof(0.0u"kW*hr")}}(missing, nbperiods, length(years))
  result = zeros(Union{Missing,typeof(0.0u"kW*hr")}, nbperiods)
  for (yearidx,y) in enumerate(years)
    yearintegral = energyperperiod(filter(f -> (year(f.time) == y), feed), period; allowedmissing=allowedmissing)
    yearintegral = filter(f -> !(month(f.date) == 2 && day(f.date) == 29), yearintegral)
    values = select(yearintegral, :energy)
    for i in eachindex(skipmissing(values))
      if i > nbperiods
        break
      end
      yearlytable[i,yearidx] = values[i]
    end
  end
  counts = length(years) .- reshape(count(ismissing, yearlytable; dims=2) , nbperiods)
  for i in 1:nbperiods
    yearvals = yearlytable[i,:]
    if count(ismissing, yearvals) == length(yearvals)
      result[i] = missing
    else
      result[i] = reduction(skipmissing(yearvals))
    end
  end
  return result, counts
end

"""
  energysummary(ds)

Summarize energy usage over a number of years for a set of feeds, adding a feed named "Unknown" by subtracting all other feeds from a set of feeds that give the total power.

Example:

```
names, energies, counts = energysummary(ds; years=[2018,2019,2020], totalpowerfeeds = ["L1"])
```

The `years` argument lists the years over which to average, the `totalpowerfeeds` argument gives a list of feeds that when added together give the total power (e.g. measurements on each incoming phase).

This returns:
* `names`: The names of the considered feeds (all feeds by default, or the names in the array passed to the `feeds` keyword argument)
* `energies`: 2D array, with each column `i` the averaged energy for feed `i` in `names`. Rows are the period numbers (months by default, change with keyword argument `period`)
* `counts`: Number of times each period was non-missing for each feed.
"""
function energysummary(ds; period=Month(1), years=[2018,2019,2020], totalpowerfeeds = ["L1_in", "L2_in", "L3_in"], feeds=select(feedlist(ds),:name), allowedmissing = 0.1, reduction=mean)
  # Collect averages per period for each feed
  energydict = Dict()
	for feedname in feeds
		energies,counts = yearlysummary(reduction, getfeed(ds, feedname), period, years; allowedmissing=allowedmissing)
		energydict[feedname] = (energies=energies,counts=counts)
	end

  # Compute the total energy
	totalenergy = energydict[totalpowerfeeds[1]].energies
  for feedname in totalpowerfeeds[2:end]
    totalenergy .+=  energydict[feedname].energies
  end

  # Store the energy per period in a 2D array
	nbpers = length(totalenergy)
	wantedkeys = setdiff(feeds, totalpowerfeeds)
	energies = zeros(Union{Missing,typeof(0.0u"kW*hr")},nbpers,length(wantedkeys)+1)
	counts = zeros(Int, size(energies))
	for (i,name) in enumerate(wantedkeys)
		(h,c) = energydict[name]
		energies[:,i+1] .= h
		counts[:,i+1] .= c
	end
	energies[:,1] .= totalenergy .- reshape(sum(energies[:,2:end];dims=2),nbpers)
	counts[:,1] .= energydict[totalpowerfeeds[1]].counts
	pushfirst!(wantedkeys, "Unknown")
	return reshape(wantedkeys,1,length(wantedkeys)), energies, counts
end

function exporttable(filename, table)
  if isfile(filename)
    throw(ErrorException("Refusing to overwrite $filename"))
  end
  open(filename,"w") do f
    println(f,join(colnames(table),','))
    writedlm(f,table,',')
  end
end

function importfeed(filename)
  return loadtable(filename; nastrings=["missing"], indexcols=[:time])
end

"""
  exportcsv(ds, path)

Export all tables as CSV files in directory `path`
"""
function exportcsv(ds, path)
  if !isdir(path)
    throw(ErrorException("Export path $path must be an existing directory"))
  end
  exporttable(joinpath(path, "feedlist.csv"), feedlist(ds))
  for feedname in select(feedlist(ds),:name)
    exporttable(joinpath(path, "$feedname.csv"), loadfeed(ds, feedname))
  end
end

end # module


text:
module Misc

    export assert_interval, subarray_checkbounds, MSG_WARN_UNSAFE

    @inline assert_interval(interval::UnitRange) =
        @assert interval.stop >= interval.start
                "Interval start should be lesser or equal than stop)"

    @inline checkbounds(::Type{Bool}, arr::AbstractArray, interval::UnitRange) =
        interval.start > 0 && interval.stop <= length(arr)

    @inline throw_bounds_error(arr::AbstractArray, interval::UnitRange) =
        throw(BoundsError(arr, interval))

    @inline subarray_checkbounds(arr::AbstractArray, interval::UnitRange) =
        checkbounds(Bool, arr, interval) || throw_bounds_error(arr, interval)

end


text:
@testset "Preprocessing (newer)" begin
	poem_no_1 = """
                \t ...
                \t this is it      !            \n
                \t 1,2,3,y,2,z                  \n
                \t  GET some h/a\\y             \n
                \t         v`v`v`v`v            \n
                \t  FIND a >>>pÎn<<<            \n
                \t         ^^^^^^^^^            \n
                \t  p-a-t-t-e-r-n-s'n f.i.n     \n
                \t  get a <tag, hold it still>  \n
                \t          ~ 0_0' ~            \n
                \t        </tag a thrill>       \n
                \t          b.y. ~z.g.o.r.n.e.l \n
                                               """
    # All flags, except stemming
    sdoc = StringDocument(poem_no_1)
    ndoc = NGramDocument(poem_no_1)
    tdoc = TokenDocument(poem_no_1)
    crps = Corpus([sdoc])
    map(x->prepare!(x, strip_everything_stem), [sdoc, ndoc, tdoc, crps])
    @test prepare(poem_no_1, strip_everything_stem) == "pin"
    @test text(sdoc) == "pin"
    @test ngrams(ndoc) == Dict("tag"=>2,"pin"=>1,"hold"=>1,"thrill"=>1)
    @test tokens(tdoc) == ["pin ", "tag", "hold", "tag", "thrill "]
    @test text(crps[1]) == "pin"
    # Flag generation
    v = [2, 5, 7]
    vs = UInt32(0)
    b = UInt32(1)
    for i in v
        vs |= b<<i
    end
    c_flags = StringAnalysis.flag_generate(v)
    @test c_flags isa UInt32
    @test c_flags == vs
    v = UInt32[strip_accents, strip_prepositions, strip_html_tags]
    c_flags = StringAnalysis.flag_generate(v...)
    @test c_flags isa UInt32
    @test c_flags == reduce(|, v)
end

# Text Analysis older tests
@testset "Preprocessing (older)" begin
    sample_text1 = "This is, 1 MESSED υπ string!..."
    sample_text1_accents = "Thîs îs meşsed string"
    sample_text1_wo_punctuation = "This is 1 MESSED υπ string"
    sample_text1_wo_punctuation_numbers = "This is  MESSED υπ string"
    sample_text1_wo_punctuation_numbers_case = "this is  messed υπ string"
    sample_text1_wo_punctuation_numbers_case_az = "this is  messed  string"

    sample_texts = [
        sample_text1,
        sample_text1_wo_punctuation,
        sample_text1_wo_punctuation_numbers,
        sample_text1_wo_punctuation_numbers_case,
        sample_text1_wo_punctuation_numbers_case_az
    ]

    # This idiom is _really_ ugly since "OR" means "AND" here.
    for str in sample_texts
        sd = StringDocument(str)
        prepare!(
            sd,
            strip_punctuation | strip_numbers |
            strip_case | strip_whitespace | strip_non_ascii |
            strip_accents
        )
        @test isequal(strip(sd.text), "this is messed string")
    end

    # Need to only remove words at word boundaries
    doc = Document("this is sample text")
    StringAnalysis.remove_words!(doc, ["sample"])
    @test isequal(doc.text, "this is   text")

    doc = Document("this is sample text")
    prepare!(doc, strip_articles)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_definite_articles)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_indefinite_articles)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_prepositions)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_pronouns)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_stopwords)
    @test isequal(strip(doc.text), "sample text")

    doc = Document("this is sample text")
    prepare!(doc, strip_whitespace)
    @test isequal(doc.text, "this is sample text")

    doc = Document("this îs sămple text")
    prepare!(doc, strip_accents)
    @test isequal(doc.text, "this is sample text")

    # stem!(sd)
    # tag_pos!(sd)

    # Do preprocessing on TokenDocument, NGramDocument, Corpus
    d = NGramDocument("this is sample text")
    @test haskey(d.ngrams, "sample")
    StringAnalysis.remove_words!(d, ["sample"])
    @test !haskey(d.ngrams, "sample")

    d = StringDocument(
        """

Hello
world

        """
    )
    StringAnalysis.prepare!(d, strip_html_tags)
    @test "Hello  world" == strip(d.text)

    #Test #62
    StringAnalysis.remove_corrupt_utf8("abc") == "abc"
    StringAnalysis.remove_corrupt_utf8(String([0x43, 0xf0])) == "C "

    # Minimal test of sparse/frequent terms
    doc = "a a a a a a a a a a a b cx c c c c c c c v"
    sd = StringDocument(doc)
    ngd = NGramDocument(doc)
    crps = Corpus(AbstractDocument[sd, ngd])

    # Sparsity/frequency at document level
    st = sparse_terms(doc, 0.1)
    @test st isa Vector{String}
    @test length(st) == 3
    @test isempty(setdiff(st, ["b","cx","v"]))

    ft = frequent_terms(doc, 0.5)
    @test ft isa Vector{String}
    @test length(ft) == 1
    @test isempty(setdiff(ft, ["a"]))

    # Sparsity/frequency at corpus level
    # (all letters are frequent since they appear in both documents)
    st = sparse_terms(crps, 0.1)
    @test st isa Vector{String}
    @test length(st) == 0

    ft = frequent_terms(crps, 0.5)
    @test ft isa Vector{String}
    @test length(ft) == 5
    @test isempty(setdiff(ft, ["a", "b","c", "cx","v"]))
end