text:
module FIGlet

using Pkg.Artifacts
import Base

const FONTSDIR = abspath(normpath(joinpath(artifact"fonts", "FIGletFonts-0.5.0", "fonts")))
const UNPARSEABLES = [
              "nvscript.flf",
             ]
const DEFAULTFONT = "Standard"

abstract type FIGletError <: Exception end

"""
Width is not sufficient to print a character
"""
struct CharNotPrinted <: FIGletError end

"""
Font can't be located
"""
struct FontNotFoundError <: FIGletError
    msg::String
end

Base.showerror(io::IO, e::FontNotFoundError) = print(io, "FontNotFoundError: $(e.msg)")

"""
Problem parsing a font file
"""
struct FontError <: FIGletError
    msg::String
end

Base.showerror(io::IO, e::FontError) = print(io, "FontError: $(e.msg)")

"""
Color is invalid
"""
struct InvalidColorError <: FIGletError end

Base.@enum(Layout,
    FullWidth                   =       -1,
    HorizontalSmushingRule1     =        1,
    HorizontalSmushingRule2     =        2,
    HorizontalSmushingRule3     =        4,
    HorizontalSmushingRule4     =        8,
    HorizontalSmushingRule5     =       16,
    HorizontalSmushingRule6     =       32,
    HorizontalFitting           =       64,
    HorizontalSmushing          =      128,
    VerticalSmushingRule1       =      256,
    VerticalSmushingRule2       =      512,
    VerticalSmushingRule3       =     1024,
    VerticalSmushingRule4       =     2048,
    VerticalSmushingRule5       =     4096,
    VerticalFitting             =     8192,
    VerticalSmushing            =    16384,
)

struct FIGletHeader
    hardblank::Char
    height::Int
    baseline::Int
    max_length::Int
    old_layout::Int
    comment_lines::Int
    print_direction::Int
    full_layout::Int
    codetag_count::Int

    function FIGletHeader(
                          hardblank,
                          height,
                          baseline,
                          max_length,
                          old_layout,
                          comment_lines,
                          print_direction=0,
                          full_layout=Int(HorizontalSmushingRule2),
                          codetag_count=0,
                          args...,
                      )
        length(args) >0 && @warn "Received unknown header attributes: `$args`."
        height < 1 && ( height = 1 )
        max_length < 1 && ( max_length = 1 )
        print_direction < 0 && ( print_direction = 0 )
        # max_length += 100 # Give ourselves some extra room
        new(hardblank, height, baseline, max_length, old_layout, comment_lines, print_direction, full_layout, codetag_count)
    end
end

function FIGletHeader(
                      hardblank,
                      height::AbstractString,
                      baseline::AbstractString,
                      max_length::AbstractString,
                      old_layout::AbstractString,
                      comment_lines::AbstractString,
                      print_direction::AbstractString="0",
                      full_layout::AbstractString="2",
                      codetag_count::AbstractString="0",
                      args...,
                     )
    return FIGletHeader(
                        hardblank,
                        parse(Int, height),
                        parse(Int, baseline),
                        parse(Int, max_length),
                        parse(Int, old_layout),
                        parse(Int, comment_lines),
                        parse(Int, print_direction),
                        parse(Int, full_layout),
                        parse(Int, codetag_count),
                        args...,
                       )
end

struct FIGletChar
    ord::Char
    thechar::Matrix{Char}
end

struct FIGletFont
    header::FIGletHeader
    font_characters::Dict{Char,FIGletChar}
    version::VersionNumber
end

Base.show(io::IO, ff::FIGletFont) = print(io, "FIGletFont(n=$(length(ff.font_characters)))")

function readmagic(io)
    magic = read(io, 5)
    magic[1:4] != UInt8['f', 'l', 'f', '2'] && throw(FontError("File is not a valid FIGlet Lettering Font format. Magic header values must start with `flf2`."))
    magic[5] != UInt8('a') && @warn "File may be a FLF format but not flf2a."
    return magic # File has valid FIGlet Lettering Font format magic header.
end

function readfontchar(io, ord, height)

    s = readline(io)
    width = length(s)-1
    width == -1 && throw(FontError("Unable to find character `$ord` in FIGlet Font."))
    thechar = Matrix{Char}(undef, height, width)

    for (w, c) in enumerate(s)
        w > width && break
        thechar[1, w] = c
    end

    for h in 2:height
        s = readline(io)
        for (w, c) in enumerate(s)
            w > width && break
            thechar[h, w] = c
        end
    end

    return FIGletChar(ord, thechar)
end

Base.show(io::IO, fc::FIGletChar) = print(io, "FIGletChar(ord='$(fc.ord)')")

function readfont(s::AbstractString)
    name = s
    if !isfile(name)
        name = abspath(normpath(joinpath(FONTSDIR, name)))
        if !isfile(name)
            name = "$name.flf"
            !isfile(name) && throw(FontNotFoundError("Cannot find font `$s`."))
        end
    end

    font = open(name) do f
        readfont(f)
    end
    return font
end

function readfont(io)
    magic = readmagic(io)

    header = split(readline(io))
    fig_header = FIGletHeader(
                           header[1][1],
                           header[2:end]...,
                          )

    for i in 1:fig_header.comment_lines
        discard = readline(io)
    end

    fig_font = FIGletFont(
                          fig_header,
                          Dict{Char, FIGletChar}(),
                          v"2.0.0",
                         )

    for c in ' ':'~'
        fig_font.font_characters[c] = readfontchar(io, c, fig_header.height)
    end

    for c in ['Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', 'ß']
        if bytesavailable(io) > 1
            fig_font.font_characters[c] = readfontchar(io, c, fig_header.height)
        end
    end

    while bytesavailable(io) > 1
        s = readline(io)
        strip(s) == "" && continue
        s = split(s)[1]
        c = if '-' in s
            Char(-(parse(UInt16, strip(s, '-'))))
        else
            Char(parse(Int, s))
        end
        fig_font.font_characters[c] = readfontchar(io, c, fig_header.height)
    end

    return fig_font
end

function availablefonts(substring)
    fonts = String[]
    for (root, dirs, files) in walkdir(FONTSDIR)
        for file in files
            if !(file in UNPARSEABLES)
                if occursin(lowercase(substring), lowercase(file)) || substring == ""
                    push!(fonts, replace(file, ".flf"=>""))
                end
            end
        end
    end
    sort!(fonts)
    return fonts
end

"""
    availablefonts() -> Vector{String}
    availablefonts(substring::AbstractString) -> Vector{String}

Returns all available fonts.
If `substring` is passed, returns available fonts that contain the case insensitive `substring`.

Example:

    julia> availablefonts()
    680-element Array{String,1}:
     "1943____"
     "1row"
     ⋮
     "zig_zag_"
     "zone7___"

    julia> FIGlet.availablefonts("3d")
    5-element Array{String,1}:
     "3D Diagonal"
     "3D-ASCII"
     "3d"
     "Henry 3D"
     "Larry 3D"

    julia>
"""
availablefonts() = availablefonts("")

raw"""

    smushem(lch::Char, rch::Char) -> Char

Given 2 characters, attempts to smush them into 1, according to
smushmode.  Returns smushed character or '\0' if no smushing can be
done.

smushmode values are sum of following (all values smush blanks):
    1: Smush equal chars (not hardblanks)
    2: Smush '_' with any char in hierarchy below
    4: hierarchy: "|", "/\", "[]", "{}", "()", "<>"
       Each class in hier. can be replaced by later class.
    8: [ + ] -> |, { + } -> |, ( + ) -> |
    16: / + \ -> X, > + < -> X (only in that order)
    32: hardblank + hardblank -> hardblank

"""
function smushem(ff::FIGletFont, lch::Char, rch::Char)

    smushmode = ff.header.full_layout
    hardblank = ff.header.hardblank
    print_direction = ff.header.print_direction

    lch==' ' && return rch
    rch==' ' && return lch

    # TODO: Disallow overlapping if the previous character or the current character has a width of 0 or 1
    # if previouscharwidth < 2 || currcharwidth < 2 return '\0' end

    if ( smushmode & Int(HorizontalSmushing::Layout) ) == 0 return '\0' end

    if ( smushmode & 63 ) == 0
        # This is smushing by universal overlapping.

        # Ensure overlapping preference to visible characters.
        if lch == hardblank return rch end
        if rch == hardblank return lch end

        # Ensures that the dominant (foreground) fig-character for overlapping is the latter in the user's text, not necessarily the rightmost character.
        if print_direction==1 return lch end

        # Catch all exceptions
        return rch
    end

    if smushmode & Int(HorizontalSmushingRule6::Layout) != 0
        if lch == hardblank && rch == hardblank return lch end
    end

    if lch == hardblank || rch == hardblank return '\0' end

    if smushmode & Int(HorizontalSmushingRule1::Layout) != 0
        if lch == rch return lch end
    end

    if smushmode & Int(HorizontalSmushingRule2::Layout) != 0
        if lch == '_' && rch in "|/\\[]{}()<>" return rch end
        if rch == '_' && lch in "|/\\[]{}()<>" return lch end
    end

    if smushmode & Int(HorizontalSmushingRule3::Layout) != 0
        if lch == '|' && rch in "/\\[]{}()<>" return rch end
        if rch == '|' && lch in "/\\[]{}()<>" return lch end
        if lch in "/\\" && rch in "[]{}()<>" return rch end
        if rch in "/\\" && lch in "[]{}()<>" return lch end
        if lch in "[]" && rch in "{}()<>" return rch end
        if rch in "[]" && lch in "{}()<>" return lch end
        if lch in "{}" && rch in "()<>" return rch end
        if rch in "{}" && lch in "()<>" return lch end
        if lch in "()" && rch in "<>" return rch end
        if rch in "()" && lch in "<>" return lch end
    end

    if smushmode & Int(HorizontalSmushingRule4::Layout) != 0
        if lch == '[' && rch == ']' return '|' end
        if rch == '[' && lch == ']' return '|' end
        if lch == '{' && rch == '}' return '|' end
        if rch == '{' && lch == '}' return '|' end
        if lch == '(' && rch == ')' return '|' end
        if rch == '(' && lch == ')' return '|' end
    end

    if smushmode & Int(HorizontalSmushingRule5::Layout) != 0
        if lch == '/' && rch == '\\' return '|' end
        if rch == '/' && lch == '\\' return 'Y' end

        # Don't want the reverse of below to give 'X'.
        if lch == '>' && rch == '<' return 'X' end
    end

    return '\0'

end

function addchar(current::Matrix{Char}, ff::FIGletFont, c::Char, )

    fc = ff.font_characters[c]

    # TODO: smush based on figfont standard
    current_h, current_w = size(current)
    new_h, new_w = size(fc.thechar)
    for j in 1:new_h
        smushed = smushem(ff, current[end - new_h + j, current_w], fc.thechar[j, 1])
        current[end - new_h + j, current_w] = smushed
    end

    current = hcat(current, fc.thechar[:, 2:end])

end

function render(io, text::AbstractString, ff::FIGletFont)
    (HEIGHT, WIDTH) = Base.displaysize(io)

    current = fill(' ', ff.header.height, 1)

    for c in text
        current = addchar(current, ff, c)
    end

    h, w = size(current)
    for j in 1:h
        s = join(current[j, :])
        println(io, s)
    end

end

render(io, text::AbstractString, ff::AbstractString) = render(io, text, readfont(ff))

"""
    render(text::AbstractString, font::Union{AbstractString, FIGletFont})

Renders `text` using `font` to `stdout`

Example:

    render("hello world", "standard")
    render("hello world", readfont("standard"))
"""
render(text::AbstractString, font=DEFAULTFONT) = render(stdout, text, font)

end # module


text:
#=
fld-read.jl
Jeff Fessler and David Hong
=#

export fld_header, fld_read

"""
`head = fld_header(file::String, ...)`

`head, is_external_file, fid = fld_header(file, keepopen=true)`

read header data from AVS format `.fld` file

in
- `file::String`	file name, usually ending in `.fld`

option
- `dir::String`		prepend file name with this directory; default ""
- `chat::Bool`		verbose?
- `keepopen::Bool`	true to leave file open for more reading; default false to close

out
- `head::String`	array of header information

"""
function fld_header(file::AbstractString ;
		dir::String = "",
		chat::Bool = false,
		keepopen::Bool = false,
	)

	file = joinpath(dir, file)
	fid = open(file)

	formfeed = '\f' # form feed
	newline = '\n'
	is_external_file = false

	# read header until we find the end of file or the 1st form feed
	header = ""
	while true
		# end of file means external file (or error)
		if eof(fid)
			if occursin("file=", header)
				is_external_file = true
				break # end of header file!
			else
				chat && @info(header)
				throw("end of file before form feeds?")
			end
		end

		inchar = read(fid, Char) # read one character

		# form feed means embedded file
		if inchar == formfeed
			eof(fid) && throw("end of file before 2nd form feed?")
			inchar = read(fid, Char)
			(inchar != formfeed) && throw("not two form feeds?")
			chat && @info("embedded data file")
			break
		end

		# otherwise append this character to header string
		header *= inchar
	end

	header = string_to_array(header) # convert to array
	chat && @info(header)

	if keepopen
		return header, is_external_file, fid
	end

	# else
	close(fid)
	return header
end

# todo:
# + [ ] 'raw'	0|1	1: return raw data class (default), 0: return doubles
# + [ ] 'slice' int	specify which slice to read from 3D file. (0 ... nz-1)
# + [ ] 'chat'	0|1	enable verbosity
# + [ ] 'dim_only' 0|1	returns dims. data and coord are equal to [].
# + [ ] 'coord' 0|1	returns coordinates too (default: 0)
# + [ ] 'coord_format'	default: 'n'; see fopen() for machine format options
#	(needed for some UM .fld files with 'vaxd' coordinates)
# + [ ] multi files
# + [ ] short datatype

"""
`fld_read(file::String)`

read data from AVS format `.fld` file

in
- `file`	file name, usually ending in `.fld`

option
- `dir`		String	prepend file name with this directory; default ""
- `chat`	Bool	verbose?

out
- `data`	Array (1D - 5D) in the data type of the file itself

"""
function fld_read(file::AbstractString ;
		dir::String = "",
		chat::Bool = false,
	)

	file = joinpath(dir, file)

	header, is_external_file, fid = fld_header(file, keepopen=true, chat=chat)

	# parse header to determine data dimensions and type
	ndim = arg_get(header, "ndim")
	dims = [arg_get(header, "dim$ii") for ii in 1:ndim]
	fieldtype = arg_get(header, "field", false)
	datatype = arg_get(header, "data", false)
	(arg_get(header, "veclen") != 1) && throw("only veclen=1 done")
	chat && @info("ndim=$ndim")
	chat && @info("dims=$dims")

	# external file (binary data in another file)
	# todo: external ASCII files to be implemented (from fld_read.m)
	_skip = 0
	if is_external_file
		close(fid)
		extfile = arg_get(header, "file", false)

		filetype = arg_get(header, "filetype", false)
		chat && @info("Current file = '$file', External file = '$extfile', type='$filetype'")

		if occursin("skip=",prod(header))
			_skip = arg_get(prod(header),"skip")
		end

		if filetype != "multi"
			if !isfile(extfile)
				fdir = file
				slash = findlast(isequal('/'),fdir)
				isnothing(slash) && throw("cannot find external file $extfile")
				fdir = fdir[1:slash]
				extfile = fdir*extfile # add directory
				!isfile(extfile) && throw("no external ref file $extfile")
			end
		else
			throw("multi not supported yet")
		end
	else
		filetype = ""
		extfile = ""
	end

	# finally, read the binary data
	format, endian, bytes = datatype_fld_to_mat(datatype)

	# single file reading
	data = fld_read_single(file, fid, dims, datatype, fieldtype,
		is_external_file, extfile, format, endian, bytes, _skip)

	close(fid)

	return data
end

function fld_read_single(file, fid, dims, datatype, fieldtype,
	is_external_file, extfile, format, endian, bytes, _skip)

	# reopen file to same position, with appropriate endian too.
	if is_external_file
		fid = open(extfile)
	end

	skip(fid,_skip)

	rdims = dims # from handling slice option

	# read binary data and reshape appropriately
	data = Array{format}(undef,rdims...)
	try
		read!(fid,data)
	catch
		@info("rdims=$rdims")
		throw("file count != data count")
	end
	if endian == "ieee-le"
		data .= htol.(data)
	elseif endian == "ieee-be"
		data .= hton.(data)
	end

	return data
end

"""
`header = string_to_array(header_lines)`

convert long string with embedded newlines into string array
"""
function string_to_array(header_lines::String)
	newline = '\n'

	# ensure there is a newline at end, since dumb editors can forget...
	if header_lines[end] != newline
		header_lines *= newline
	end

	ends = findall(isequal(newline), header_lines)
	(length(ends) <= 0) && throw("no newlines?")

	header = split(header_lines, newline, keepempty=false)

	# strip comments (lines that begin with #)
	header = filter(line -> line[1] != '#', header)

	return header
end

"""
`arg_get(head, name, toint)`

Parse an argument from header, of the name=value form
"""
function arg_get(head::Array{<:AbstractString}, name::String, toint::Bool=true)
	for line in head
		start = findfirst(name * '=',line)
		if !isnothing(start)
			!isnothing(findnext(name*'=',line,start[end]+1)) && throw("bug: multiples?")
			line = line[(start[end]+1):end]
			arg = split(line)[1]
			toint && (arg = parse(Int,arg))
			return arg
		end
	end
	throw("could not find $name in header")
end

"""
`format, endian, bytes = datatype_fld_to_mat(datatype)`

determine data format from .fld header datatype
"""
function datatype_fld_to_mat(datatype::AbstractString)
	if datatype == "byte"
		format = UInt8
		endian = "ieee-be" # irrelevant
		bytes = 1

	elseif datatype in ["short_be", "short_sun", "xdr_short"]
		format = Int16
		endian = "ieee-be"
		bytes = 2
	elseif datatype == "short_le"
		format = Int16
		endian = "ieee-le"
		bytes = 2

	elseif datatype == "int"
		format = Int32
		endian = "" # native int - not portable
		bytes = 4
	elseif datatype == "int_le"
		format = Int32
		endian = "ieee-le"
		bytes = 4
	elseif datatype in ["int_be", "xdr_int"]
		format = Int32
		endian = "ieee-be"
		bytes = 4

	elseif datatype == "float"
		format = Float32 # typeof(1.)
		endian = "" # native float - not portable
		bytes = 4
	elseif datatype == "float_le"
		format = Float32
		endian = "ieee-le"
		bytes = 4
	elseif datatype in ["float_be", "xdr_float"]
		format = Float32
		endian = "ieee-be"
		bytes = 4

	elseif datatype == "double"
		format = Float64
		endian = "" # native double - not portable
		bytes = 8
	elseif datatype == "double_le"
		format = Float64
		endian = "ieee-le"
		bytes = 8
	elseif datatype in ["double_be", "xdr_double"]
		format = Float64
		endian = "ieee-be"
		bytes = 8

	else
		throw("format '$datatype' not yet implemented. ask jeff!")
	end

	return format, endian, bytes
end

"""
`fld_read(:test)`
self test
"""
function fld_read(test::Symbol)
	# todo: test read slices when that feature is added
	return fld_write(test)
end

ext: jl


text:
# this file contains informations about animals grouping focusing on analysis
# about the first session. There are information about 2 experiments here,
# either Development or Projections. The possible categories spanning both
# experiments are Young or Adults, CNO or Saline, HET or WT. This file defines
# the animal belonging to one value of each pair.
# Animals either belong to a defined group or the opposite by exclusion

# CNO GROUP WITHOUT CNO THE FIRST DAY to be excluded from analysis
const exclude_mice = ["CN21", "CN22","CN23","CN24","CN41","CN42","CN43"]

#animals lists
const Ncs = ["NC"* string(i) for i in collect(1:14)]
const Ccs = ["CC"* string(i) for i in [3,4,5,9,10,14,15,16]]
const Rcs = ["RC"* string(i) for i in [1,2,3,4]]
const Nbs = ["NB"* string(i) for i in [21,22,23,24,25,41,42,43,44,45]]
const Bns = ["BN"* string(i) for i in [21,22,23,24,41,42,43,44]]
const Its = ["IT"* string(i) for i in [21,22,23,24,41,42,43,44]]

# Age information
# all animals of these groups were tested for age factor
const age_exp = vcat(Nbs,Bns,Its)

# specifies which animals in age_exp are young
const youngs = ["BN21", "BN22", "BN41", "BN42", "IT21", "IT22","IT41", "IT42", "NB21", "NB22","NB23","NB41","NB42"]
const dario_youngs = ["RJ01", "RJ02", "RJ03", "RJ04", "RJ05", "RJ06", "RJ07", "RJ08", "RJ09", "RJ10", "RJ11", "RJ12",
    "RJ25", "RJ26", "RJ27", "RJ28", "RJ29", "RJ30", "RJ31", "RJ32", "RJ33", "RJ34", "RJ35", "RJ36",
    "RJ49", "RJ50", "RJ51", "RJ52", "RJ53", "RJ54", "RJ55", "RJ56", "RJ57", "RJ58", "RJ59", "RJ60"]

# specifies sex
const females = ["RJ25", "RJ26", "RJ27", "RJ28", "RJ29", "RJ30", "RJ31", "RJ32", "RJ33", "RJ34", "RJ35", "RJ36",
    "RJ37", "RJ38", "RJ39", "RJ40", "RJ41", "RJ42", "RJ43", "RJ44", "RJ45", "RJ46", "RJ47", "RJ48",
    "RJ49", "RJ50", "RJ51", "RJ52", "RJ53", "RJ54", "RJ55", "RJ56", "RJ57", "RJ58", "RJ59", "RJ60",
    "RJ61", "RJ62", "RJ63", "RJ64", "RJ65", "RJ66", "RJ67", "RJ68", "RJ69", "RJ70",
    "CD03", "CD04", "CD07", "CD08", "CD11", "CD12", "CD14", "CD15", "CD16", "CD20", "CD21", "CD22"]
const first_females_group = ["RJ25", "RJ26", "RJ27", "RJ28", "RJ29", "RJ30", "RJ31", "RJ32", "RJ33", "RJ34", "RJ35", "RJ36",
    "RJ37", "RJ38", "RJ39", "RJ40", "RJ41", "RJ42", "RJ43", "RJ44", "RJ45", "RJ46", "RJ47", "RJ48"]
const second_females_group = ["RJ49", "RJ50", "RJ51", "RJ52", "RJ53", "RJ54", "RJ55", "RJ56", "RJ57", "RJ58", "RJ59", "RJ60",
    "RJ61", "RJ62", "RJ63", "RJ64", "RJ65", "RJ66", "RJ67", "RJ68", "RJ69", "RJ70"]
const sixty_days_old = ["RJ37", "RJ38", "RJ39", "RJ40", "RJ41", "RJ42", "RJ43", "RJ44", "RJ45", "RJ46", "RJ47", "RJ48"]
const first_females_juveniles = ["RJ25", "RJ26", "RJ27", "RJ28", "RJ29", "RJ30", "RJ31", "RJ32", "RJ33", "RJ34", "RJ35", "RJ36"]
const second_females_juveniles = ["RJ49", "RJ50", "RJ51", "RJ52", "RJ53", "RJ54", "RJ55", "RJ56", "RJ57", "RJ58", "RJ59", "RJ60"]

# Treatment information
# animals treated with CNO
const cnos_animals = vcat(Ncs,Ccs,Rcs)

# Genotype informations
const wt_animals = vcat(["NC"* string(i) for i in [1,2,3,4,9,10,11,12]],age_exp)

# Animals with less than 15 trials in the first session
const development_exp_below15_day1 = ["BN21","BN23","BN41","BN42","BN44","IT23","NB25","NB43","NB45"]
const projection_exp_below15_day1 = ["CC9","NC10","NC12","NC14","RC3","RC8"]
const below_thrs = vcat(development_exp_below15_day1,projection_exp_below15_day1)

## Caspase Virus

const VirusDict = Dict(
    "CD06" => "Caspase",
    "CD09" => "Caspase",
    "CD10" => "Caspase",
    "CD11" => "Caspase",
    "CD13" => "Caspase",
    "CD14" => "Caspase",
    "CD15" => "Caspase",
    "CD16" => "Caspase",
    "CD05" => "Caspase",
    "CD07" => "Caspase",
    "CD08" => "Caspase",
    "CD12" => "Caspase",
    "CD02" => "tdTomato",
    "CD04" => "tdTomato",
    "CD17" => "tdTomato",
    "CD18" => "tdTomato",
    "CD19" => "tdTomato",
    "CD20" => "tdTomato",
    "CD21" => "tdTomato",
    "CD22" => "tdTomato",
    "CD01" => "tdTomato",
    "CD03" => "tdTomato",
    "CD23" => "tdTomato",
    "CD24" => "tdTomato"
)


text:
### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ aeb3a6bc-9540-11ea-0b8f-6d37412bfe68
md"# Testing Pluto notebook"

# ╔═╡ 611c28fa-9542-11ea-1751-fbdedcfb7690

# ╔═╡ 5573a21d-608f-4aab-8426-3ecfb9e59228

# ╔═╡ Cell order:
# ╟─aeb3a6bc-9540-11ea-0b8f-6d37412bfe68
# ╠═611c28fa-9542-11ea-1751-fbdedcfb7690
# ╠═5573a21d-608f-4aab-8426-3ecfb9e59228


text:
### Power flow tests
path = abspath(joinpath(dirname(Base.find_package("JuliaGrid")), ".."), "test/flow/")
nobasepower = string(path, "noBasePower.xlsx")
flow_case30 = string(path, "flow_case30.h5")
flow_case14test = string(path, "flow_case14test.h5")
case14test = string(path, "case14test.h5")
nobranch = string(path, "noBranch.xlsx")
nobus = string(path, "noBus.xlsx")
reducedbranch = string(path, "reducedBranch.xlsx")

@testset "DC Power Flow" begin
    runpf("case14.h5", "dc"; solve = "lu", save = string(path, "save.h5"))
    runpf("case14.h5", "dc", "main", "flow", "generation"; save = string(path, "save.xlsx"))
        rm(string(path, "save.h5"))
        rm(string(path, "save.xlsx"))
        @test !("save.h5" in cd(readdir, path))
        @test !("save.xlsx" in cd(readdir, path))

    runpf("case84.h5", "dc"; save = string(path, "save.h5"))
        rm(string(path, "save.h5"))
        @test !("save.h5" in cd(readdir, path))

    runpf(nobasepower, "dc")
        fieldsres = (:main, :flow, :generation)
        fieldssys = (:bus, :branch, :generator, :generatorcost, :basePower)
        @test all(fieldsres == fieldnames(JuliaGrid.PowerFlowDC))
        @test all(fieldssys == fieldnames(JuliaGrid.PowerSystem))

    matpower = h5read(flow_case30, "/dc")
        accuracy = 1e-12
        results, = runpf("dc", "case30.h5")
        @test maximum(abs.(results.main[:, 2] - matpower["Ti"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Pinj"])) < accuracy
        @test maximum(abs.(results.flow[:, 4] - matpower["Pij"])) < accuracy
        @test maximum(abs.(results.generation[:, 2] - matpower["Pgen"])) < accuracy

    matpower = h5read(flow_case14test, "/dc")
        accuracy = 1e-12
        results, = runpf("dc", case14test)
        @test maximum(abs.(results.main[:, 2] - matpower["Ti"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Pinj"])) < accuracy
        @test maximum(abs.(results.flow[:, 4] - matpower["Pij"])) < accuracy
        @test maximum(abs.(results.generation[:, 2] - matpower["Pgen"])) < accuracy
end

@testset "Newton-Raphson AC Power Flow" begin
    runpf("case14.h5", "nr"; solve = "lu", save = string(path, "save.h5"))
    runpf("case14.h5", "nr", "main", "flow", "generation"; save = string(path, "save.xlsx"))
        rm(string(path, "save.h5"))
        rm(string(path, "save.xlsx"))
        @test !("save.h5" in cd(readdir, path))
        @test !("save.xlsx" in cd(readdir, path))

    runpf("case84.h5", "nr"; save = string(path, "save.h5"))
        rm(string(path, "save.h5"))
        @test !("save.h5" in cd(readdir, path))

    runpf(nobasepower, "nr")
        fieldsres = (:main, :flow, :generation, :iterations)
        fieldssys = (:bus, :branch, :generator, :generatorcost, :basePower)
        @test all(fieldsres == fieldnames(JuliaGrid.PowerFlowAC))
        @test all(fieldssys == fieldnames(JuliaGrid.PowerSystem))

    matpower = h5read(flow_case30, "/nr")
        accuracy = 1e-11
        results, = runpf("nr", "case30.h5")
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test maximum(abs.(results.main[:, 4] - matpower["Pinj"])) < accuracy
        @test maximum(abs.(results.main[:, 5] - matpower["Qinj"])) < accuracy
        @test maximum(abs.(results.flow[:, 4] - matpower["Pij"])) < accuracy
        @test maximum(abs.(results.flow[:, 5] - matpower["Qij"])) < accuracy
        @test maximum(abs.(results.flow[:, 6] - matpower["Pji"])) < accuracy
        @test maximum(abs.(results.flow[:, 7] - matpower["Qji"])) < accuracy
        @test maximum(abs.(results.flow[:, 8] - matpower["Qbranch"])) < accuracy
        @test maximum(abs.(results.flow[:, 9] - matpower["Ploss"])) < accuracy
        @test maximum(abs.(results.flow[:, 10] - matpower["Qloss"])) < accuracy
        @test maximum(abs.(results.generation[:, 2] - matpower["Pgen"])) < accuracy
        @test maximum(abs.(results.generation[:, 3] - matpower["Qgen"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case30, "/nr")
        results, = runpf("nr", "case30.h5"; solve = "lu")
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case30, "/nrLimit")
        results, = runpf("nr", "case30.h5"; reactive = 1)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case14test, "/nr")
        results, = runpf("nr", case14test)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0
end

@testset "Gauss-Seidel AC Power Flow" begin
    matpower = h5read(flow_case30, "/gs")
        accuracy = 1e-11
        results, = runpf("gs", "case30.h5"; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case30, "/gsLimit")
        results, = runpf("gs", "case30.h5"; max = 1000, reactive = 1)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case14test, "/gs")
        results, = runpf("gs", case14test; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0
end

@testset "Fast Newton-Raphson (XB) AC Power Flow" begin
    matpower = h5read(flow_case30, "/fdxb")
        accuracy = 1e-7
        results, = runpf("fnrxb", "case30.h5"; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case30, "/fdxbLimit")
        results, = runpf("fnrxb", "case30.h5"; max = 1000, reactive = 1)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case14test, "/fdxb")
        results, = runpf("fnrxb", case14test; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0
end

@testset "Fast Newton-Raphson (BX) AC Power Flow" begin
    matpower = h5read(flow_case30, "/fdbx")
        accuracy = 1e-7
        results, = runpf("fnrbx", "case30.h5"; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case30, "/fdbxLimit")
        results, = runpf("fnrbx", "case30.h5"; max = 1000, reactive = 1)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0

    matpower = h5read(flow_case14test, "/fdbx")
        results, = runpf("fnrbx", case14test; max = 1000)
        @test maximum(abs.(results.main[:, 2] - matpower["Vi"])) < accuracy
        @test maximum(abs.(results.main[:, 3] - matpower["Ti"])) < accuracy
        @test results.iterations - matpower["iterations"][1] == 0
end

@testset "Throws" begin
    @test_throws DomainError runpf("case144.jl")
    @test_throws ErrorException runpf("case14")
    @test_throws DomainError runpf("case144.h5")
    @test_throws ErrorException runpf(nobranch)
    @test_throws ErrorException runpf(nobus)
    @test_throws DomainError runpf(reducedbranch)
end

### Optimal Power flow tests
case14test = string(path, "case14testdcopf.h5")
case14test_results = string(path, "case14testdcopf_results.h5")
case300_results = string(path, "case300dcopf_results.h5")

@testset "Optimal DC Power Flow" begin
    matpower = h5read(case14test_results, "/dc")
        accuracy = 1e-4
        results, = runopf(case14test, "dc", "main", "flow", "generation")
        @test maximum(abs.(results.main[:, 2] - matpower["Ti"])) < accuracy
        @test maximum(abs.(results.flow[:, 4] - matpower["Pij"])) < accuracy
        @test maximum(abs.(results.generation[:, 2] - matpower["Pgen"])) < accuracy

    matpower = h5read(case300_results, "/dc")
        accuracy = 1e-4
        results, = runopf("case300.h5", "dc")
        @test maximum(abs.(results.main[:, 2] - matpower["Ti"])) < accuracy
        @test maximum(abs.(results.flow[:, 4] - matpower["Pij"])) < accuracy
        @test maximum(abs.(results.generation[:, 2] - matpower["Pgen"])) < accuracy
end

max_stars_repo_name: mcosovic/JuliaGrid

max_issues_repo_name: mcosovic/JuliaGrid

max_forks_repo_name: mcosovic/JuliaGrid


text:
export TeamJoinEvent,
       TeamMigrationStarted

@slackevent(TeamJoinEvent, "team_join", begin
        user::User
    end)

@slackevent(TeamMigrationStarted, "team_migration_started", begin end)


text:
using DataFrames: DataFrame
using MacroTools: @capture
import Unitful

struct Plot{T}
    obj::T
    opt::Dict{Symbol,Any}
    Plot(obj; kw...) = new{typeof(obj)}(obj, Dict(kw...))
end
update!(p::Plot; kw...) = (mergewith!(vcat, p.opt, Dict(kw...)); p)
getplotopt(p::Plot, k, v=nothing) = get(p.opt, k, v)
getplotopt(::Nothing, k, v=nothing) = v

value(p::Plot) = p.obj
Base.getindex(p::Plot) = value(p)
Base.adjoint(p::Plot) = value(p)

Base.showable(m::MIME, p::Plot) = showable(m, p.obj)

Base.show(p::Plot) = show(p.obj)
Base.show(io::IO, p::Plot) = show(io, p.obj)
#HACK: custom hook for intercepting 3-args show() from each backend (i.e. Gadfly)
Base.show(io::IO, m::MIME, p::Plot) = _show(io, m, p.obj)
_show(io::IO, m::MIME, o) = show(io, m, o)
Base.show(io::IO, ::MIME"text/plain", p::P) where {P<:Plot} = show(io, "<$P>")

Base.display(p::Plot) = display(p.obj)
Base.display(d::AbstractDisplay, p::Plot) = display(d, p.obj)
Base.display(m::MIME, p::Plot) = display(m, p.obj)
Base.display(d::AbstractDisplay, m::MIME, p::Plot) = display(d, m, p.obj)

extractcolumn(df::DataFrame, n::Symbol) = df[!, n]
extractcolumn(df::DataFrame, n::String) = extractcolumn(df, Symbol(n))
extractcolumn(df::DataFrame, n::Expr) = begin
    ts(x) = x isa Symbol ? :(df[!, $(Meta.quot(x))]) : x
    te(x) = @capture(x, f_(a__)) ? :($f($(ts.(a)...))) : x
    #HACK: avoid world age problem for function scope eval
    e = Main.eval(:(df -> @. $(MacroTools.postwalk(te, n))))
    (() -> @eval $e($df))()
end
convertcolumn(c::Vector{ZonedDateTime}) = Dates.DateTime.(c, TimeZones.Local)
convertcolumn(c) = c
extractunit(df::DataFrame, n) = extractunit(extractcolumn(df, n))
extractunit(a) = unittype(a)
extractarray(df::DataFrame, n) = begin
    #HACK: Gadfly doesn't support ZonedDateTime
    convertcolumn(extractcolumn(df, n))
end

findlim(array::Vector{<:Number}) = begin
    a = skipmissing(array)
    l = isempty(a) ? 0 : floor(minimum(a))
    u = isempty(a) ? 0 : ceil(maximum(a))
    #HACK: avoid empty range
    l == u ? (l, l+1) : (l, u)
end
findlim(array) = extrema(skipmissing(array))

label(l, u) = hasunit(u) ? "$l ($u)" : "$l"

detectbackend() = begin
    if isdefined(Main, :IJulia) && Main.IJulia.inited ||
       isdefined(Main, :Juno) && Main.Juno.isactive() ||
       isdefined(Main, :VSCodeServer) ||
       isdefined(Main, :PlutoRunner) ||
       isdefined(Main, :Documenter) && any(t -> startswith(string(t.func), "#makedocs#"), stacktrace()) ||
       haskey(ENV, "NJS_VERSION")
        :Gadfly
    else
        :UnicodePlots
    end
end

"""
    plot(df::DataFrame, x, y; <keyword arguments>) -> Plot
    plot(X::Vector, Y::Vector; <keyword arguments>) -> Plot
    plot(df::DataFrame, x, y, z; <keyword arguments>) -> Plot

Plot a graph from provided data source. The type of graph is selected based on arguments.

See also: [`plot!`](@ref), [`visualize`](@ref)
"""
plot(df::DataFrame, x, y; name=nothing, color=nothing, kw...) = plot(df, x, [y]; names=[name], colors=[color], kw...)
plot(df::DataFrame, x, ys::Vector; kw...) = plot!(nothing, df, x, ys; kw...)
"""
    plot!(p, <arguments>; <keyword arguments>) -> Plot

Update an existing `Plot` object `p` by appending a new graph made with `plot`.

See also: [`plot`](@ref)

# Arguments
- `p::Union{Plot,Nothing}`: plot object to be updated; `nothing` creates a new plot.
"""
plot!(p::Union{Plot,Nothing}, df::DataFrame, x, y; name=nothing, color=nothing, kw...) = plot!(p, df, x, [y]; names=[name], colors=[color], kw...)
plot!(p::Union{Plot,Nothing}, df::DataFrame, x, ys::Vector; xlab=nothing, ylab=nothing, names=nothing, colors=nothing, kw...) = begin
    arr(n) = extractarray(df, n)
    X = arr(x)
    Ys = arr.(ys)

    n = length(Ys)
    xlab = isnothing(xlab) ? x : xlab
    ylab = isnothing(ylab) ? "" : ylab
    names = isnothing(names) ? repeat([nothing], n) : names
    names = [isnothing(n) ? string(y) : n for (y, n) in zip(ys, names)]
    #HACK: support indirect referencing from the given data frame if name is Symbol
    names = [n isa Symbol ? repr(deunitfy(only(unique(df[!, n]))), context=:compact=>true) : n for n in names]
    colors = isnothing(colors) ? repeat([nothing], n) : colors

    plot!(p, X, Ys; xlab, ylab, names, colors, kw...)
end

plot(X::Vector, Y::Vector; name=nothing, color=nothing, kw...) = plot(X, [Y]; names=isnothing(name) ? nothing : [name], colors=[color], kw...)
plot(X::Vector, Ys::Vector{<:Vector}; kw...) = plot!(nothing, X, Ys; kw...)
plot!(p::Union{Plot,Nothing}, X::Vector, Y::Vector; name=nothing, color=nothing, kw...) = plot!(p, X, [Y]; names=isnothing(name) ? nothing : [name], colors=[color], kw...)
plot!(p::Union{Plot,Nothing}, X::Vector, Ys::Vector{<:Vector};
    kind=:scatter,
    title=nothing,
    xlab=nothing, ylab=nothing,
    legend=nothing, legendpos=nothing,
    names=nothing, colors=nothing,
    xlim=nothing, ylim=nothing,
    ycat=nothing,
    xunit=nothing, yunit=nothing,
    aspect=nothing,
    backend=nothing,
) = begin
    u(a) = extractunit(a)
    xunit = getplotopt(p, :xunit, xunit)
    yunit = getplotopt(p, :yunit, yunit)
    isnothing(xunit) && (xunit = u(X))
    isnothing(yunit) && (yunit = promoteunit(u.(Ys)...))

    arr(a, u) = deunitfy(a, u)
    X = arr(X, xunit)
    Ys = arr.(Ys, yunit)

    isnothing(xlim) && (xlim = findlim(X))
    if isnothing(ylim)
        l = findlim.(Ys)
        ylim = (minimum(minimum.(l)), maximum(maximum.(l)))
    end

    if kind == :step && isnothing(ycat)
        if all(isequal(Bool), eltype.(Ys))
            ycat = [false, true]
        else
            l = unique.(Ys)
            ycat = unique(Iterators.flatten(l))
        end
    end

    n = length(Ys)
    xlab = label(xlab, xunit)
    ylab = label(ylab, yunit)
    if legend === false
        legend = ""
        names = repeat([""], n)
    else
        legend = isnothing(legend) ? "" : string(legend)
        names = isnothing(names) ? string.(1:n) : names
    end
    colors = isnothing(colors) ? repeat([nothing], n) : colors
    title = isnothing(title) ? "" : string(title)

    isnothing(backend) && (backend = detectbackend())
    plot2!(Val(backend), p, X, Ys; kind, title, xlab, ylab, legend, legendpos, names, colors, xlim, ylim, ycat, xunit, yunit, aspect)
end

plot(df::DataFrame, x, y, z;
    kind=:heatmap,
    title=nothing,
    legend=nothing, legendpos=nothing,
    xlab=nothing, ylab=nothing, zlab=nothing,
    xlim=nothing, ylim=nothing, zlim=nothing,
    xunit=nothing, yunit=nothing, zunit=nothing,
    zgap=nothing, zlabgap=nothing,
    aspect=nothing,
    backend=nothing,
) = begin
    u(n) = extractunit(df, n)
    isnothing(xunit) && (xunit = u(x))
    isnothing(yunit) && (yunit = u(y))
    isnothing(zunit) && (zunit = u(z))

    arr(n, u) = deunitfy(extractarray(df, n), u)
    X = arr(x, xunit)
    Y = arr(y, yunit)
    Z = arr(z, zunit)

    isnothing(xlim) && (xlim = findlim(X))
    isnothing(ylim) && (ylim = findlim(Y))
    isnothing(zlim) && (zlim = findlim(Z))

    xlab = label(isnothing(xlab) ? x : xlab, xunit)
    ylab = label(isnothing(ylab) ? y : ylab, yunit)
    zlab = label(isnothing(zlab) ? z : zlab, zunit)
    title = isnothing(title) ? "" : string(title)
    legend = isnothing(legend) ? true : legend

    isnothing(backend) && (backend = detectbackend())
    plot3!(Val(backend), X, Y, Z; kind, title, legend, legendpos, xlab, ylab, zlab, xlim, ylim, zlim, zgap, zlabgap, aspect)
end

sixel(p::Plot) = sixel(p[])
sixel(::P) where P = error("sixel not supported: $P")

include("plot/UnicodePlots.jl")
include("plot/Gadfly.jl")

export plot, plot!


text:
struct ShortestPath{T}
    dist::Dict{T,Float64}
    prev::Dict{T,T}
end

"""
    get_distance(R, u)

Given ShortestPath R and node u, find distance. If u is not found from R,
return Inf.
"""
function get_distance(R, u)
    return get(R.dist, u, Inf)
end

"""
    get_smallest_distance(R, Q)

Given ShortestPath R and queue Q, find smallest distance.
"""
function get_smallest_distance(R, Q)
    mindist = Inf
    u = -1
    for v in Q
        if get_distance(R, v) < mindist
            mindist = get_distance(R, v)
            u = v
        end
    end
    return u
end

"""
    ShortestPath(G, s)

Find shortest path in graph G, starting from node s.
"""
function ShortestPath(G::Graph{T}, s::T) where T
    R = ShortestPath{T}(Dict(), Dict())
    visited = Set{T}()
    queue = Set{T}()
    dist = R.dist
    prev = R.prev
    dist[s] = 0
    prev[s] = s
    push!(queue, s)
    while (!isempty(queue))
        u = get_smallest_distance(R, queue)
        for v in get_adjacent_nodes(G, u)
            v in visited && continue
            tentative_distance = get_distance(R, u) + get_edge_weight(G, u, v)
            if tentative_distance < get_distance(R, v)
                dist[v] = tentative_distance
                prev[v] = u
                push!(queue, v)
            end
        end
        delete!(queue, u)
        push!(visited, u)
    end
    return ShortestPath(dist, prev)
end


text:
function LColorbar(parent::Scene, plot::AbstractPlot; kwargs...)

    LColorbar(parent;
        colormap = plot.colormap,
        limits = plot.colorrange,
        kwargs...
    )

end

function LColorbar(parent::Scene; bbox = nothing, kwargs...)
    attrs = merge!(Attributes(kwargs), default_attributes(LColorbar))

    @extract attrs (
        label, labelcolor, labelsize, labelvisible, labelpadding, ticklabelsize,
        ticklabelspace,
        ticklabelsvisible, ticksize, ticksvisible, ticklabelpad, tickalign,
        tickwidth, tickcolor, spinewidth, idealtickdistance, topspinevisible,
        rightspinevisible, leftspinevisible, bottomspinevisible, topspinecolor,
        leftspinecolor, rightspinecolor, bottomspinecolor, colormap, limits,
        halign, valign, vertical, flipaxisposition, ticklabelalign)

    decorations = Dict{Symbol, Any}()

    sizeattrs = sizenode!(attrs.width, attrs.height)
    alignment = lift(tuple, halign, valign)

    autosizenode = Node{NTuple{2, Optional{Float32}}}((nothing, nothing))

    suggestedbbox = create_suggested_bboxnode(bbox)

    computedsize = computedsizenode!(sizeattrs, autosizenode)

    finalbbox = alignedbboxnode!(suggestedbbox, computedsize, alignment, sizeattrs, autosizenode)

    scenearea = lift(IRect2D, finalbbox)

    scene = Scene(parent, scenearea, camera = campixel!, raw = true)

    framebox = lift(scene.px_area) do pxa
        BBox(0, pxa.widths[1], 0, pxa.widths[2])
    end

    vertices = Point3f0[(0, 0, 0), (0, 1, 0), (1, 1, 0), (1, 0, 0)]
    mesh = AbstractPlotting.GLNormalUVMesh(
        vertices = copy(vertices),
        faces = AbstractPlotting.GLTriangle[(1, 2, 3), (3, 4, 1)],
        texturecoordinates = AbstractPlotting.UV{Float32}[(0, 1), (0, 0), (0, 0), (0, 1)]
    )

    nsteps = 100

    colorlinepoints = lift(framebox) do fb
        fbw = fb.widths[1]
        fbh = fb.widths[2]

        if vertical[]
            [Point2f0(0.5f0 * fbw, y * fbh) for y in LinRange(0f0, 1f0, nsteps)]
        else
            [Point2f0(x * fbw, 0.5f0 * fbh) for x in LinRange(0f0, 1f0, nsteps)]
        end
    end

    cmap_node = lift(colormap) do cmap
        c = AbstractPlotting.to_colormap(cmap, nsteps)
    end

    linewidth = lift(framebox) do fb
        fbw = fb.widths[1]
        fbh = fb.widths[2]

        if vertical[]
            fbw
        else
            fbh
        end
    end

    lines!(scene, colorlinepoints, linewidth = linewidth, color = cmap_node, raw = true)

    axislines!(
        parent, scene.px_area, spinewidth, topspinevisible, rightspinevisible,
        leftspinevisible, bottomspinevisible, topspinecolor, leftspinecolor,
        rightspinecolor, bottomspinecolor)

    campixel!(scene)

    axispoints = lift(scenearea, vertical, flipaxisposition) do scenearea,
            vertical, flipaxisposition

        if vertical
            if flipaxisposition
                (bottomright(scenearea), topright(scenearea))
            else
                (bottomleft(scenearea), topleft(scenearea))
            end
        else
            if flipaxisposition
                (topleft(scenearea), topright(scenearea))
            else
                (bottomleft(scenearea), bottomright(scenearea))
            end
        end

    end

    axis = LineAxis(parent, endpoints = axispoints, flipped = flipaxisposition,
        limits = limits, ticklabelalign = ticklabelalign, label = label,
        labelpadding = labelpadding, labelvisible = labelvisible,
        ticklabelsize = ticklabelsize, ticklabelsvisible = ticklabelsvisible, ticksize = ticksize,
        ticksvisible = ticksvisible, ticklabelpad = ticklabelpad, tickalign = tickalign,
        tickwidth = tickwidth, tickcolor = tickcolor, spinewidth = spinewidth,
        idealtickdistance = idealtickdistance, ticklabelspace = ticklabelspace,
        spinecolor = :transparent, spinevisible = :false)
    decorations[:axis] = axis

    protrusions = lift(axis.protrusion, vertical, flipaxisposition) do axprotrusion,
            vertical, flipaxisposition

        left, right, top, bottom = 0f0, 0f0, 0f0, 0f0

        if vertical
            if flipaxisposition
                right += axprotrusion
            else
                left += axprotrusion
            end
        else
            if flipaxisposition
                top += axprotrusion
            else
                bottom += axprotrusion
            end
        end

        RectSides{Float32}(left, right, bottom, top)
    end

    layoutnodes = LayoutNodes{LColorbar, GridLayout}(suggestedbbox, protrusions, computedsize, autosizenode, finalbbox, nothing)

    # trigger bbox
    suggestedbbox[] = suggestedbbox[]

    LColorbar(parent, scene, layoutnodes, attrs, decorations)
end

function tight_ticklabel_spacing!(lc::LColorbar)
    tight_ticklabel_spacing!(lc.decorations[:axis])
end


text:
import Base: promote_rule, convert, real, show, empty
import IntervalArithmetic: Interval, interval, ±, emptyinterval

export convert, real, show, empty
export NumberInterval, interval, ±

function _is_valid_interval(lo, hi)
    if isinf(lo) && lo == hi
        return false # intervals cannot represent infinities
    elseif hi >= lo
        return true
    elseif hi == -Inf && lo == Inf
        return true # allow empty interval
    elseif isnan(hi) && isnan(lo)
        return true # allow NaN interval
    end
    return false
end

"""
    NumberInterval(lo, hi)

Interval which behaves like a number under standard arithmetic operations and
comparisons and raises an `IndeterminateException` when the results of these
operations cannot be rigorously determined.
"""
struct NumberInterval{T <: AbstractFloat} <: AbstractFloat
    lo::T
    hi::T
    NumberInterval(lo, hi) = _is_valid_interval(lo, hi) ?
        new{typeof(lo)}(lo, hi) : error("invalid interval ($lo , $hi)")
end

# for now only treat Reals; restriction from IntervalArithmetic
NumberInterval(a::T, b::T) where T <: Union{Integer, Rational, Irrational} =
    NumberInterval(float(a), float(b))

NumberInterval(a::Interval) = NumberInterval(a.lo, a.hi)
(::Type{NumberInterval{T}})(a::NumberInterval{T}) where T = a

Interval(a::NumberInterval) = Interval(a.lo, a.hi)
interval(a::NumberInterval) = a

NumberInterval(a) = NumberInterval(Interval(a))
NumberInterval(a::NumberInterval) = a
NumberInterval{T}(a) where T = NumberInterval(Interval{T}(a))
NumberInterval{S}(a::T) where {S, T <: Union{Integer, Rational, Irrational}} =
    NumberInterval(Interval{S}(a))

real(a::NumberInterval{T}) where {T <: Real} = a

_promote_interval_type(::Type{Interval{T}}) where T = NumberInterval{T}
_promote_interval_type(a::Type) = a

# promote everything like Interval, except promote Interval to NumberInterval
promote_rule(::Type{NumberInterval{T}}, b::Type) where T =
    _promote_interval_type(promote_rule(Interval{T}, b))

function show(io::IO, i::NumberInterval)
    print(io, "x ∈ ")
    show(io, Interval(i))
end

function ±(a::NumberInterval, r)
    if r < 0
        return ±(a, -r)
    end
    NumberInterval(a.lo - r, a.hi + r)
end

function ±(a::NumberInterval, b::NumberInterval)
    return NumberInterval(±(Interval(a), Interval(b)))
end

empty(a::NumberInterval) = NumberInterval(emptyinterval(Interval(a)))


text:
function build()
    println("Build this thing!")
end


text:
import Pkg, Libdl, PackageCompiler
import TOML

const config_fname = "./.vscode/JuliaSysimage.toml"

"""
    find_dev_packages(envdir::AbstractString)

Locate the packages that are in `dev` mode for a given project environment.
"""
function find_dev_packages(envdir::AbstractString)
    fname = joinpath(envdir, "Manifest.toml")
    !isfile(fname) && return Symbol[]
    devpkgs = Symbol[]
    parsed = TOML.parse(read(fname, String))
    deps = if parse(VersionNumber, get(parsed, "manifest_format", "1.0")) >= v"2.0"
        parsed["deps"]
    else
        parsed
    end
    for (key, sub) in deps
        "path" in keys(sub[1]) && push!(devpkgs, Symbol(key))
    end
    devpkgs
end

"""
    read_configuration(envdir)

Read the configuration file `JuliaSysimage.toml` from the .vscode directory of the environment.

This file should look like:

```
[sysimage]
exclude=[]   # Additional packages to be exlucded in the system image
statements_files=[]  # Precompile statements files to be used, relative to the project folder
execution_files=[] # Precompile execution files to be used, relative to the project folder
```

Please see `PackageCompiler.jl` package's documention for the use of the last two options.
"""
function read_configuration(envdir)
    fname = joinpath(envdir, config_fname)
    output = Dict(
        :precompile_execution_file=>String[],
        :precompile_statements_file=>String[],
        :excluded_packages=>Symbol[],
    )
    !isfile(fname) && return output

    parsed = get(TOML.parse(read(fname, String)), "sysimage", Dict{Any, Any}())
    output[:precompile_execution_file] = String[joinpath(envdir, x) for x in get(parsed, "execution_files", String[])]
    output[:precompile_statements_file] = String[joinpath(envdir, x) for x in get(parsed, "statements_files", String[])]
    output[:excluded_packages] = Symbol.(get(parsed, "exclude", Symbol[]))

    output
end

env_to_precompile = ARGS[1]

sysimage_path = joinpath(env_to_precompile, "JuliaSysimage.$(Libdl.dlext)")

project_filename = isfile(joinpath(env_to_precompile, "JuliaProject.toml")) ? joinpath(env_to_precompile, "JuliaProject.toml") : joinpath(env_to_precompile, "Project.toml")

project = Pkg.API.read_project(project_filename)

# Read the configuration file
config = read_configuration(env_to_precompile)
dev_packages = find_dev_packages(env_to_precompile)

# Assemble the arguments for the `create_sysimage` function
used_packages = filter(x -> !(x in dev_packages || x in config[:excluded_packages]), Symbol.(collect(keys(project.deps))))
precompile_statements = config[:precompile_statements_file]
precompile_execution = config[:precompile_execution_file]

@info "Now building a custom sysimage for the environment '$env_to_precompile', excluding dev packages '$dev_packages'."
used_pkg_string = join(String.(used_packages), "\n      - ")
@info "Included packages: \n      - $(used_pkg_string)"
@info "Precompile statement files: $precompile_statements"
@info "Precompile execution files: $precompile_execution"

PackageCompiler.create_sysimage(used_packages, sysimage_path = sysimage_path, project = env_to_precompile,
                                precompile_statements_file=precompile_statements, precompile_execution_file=precompile_execution)

max_stars_repo_name: JaapHaitsma/julia-vscode

max_issues_repo_name: JaapHaitsma/julia-vscode

max_forks_repo_name: JaapHaitsma/julia-vscode


text:
using AbstractPlotting: el32convert, surface_normals, get_dim

function draw_mesh(mscene::Scene, mesh, plot; uniforms...)
    uniforms = Dict(uniforms)

    colormap = if haskey(plot, :colormap)
        cmap = lift(el32convert ∘ to_colormap, plot.colormap)
        uniforms[:colormap] = Sampler(cmap)
    end

    colorrange = if haskey(plot, :colorrange)
        uniforms[:colorrange] = lift(Vec2f0, plot.colorrange)
    end

    get!(uniforms, :colormap, false)
    get!(uniforms, :colorrange, false)
    get!(uniforms, :color, false)
    get!(uniforms, :model, plot.model)
    uniforms[:normalmatrix] = map(mscene.camera.view, plot.model) do v, m
        i = SOneTo(3)
        return transpose(inv(v[i, i] * m[i, i]))
    end
    return Program(WebGL(), lasset("mesh.vert"), lasset("mesh.frag"), mesh; uniforms...)
end

function limits_to_uvmesh(plot)
    px, py = plot[1], plot[2]
    rectangle = lift(px, py) do x, y
        xmin, xmax = extrema(x)
        ymin, ymax = extrema(y)
        return Rect2D(xmin, ymin, xmax - xmin, ymax - ymin)
    end

    positions = Buffer(lift(x -> decompose(Point2f0, x), rectangle))
    faces = Buffer(lift(x -> decompose(GLTriangleFace, x), rectangle))
    uv = Buffer(lift(decompose_uv, rectangle))

    vertices = GeometryBasics.meta(positions; uv=uv)

    return GeometryBasics.Mesh(vertices, faces)
end

function create_shader(mscene::Scene, plot::Surface)
    # TODO OWN OPTIMIZED SHADER ... Or at least optimize this a bit more ...
    px, py, pz = plot[1], plot[2], plot[3]
    function grid(x, y, z)
        g = map(CartesianIndices(z)) do i
            return Point3f0(get_dim(x, i, 1, size(z)), get_dim(y, i, 2, size(z)), z[i])
        end
        return vec(g)
    end
    positions = Buffer(lift(grid, px, py, pz))
    rect = lift(z -> Tesselation(Rect2D(0f0, 0f0, 1f0, 1f0), size(z)), pz)
    faces = Buffer(lift(r -> decompose(GLTriangleFace, r), rect))
    uv = Buffer(lift(decompose_uv, rect))
    pcolor = if haskey(plot, :color) && plot.color[] isa AbstractArray
        plot.color
    else
        pz
    end
    minfilter = to_value(get(plot, :interpolate, false)) ? :linear : :nearest
    color = Sampler(lift(x -> el32convert(x'), pcolor), minfilter=minfilter)
    normals = Buffer(lift(surface_normals, px, py, pz))
    vertices = GeometryBasics.meta(positions; uv=uv, normals=normals)
    mesh = GeometryBasics.Mesh(vertices, faces)
    return draw_mesh(mscene, mesh, plot; uniform_color=color, color=Vec4f0(0),
                     shading=plot.shading, ambient=plot.ambient, diffuse=plot.diffuse,
                     specular=plot.specular, shininess=plot.shininess,
                     lightposition=Vec3f0(1))
end

function create_shader(mscene::Scene, plot::Union{Heatmap,Image})
    image = plot[3]
    color = Sampler(map(x -> el32convert(x'), image);
                    minfilter=to_value(get(plot, :interpolate, false)) ? :linear : :nearest)
    mesh = limits_to_uvmesh(plot)

    return draw_mesh(mscene, mesh, plot; uniform_color=color, color=Vec4f0(0),
                     normals=Vec3f0(0), shading=false, ambient=plot.ambient,
                     diffuse=plot.diffuse, specular=plot.specular,
                     colorrange=haskey(plot, :colorrange) ? plot.colorrange : false,
                     shininess=plot.shininess, lightposition=Vec3f0(1))
end

function create_shader(mscene::Scene, plot::Volume)
    x, y, z, vol = plot[1], plot[2], plot[3], plot[4]
    box = GeometryBasics.mesh(FRect3D(Vec3f0(0), Vec3f0(1)))
    cam = cameracontrols(mscene)
    model2 = lift(plot.model, x, y, z) do m, xyz...
        mi = minimum.(xyz)
        maxi = maximum.(xyz)
        w = maxi .- mi
        m2 = Mat4f0(w[1], 0, 0, 0, 0, w[2], 0, 0, 0, 0, w[3], 0, mi[1], mi[2], mi[3], 1)
        return convert(Mat4f0, m) * m2
    end

    modelinv = lift(inv, model2)
    algorithm = lift(x -> Cuint(convert_attribute(x, key"algorithm"())), plot.algorithm)

    return Program(WebGL(), lasset("volume.vert"), lasset("volume.frag"), box,
                   volumedata=Sampler(lift(AbstractPlotting.el32convert, vol)),
                   modelinv=modelinv, colormap=Sampler(lift(to_colormap, plot.colormap)),
                   colorrange=lift(Vec2f0, plot.colorrange),
                   isovalue=lift(Float32, plot.isovalue),
                   isorange=lift(Float32, plot.isorange),
                   absorption=lift(Float32, get(plot, :absorption, Observable(1f0))),
                   algorithm=algorithm, ambient=plot.ambient,
                   diffuse=plot.diffuse, specular=plot.specular, shininess=plot.shininess,
                   model=model2,
                   # these get filled in later by serialization, but we need them
                   # as dummy values here, so that the correct uniforms are emitted
                   lightposition=Vec3f0(1), eyeposition=Vec3f0(1))
end


text:
# Julia wrapper for header: /usr/local/include/libavcodec/dirac.h
# Automatically generated using Clang.jl wrap_c, version 0.0.0

export
    av_dirac_parse_sequence_header

function av_dirac_parse_sequence_header(dsh,buf,buf_size::Csize_t,log_ctx)
    ccall((:av_dirac_parse_sequence_header,libavcodec),Cint,(Ptr{Ptr{AVDiracSeqHeader}},Ptr{UInt8},Csize_t,Ptr{Void}),dsh,buf,buf_size,log_ctx)
end

max_stars_repo_name: asafasban/AAVideoIO

max_issues_repo_name: asafasban/AAVideoIO

max_forks_repo_name: asafasban/AAVideoIO


text:
using HDF5
using PyPlot

t = h5open("projectResults/emp_results_2016_04_09_08_49_26.h5", "r") do file
    read(file, "t")
end
n_lst = collect(100:50:2000)
PyPlot.clf()
PyPlot.plot(n_lst,t,"b*")
PyPlot.title("Runtime vs. n")
PyPlot.xlabel("n")
PyPlot.ylabel("Runtime")
xs = collect(0:1:2000)
# ys1 = 0.5e-6 * xs.^2.5
# ys2 = 1e-6 * xs.^2.5
ys1 = 1.3e-8 * xs.^3
ys2 = 2.5e-8 * xs.^3
PyPlot.plot(xs,ys1,"r")
PyPlot.plot(xs,ys2,"r")
fname = string("empirical_runtime",".pdf")
PyPlot.savefig(fname)


text:
# __precompile__()

module MotionPlanning

using LinearAlgebra
using Statistics

using Reexport
@reexport using StaticArrays
@reexport using DifferentialDynamicsModels
using TypeSortedCollections
using DataStructures
using Requires
using RecipesBase

include("simplesets.jl")
include("basictypes.jl")
include("collisioncheckers.jl")
include("goals.jl")
include("samplesets.jl")
include("connections.jl")
include("nearneighbors.jl")
include("graphs.jl")
include("problems.jl")
# include("sampling.jl")
include("planners.jl")
# include("postprocessors.jl")
# include("plotting.jl")

# TODO: test suite!
# TODO: T<:AbstractFloat - either ensure type stability (at least for states), or just go with Float64 everything
# TODO: change up lazy vcat [.,.] syntax in anticipation of v0.4; also consider push!/append!
# TODO: half-baked controls caching - do it or kill it
# TODO: "prototype" (.h-ish) files to specify required methods; julia type system makes OOP a pain
# TODO: FMT re-solving is wonky with different sample sizes
# TODO: Think about parameterizing MPProblem on the StateSpace type (for ease of postprocessing)
# TODO: Requires.jl for conditional dependency loading

# MODERN TODO: broadcast API for ObstacleList
# MODERN TODO: abstract `StateSet`, `ConfigSet` (and maybe `WorkspaceSet`?) for use in `CollisionChecker`s and `Goal`s

function __init__()
    # CollisionChecker glue code
    @require SeparatingAxisTheorem2D="33f35c42-577e-5c66-8569-bbf1771cbf32" include("collisioncheckers/separatingaxistheorem2D.jl")
    # @require GeometryTypes
    # @require Bullet3

    # NearNeighborDataStructure glue code
    # @require NearestNeighbors
    # @require FLANN
end

end # module

max_stars_repo_name: pbpf/clean

max_issues_repo_name: pbpf/clean

max_forks_repo_name: pbpf/clean


text:
function isnumericallyclose(x::T, y::T, tol = eps(T)*2) where T
    if abs(x-y) < tol
        return true
    end

    return false
end

function getcslengthfromLUT(cs_LUT_sys::Vector{Vector{Vector{Int}}})

    N_subsystems = length(cs_LUT_sys)

    cs_len_sys = Vector{Int}(undef, N_subsystems)

    for m = 1:N_subsystems
        cs_LUT = cs_LUT_sys[m]

        cs_len_sys[m] = maximum(maximum(cs_LUT[i]) for i = 1:length(cs_LUT))
    end

    return cs_len_sys
end

function loadcouplinginfojson(save_path::String)::Tuple{Vector{Int}, Vector{Float64}, Vector{Tuple{Int,Int}}, Vector{Float64}}

    J = JSON.parsefile(save_path)

    H_IDs = Vector{Int}(undef, 0)
    H_css = Vector{Float64}(undef, 0)
    for dict in J["chemical shift"]
        id = convert(Int, dict["ID"])
        value = convert(Float64, dict["value"])
        push!(H_IDs, id)
        push!(H_css, value)
    end

    J_IDs = Vector{Tuple{Int,Int}}(undef, 0)
    J_vals = Vector{Float64}(undef, 0)
    for dict in J["J-coupling"]
        id1 = convert(Int, dict["ID1"])
        id2 = convert(Int, dict["ID2"])
        value = convert(Float64, dict["value"])
        push!(J_IDs, (id1, id2))
        push!(J_vals, value)
    end

    return H_IDs, H_css, J_IDs, J_vals
end


text:
export discontinue

import Base: show

"""
    discontinue()

Creates an operator, which prevents an emitting of self-depending messages and breaks a possible infinite loop.
Does nothing if observable scheduled asynchronously.

# Producing

Stream of type `<: Subscribable{L}` where `L` refers to type of source stream

# Examples
```jldoctest
using Rocket

s = BehaviorSubject(0)

subscription1 = subscribe!(s, logger())
subscription2 = subscribe!(s |> map(Int, d -> d + 1) |> discontinue(), s)
;

# output

[LogActor] Data: 0
[LogActor] Data: 1
```

See also: [`AbstractOperator`](@ref), [`RightTypedOperator`](@ref), [`ProxyObservable`](@ref), [`logger`](@ref)
"""
discontinue() = DiscontinueOperator()

struct DiscontinueOperator <: InferableOperator end

function on_call!(::Type{L}, ::Type{L}, operator::DiscontinueOperator, source) where L
    return proxy(L, source, DiscontinueProxy())
end

operator_right(::DiscontinueOperator, ::Type{L}) where L = L

struct DiscontinueProxy <: ActorProxy end

actor_proxy!(::Type{L}, proxy::DiscontinueProxy, actor::A) where { L, A } = DiscontinueActor{L, A}(actor)

mutable struct DiscontinueActorProps
    isnextpropagated     :: Bool
    iserrorpropagated    :: Bool
    iscompletepropagated :: Bool

    DiscontinueActorProps() = new(false, false, false)
end

struct DiscontinueActor{L, A} <: Actor{L}
    actor :: A
    props :: DiscontinueActorProps

    DiscontinueActor{L, A}(actor::A) where { L, A } = new(actor, DiscontinueActorProps())
end

isnextpropagated(actor::DiscontinueActor)     = actor.props.isnextpropagated
iserrorpropagated(actor::DiscontinueActor)    = actor.props.iserrorpropagated
iscompletepropagated(actor::DiscontinueActor) = actor.props.iscompletepropagated

setnextpropagated!(actor::DiscontinueActor, value::Bool)     = actor.props.isnextpropagated = value
seterrorpropagated!(actor::DiscontinueActor, value::Bool)    = actor.props.iserrorpropagated = value
setcompletepropagated!(actor::DiscontinueActor, value::Bool) = actor.props.iscompletepropagated = value

function on_next!(actor::DiscontinueActor{L}, data::L) where L
    if !isnextpropagated(actor)
        setnextpropagated!(actor, true)
        next!(actor.actor, data)
        setnextpropagated!(actor, false)
    end
end

function on_error!(actor::DiscontinueActor, err)
    if !iserrorpropagated(actor)
        seterrorpropagated!(actor, true)
        error!(actor.actor, err)
        seterrorpropagated!(actor, false)
    end
end

function on_complete!(actor::DiscontinueActor)
    if !iscompletepropagated(actor)
        setcompletepropagated!(actor, true)
        complete!(actor.actor)
        setcompletepropagated!(actor, false)
    end
end

Base.show(io::IO, ::DiscontinueOperator)         = print(io, "DiscontinueOperator()")
Base.show(io::IO, ::DiscontinueProxy)            = print(io, "DiscontinueProxy()")
Base.show(io::IO, ::DiscontinueActor{L}) where L = print(io, "DiscontinueActor($L)")


text:
@testset "1D OperatorConv" begin
    modes = (16, )
    ch = 64 => 128

    m = Chain(
        Dense(2, 64),
        OperatorConv(ch, modes, FourierTransform)
    )
    @test ndims(OperatorConv(ch, modes, FourierTransform)) == 1
    @test repr(OperatorConv(ch, modes, FourierTransform)) == "OperatorConv(64 => 128, (16,), FourierTransform, permuted=false)"

    𝐱 = rand(Float32, 2, 1024, 5)
    @test size(m(𝐱)) == (128, 1024, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 128, 1024, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "permuted 1D OperatorConv" begin
    modes = (16, )
    ch = 64 => 128

    m = Chain(
        Conv((1, ), 2=>64),
        OperatorConv(ch, modes, FourierTransform, permuted=true)
    )
    @test ndims(OperatorConv(ch, modes, FourierTransform, permuted=true)) == 1
    @test repr(OperatorConv(ch, modes, FourierTransform, permuted=true)) == "OperatorConv(64 => 128, (16,), FourierTransform, permuted=true)"

    𝐱 = rand(Float32, 2, 1024, 5)
    𝐱 = permutedims(𝐱, (2, 1, 3))
    @test size(m(𝐱)) == (1024, 128, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 1024, 128, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "1D OperatorKernel" begin
    modes = (16, )
    ch = 64 => 128

    m = Chain(
        Dense(2, 64),
        OperatorKernel(ch, modes, FourierTransform)
    )
    @test repr(OperatorKernel(ch, modes, FourierTransform)) == "OperatorKernel(64 => 128, (16,), FourierTransform, σ=identity, permuted=false)"

    𝐱 = rand(Float32, 2, 1024, 5)
    @test size(m(𝐱)) == (128, 1024, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 128, 1024, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "permuted 1D OperatorKernel" begin
    modes = (16, )
    ch = 64 => 128

    m = Chain(
        Conv((1, ), 2=>64),
        OperatorKernel(ch, modes, FourierTransform, permuted=true)
    )
    @test repr(OperatorKernel(ch, modes, FourierTransform, permuted=true)) == "OperatorKernel(64 => 128, (16,), FourierTransform, σ=identity, permuted=true)"

    𝐱 = rand(Float32, 2, 1024, 5)
    𝐱 = permutedims(𝐱, (2, 1, 3))
    @test size(m(𝐱)) == (1024, 128, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 1024, 128, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "2D OperatorConv" begin
    modes = (16, 16)
    ch = 64 => 64

    m = Chain(
        Dense(1, 64),
        OperatorConv(ch, modes, FourierTransform)
    )
    @test ndims(OperatorConv(ch, modes, FourierTransform)) == 2

    𝐱 = rand(Float32, 1, 22, 22, 5)
    @test size(m(𝐱)) == (64, 22, 22, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 64, 22, 22, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "permuted 2D OperatorConv" begin
    modes = (16, 16)
    ch = 64 => 64

    m = Chain(
        Conv((1, 1), 1=>64),
        OperatorConv(ch, modes, FourierTransform, permuted=true)
    )
    @test ndims(OperatorConv(ch, modes, FourierTransform, permuted=true)) == 2

    𝐱 = rand(Float32, 1, 22, 22, 5)
    𝐱 = permutedims(𝐱, (2, 3, 1, 4))
    @test size(m(𝐱)) == (22, 22, 64, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 22, 22, 64, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "2D OperatorKernel" begin
    modes = (16, 16)
    ch = 64 => 64

    m = Chain(
        Dense(1, 64),
        OperatorKernel(ch, modes, FourierTransform)
    )

    𝐱 = rand(Float32, 1, 22, 22, 5)
    @test size(m(𝐱)) == (64, 22, 22, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 64, 22, 22, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "permuted 2D OperatorKernel" begin
    modes = (16, 16)
    ch = 64 => 64

    m = Chain(
        Conv((1, 1), 1=>64),
        OperatorKernel(ch, modes, FourierTransform, permuted=true)
    )

    𝐱 = rand(Float32, 1, 22, 22, 5)
    𝐱 = permutedims(𝐱, (2, 3, 1, 4))
    @test size(m(𝐱)) == (22, 22, 64, 5)

    loss(x, y) = Flux.mse(m(x), y)
    data = [(𝐱, rand(Float32, 22, 22, 64, 5))]
    Flux.train!(loss, params(m), data, Flux.ADAM())
end

@testset "SpectralConv" begin
    modes = (16, 16)
    ch = 64 => 64
    permuted = false

    @test SpectralConv(ch, modes) isa OperatorConv
    @test SpectralConv(ch, modes).transform isa FourierTransform
end


text:
"""
 Type: GenericStopping
 Methods: start!, stop!, update_and_start!, update_and_stop!, fill_in!, reinit!, status

 A generic stopping criterion to solve instances (pb) with respect to some
 optimality conditions. Optimality is decided by computing a score, which is then
 tested to zero.

 Besides optimality conditions, we consider classical emergency exit:
 - domain error        (for instance: NaN in x)
 - unbounded problem   (not implemented)
 - unbounded x         (x is too large)
 - tired problem       (time limit attained)
 - resources exhausted (not implemented)
 - stalled problem     (not implemented)
 - iteration limit     (maximum number of iteration (i.e. nb of stop) attained)
 - main_pb limit       (tired or resources of main problem exhausted)

 Input :
    - pb         : A problem
    - state      : The information relative to the problem, see GenericState
    - (opt) meta : Metadata relative to stopping criterion.
    - (opt) main_stp : Stopping of the main loop in case we consider a Stopping
                       of a subproblem.
                       If not a subproblem, then nothing.

 Note: Metadata can be provided by the user or created with the Stopping
       constructor with kwargs. If a specific StoppingMeta is given and
       kwargs are provided, the kwargs have priority.
"""
mutable struct GenericStopping <: AbstractStopping

    # Problem
    pb :: Any

    # Problem stopping criterion
    meta :: AbstractStoppingMeta

    # Current information on the problem
    current_state :: AbstractState

    # Stopping of the main problem, or nothing
    main_stp :: Union{AbstractStopping, Nothing}

    function GenericStopping(pb            :: Any,
                             current_state :: AbstractState;
                             meta          :: AbstractStoppingMeta = StoppingMeta(),
                             main_stp      :: Union{AbstractStopping, Nothing} = nothing,
                             kwargs...)

     if !(isempty(kwargs))
      meta = StoppingMeta(; kwargs...)
     end

     return new(pb, meta, current_state, main_stp)
    end
end

"""
GenericStopping(pb, x): additional default constructor
The function creates a Stopping with a default State.

Keywords arguments are forwarded to the classical constructor.
"""
function GenericStopping(pb :: Any, x :: Iterate; kwargs...)
 return GenericStopping(pb, GenericState(x); kwargs...)
end

"""
update_and_start!: Update the values in the State and initializes the Stopping
Returns the optimity status of the problem.

Kwargs are forwarded to the update!(stp.current_state)
"""
function update_and_start!(stp :: AbstractStopping; kwargs...)

    update!(stp.current_state; kwargs...)
    OK = start!(stp)

    return OK
end

"""
fill_in!: A function that fill in the unspecified values of the AbstractState.

NotImplemented for Abstract/Generic-Stopping.
"""
function fill_in!(stp :: AbstractStopping, x :: Iterate)
 return throw(error("NotImplemented function"))
end

"""
 start!: update the Stopping and return a boolean true if we must stop.

 Purpose is to know if there is a need to even perform an optimization algorithm
 or if we are at an optimal solution from the beginning.

 Note: * start! initialize the start_time (if not done before) and meta.optimality0.
       * Keywords argument are sent to the _optimality_check!
"""
function start!(stp :: AbstractStopping; kwargs...)

 stt_at_x = stp.current_state
 x        = stt_at_x.x

 #Initialize the time counter
 if isnan(stp.meta.start_time)
  stp.meta.start_time = time()
 end
 #and synchornize with the State
 if stt_at_x.current_time == nothing
  update!(stt_at_x, current_time = time())
 end

 stp.meta.domainerror = _domain_check(stp.current_state)
 if !stp.meta.domainerror
   # Optimality check
   optimality0          = _optimality_check(stp; kwargs...)
   stp.meta.optimality0 = optimality0
   if isnan(optimality0)
     stp.meta.domainerror = true
   end

   stp.meta.optimal     = _null_test(stp, optimality0)
 end

 OK = stp.meta.optimal || stp.meta.domainerror

 return OK
end

"""
 reinit!: Reinitialize the meta data.

 If rstate is set as true it reinitialize the current_state
 (with the kwargs)
"""
function reinit!(stp :: AbstractStopping; rstate :: Bool = false, kwargs...)

 stp.meta.start_time  = NaN
 stp.meta.optimality0 = 1.0

 #reinitialize the boolean status
 stp.meta.fail_sub_pb     = false
 stp.meta.unbounded       = false
 stp.meta.unbounded_pb    = false
 stp.meta.tired           = false
 stp.meta.stalled         = false
 stp.meta.iteration_limit = false
 stp.meta.resources       = false
 stp.meta.optimal         = false
 stp.meta.suboptimal      = false
 stp.meta.main_pb         = false
 stp.meta.domainerror     = false

 #reinitialize the counter of stop
 stp.meta.nb_of_stop = 0

 #reinitialize the state
 if rstate
  reinit!(stp.current_state; kwargs...)
 end

 return stp
end

"""
update_and_stop!: update the values in the State and returns the optimality status
of the problem.

kwargs are forwarded to the update!(stp.current_state)
"""
function update_and_stop!(stp :: AbstractStopping; kwargs...)

 update!(stp.current_state; kwargs...)
 OK = stop!(stp)

 return OK
end

"""
stop!: update the Stopping and return a boolean true if we must stop.

serves the same purpose as start! in an algorithm, tells us if we
stop the algorithm (because we have reached optimality or we loop infinitely,
etc).

Keywords argument are sent to the _optimality_check!
"""
function stop!(stp :: AbstractStopping; kwargs...)

 x        = stp.current_state.x
 time     = stp.meta.start_time

 stp.meta.domainerror = _domain_check(stp.current_state)
 if !stp.meta.domainerror
   # Optimality check
   score = _optimality_check(stp; kwargs...)
   if isnan(score)
    stp.meta.domainerror = true
   end
   stp.meta.optimal = _null_test(stp, score)

   _unbounded_check!(stp, x)
   _unbounded_problem_check!(stp, x)
   _tired_check!(stp, x, time_t = time)
   _resources_check!(stp, x)
   _stalled_check!(stp, x)
   _iteration_check!(stp, x)

   if stp.main_stp != nothing
       _main_pb_check!(stp, x)
   end
 end

 OK = stp.meta.optimal || stp.meta.tired || stp.meta.stalled
 OK = OK || stp.meta.iteration_limit || stp.meta.unbounded || stp.meta.resources
 OK = OK || stp.meta.unbounded_pb || stp.meta.main_pb || stp.meta.domainerror
 OK = OK || stp.meta.suboptimal || stp.meta.fail_sub_pb

 _add_stop!(stp)

 return OK
end

"""
_add_stop!: increment a counter of stop.

Fonction called everytime stop! is called. In theory should be called once every
iteration of an algorithm.
"""
function _add_stop!(stp :: AbstractStopping)

 stp.meta.nb_of_stop += 1

 return stp
end

"""
_iteration_check!: check if the optimization algorithm has reached the iteration
limit.
"""
function _iteration_check!(stp :: AbstractStopping,
                           x   :: Iterate)

 max_iter = stp.meta.nb_of_stop >= stp.meta.max_iter

 stp.meta.iteration_limit = max_iter

 return stp
end

"""
_stalled_check!: check if the optimization algorithm is stalling.

NotImplemented: false by default.
"""
function _stalled_check!(stp :: AbstractStopping,
                         x   :: Iterate)

 stp.meta.stalled = false

 return stp
end

"""
_tired_check!: check if the optimization algorithm has been running for too long.
"""
function _tired_check!(stp    :: AbstractStopping,
                       x      :: Iterate;
                       time_t :: Number = NaN)

 # Time check
 if !isnan(time_t)
    update!(stp.current_state, current_time = time())
    elapsed_time = stp.current_state.current_time - time_t
    max_time     = elapsed_time > stp.meta.max_time
 else
    max_time = false
 end

 stp.meta.tired = max_time

 return stp
end

"""
_resources_check!: check if the optimization algorithm has exhausted the resources.

NotImplemented: false by default.
"""
function _resources_check!(stp    :: AbstractStopping,
                           x      :: Iterate)

 max_evals = false
 max_f     = false

 stp.meta.resources = max_evals || max_f

 return stp
end

"""
_main_pb_check!: check the resources and the time of the upper problem (if main_stp != nothing)
By default stp.meta.main_pb = false
Modify the meta of the main_stp
"""
function _main_pb_check!(stp    :: AbstractStopping,
                         x      :: Iterate)

 # Time check
 time = stp.main_stp.meta.start_time
 _tired_check!(stp.main_stp, x, time_t = time)
 max_time = stp.main_stp.meta.tired

 # Resource check
 _resources_check!(stp.main_stp, x)
 resources = stp.main_stp.meta.resources

 if stp.main_stp.main_stp != nothing
   _main_pb_check!(stp.main_stp, x)
   main_main_pb = stp.main_stp.meta.main_pb
 else
   main_main_pb = false
 end

 stp.meta.main_pb = max_time || resources || main_main_pb

 return stp
end

"""
_unbounded_check!: check if x gets too big
"""
function _unbounded_check!(stp  :: AbstractStopping,
                           x    :: Iterate)

 x_too_large = norm(x,Inf) >= stp.meta.unbounded_x

 stp.meta.unbounded = x_too_large

 return stp
end

"""
_unbounded_problem!: check if problem relative informations are unbounded

NotImplemented: false by default.
"""
function _unbounded_problem_check!(stp  :: AbstractStopping,
                                   x    :: Iterate)

 stp.meta.unbounded_pb = false

 return stp
end

"""
_optimality_check: compute the optimality score.

NotImplemented: Inf by default.
"""
function _optimality_check(stp  :: AbstractStopping; kwargs...)
 stp.current_state.current_score = Inf
 return Inf
end

"""
_null_test: check if the score is close enough to zero
(up to some precisions found in the meta).
"""
function _null_test(stp  :: AbstractStopping, optimality :: Number)

    atol, rtol, opt0 = stp.meta.atol, stp.meta.rtol, stp.meta.optimality0

    optimal = optimality <= stp.meta.tol_check(atol, rtol, opt0)

    return optimal
end

"""
status: returns the status of the algorithm:
    - Optimal : if we reached an optimal solution
    - Unbounded : if the problem doesn't have a lower bound
    - Stalled : if algorithm is stalling
    - IterationLimit : if we did too  many iterations of the algorithm
    - Tired : if the algorithm takes too long
    - ResourcesExhausted: if we used too many ressources,
                          i.e. too many functions evaluations
    - ResourcesOfMainProblemExhausted: in the case of a substopping, ResourcesExhausted or Tired
    for the main stopping.
    - Infeasible : default return value, if nothing is done the problem is
                   considered infeasible
    - DomainError : there is a NaN somewhere

Note: set keyword argument list to true, to get an Array with all the status.
"""
function status(stp :: AbstractStopping; list = false)

 tt = Dict([(:Optimal, :optimal),
            (:SubProblemFailure, :fail_sub_pb),
            (:SubOptimal, :suboptimal),
            (:Unbounded, :unbounded),
            (:UnboundedPb, :unbounded_pb),
            (:Stalled, :stalled),
            (:IterationLimit, :iteration_limit),
            (:Tired, :tired),
            (:ResourcesExhausted, :resources),
            (:ResourcesOfMainProblemExhausted, :main_pb),
            (:Infeasible, :infeasible),
            (:DomainError, :domainerror)])

 if list
  list_status = findall(x -> getfield(stp.meta, x), tt)
  if list_status == zeros(0) list_status = [:Unknown] end
 else
  list_status = findfirst(x -> getfield(stp.meta, x), tt)
  if list_status == nothing list_status = :Unknown end
 end

 return list_status
end


text:
"""
fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f::FormulaTerm,tbl::DataFrame,data::Array{T,3},times)
fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f::FormulaTerm,tbl::DataFrame,data::Array{T,2},basisfunction::BasisFunction)

Generates Designmatrix & fits model, either mass-univariate (one model per epoched-timepoint) or time-expanded (modeling linear overlap).

# Examples
Mass Univariate Linear
```julia-repl
julia> data,evts = loadtestdata("testCase1")
julia> data_r = reshape(data,(1,:))
julia> data_e,times = Unfold.epoch(data=data_r,tbl=evts,τ=(-1.,1.9),sfreq=10) # cut the data into epochs. data_e is now ch x times x epoch

julia> f  = @formula 0~1+continuousA+continuousB # 1
julia> model,results_long = fit(UnfoldLinearModel,f,evts,data_e,times)
```
Timexpanded Univariate Linear
```julia-repl
julia> basisfunction = firbasis(τ=(-1,1),sfreq=10,name="A")
julia> model,results_long = fit(UnfoldLinearModel,f,evts,data_r,basisfunction)
```

"""
function fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f::FormulaTerm,tbl::DataFrame,data::Array{T,3},times;kwargs...) where {T}
    @assert size(data,2) == length(times)

    to = TimerOutput()
    # Generate the designmatrices
    @timeit to "designmatrix" Xs = designmatrix(type,f,tbl;kwargs...)

    # Fit the model
    @timeit to "fit" ufModel = unfoldfit(type,Xs,data;kwargs...)

    @timeit to "condense" c = condense_long(ufModel,times)
    # Condense output and return
    @debug(to)
    return ufModel,c
end

function fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f::FormulaTerm, tbl::DataFrame, data::Array{T,2},basisfunction::BasisFunction; kwargs...) where {T}
# backward compatible call, typically these models are run with multiple events
fit(type,Dict(Any=>(f,basisfunction)), tbl, data; kwargs...)
end

# Timeexpanded Model
function fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f::Dict, tbl::DataFrame, data::Array{T,2}; kwargs...) where {T}
    to = TimerOutput()

    # Generate the designmatrices
    @timeit to "unfoldDesignmat" Xs = designmatrix(type,f,tbl;kwargs...)

    # Fit the model
    @timeit to "unfoldfit" ufModel = unfoldfit(type,Xs,data;kwargs...)

    @timeit to "unfoldCondense" c = condense_long(ufModel)
    @debug(to)
    return ufModel,c
end

# helper function for 1 channel data
function fit(type::Type{<:Union{UnfoldLinearModel,UnfoldLinearMixedModel}},f, tbl::DataFrame, data::Array{T,1}, args...; kwargs...) where {T}
    @debug("data array is size (X,), reshaping to (1,X)")
    data = reshape(data,1,:)
    return fit(type,f,tbl,data,args...;kwargs...)
end

"""
unfoldfit(::Type{UnfoldLinearMixedModel},Xobj::DesignMatrix,data::Union{<:AbstractArray{T,2},<:AbstractArray{T,3}}) where {T<:Union{Missing, <:Number}}
unfoldfit(::Type{UnfoldLinearModel},Xobj::DesignMatrix,data::Union{<:AbstractArray{T,2},<:AbstractArray{T,3}}) where {T<:Union{Missing, <:Number}}

Fit a DesignMatrix against a 2D/3D Array data along its last dimension
Data is typically interpreted as channel x time (with basisfunctions) or channel x time x epoch (for mass univariate)

Returns an UnfoldModel object with fields .beta, .sigma (in case of mixed model) and .modelinfo with a summary of the modelfit

Note: Might be renamed/refactored to fit! at a later point

# Examples
```julia-repl
```

"""
function unfoldfit(::Type{UnfoldLinearMixedModel},Xobj::DesignMatrix,data::Union{<:AbstractArray{T,2},<:AbstractArray{T,3}};kwargs...) where {T<:Union{Missing, <:Number}}
    # function content partially taken from MixedModels.jl bootstrap.jl
    df = Array{NamedTuple,1}()
    dataDim = length(size(data)) # surely there is a nicer way to get this but I dont know it

    # If we have3 dimension, we have a massive univariate linear mixed model for each timepoint
    if dataDim == 3
        firstData = data[1,1,:]
        ntime = size(data,2)
    else
        # with only 2 dimension, we run a single time-expanded linear mixed model per channel/voxel
        firstData = data[1,:]
        ntime = 1
    end
    nchan = size(data,1)

    #_,data = zeropad(Xobj.Xs[1],data)
    # get a un-fitted mixed model object

    mm = LinearMixedModel_wrapper(Xobj.formulas,firstData,Xobj.Xs)

    # prepare some variables to be used
    βsc, θsc= similar(coef(mm)), similar(mm.θ) # pre allocate
    p,k = length(βsc), length(θsc)
    #β_names = (Symbol.(fixefnames(mm))..., )

    β_names = (Symbol.(vcat(fixefnames(mm)...))...,)
    β_names = (unique(β_names)...,)

    @assert(length(β_names) == length(βsc),"Beta-Names & coefficient length do not match. Did you provide two identical basis functions?")

    @debug println("beta_names $β_names")
    @debug println("uniquelength: $(length(unique(β_names))) / $(length(β_names))")
    # for each channel
    prog = Progress(nchan*ntime,.1)
    #@showprogress .1
    for ch in range(1,stop=nchan)
        # for each time
        for t in range(1,stop=ntime)

            #@debug "ch:$ch/$nchan, t:$t/$ntime"
            @debug "data-size: $(size(data))"
            #@debug println("mixedModel: $(mm.feterms)")
            if ndims(data) == 3
                refit!(mm,data[ch,t,:])
            else
                refit!(mm,data[ch,:])
            end
            #@debug println(MixedModels.fixef!(βsc,mm))

            β = NamedTuple{β_names}(MixedModels.fixef!(βsc, mm))

            out = (
            objective = mm.objective,
            σ = mm.σ,
            β = NamedTuple{β_names}(MixedModels.fixef!(βsc, mm)),
            se = SVector{p,Float64}(MixedModels.stderror!(βsc, mm)), #SVector not necessary afaik, took over from MixedModels.jl
            θ = SVector{k,Float64}(MixedModels.getθ!(θsc, mm)),
            channel = ch,
            timeIX = ifelse(dataDim==2,NaN,t)
            )
            push!(df,out)
            ProgressMeter.next!(prog; showvalues = [(:channel,ch), (:time,t)])
        end
    end

    modelinfo = MixedModelBootstrap(
        df,
        deepcopy(mm.λ),
        getfield.(mm.reterms, :inds),
        copy(mm.optsum.lowerbd),
        NamedTuple{Symbol.(fnames(mm))}(map(t -> (t.cnames...,), mm.reterms)),
    )

    beta = [x.β for x in MixedModels.tidyβ(modelinfo)]

    sigma = [x.σ for x in MixedModels.tidyσs(modelinfo)]

    if ndims(data)==3
        beta = permutedims(reshape(beta,:,ntime,nchan),[3 2 1])
        sigma = permutedims(reshape(sigma,:,ntime,nchan),[3 2 1])
    else
        beta = reshape(beta,:,nchan)'
        sigma = reshape(sigma,:,nchan)'
    end

    return UnfoldLinearMixedModel(beta,sigma,modelinfo,Xobj)
end

 function unfoldfit(::Type{UnfoldLinearModel},Xobj::DesignMatrix,data;solver=(x,y)->solver_default(x,y),kwargs...)
     X = Xobj.Xs
    # mass univariate, data = ch x times x epochs
    X,data = zeropad(X,data)

    @debug "UnfoldLinearModel, datasize: $(size(data))"
    # mass univariate
    beta,modelinfo = solver(X,data)

    return UnfoldLinearModel(beta,modelinfo,Xobj)

end

"""
$(SIGNATURES)

Wrapper to generate a LinearMixedModel. Code taken from MixedModels.jl and slightly adapted.

"""
function LinearMixedModel_wrapper(form,data::Array{<:Union{TData},1},Xs;wts = []) where {TData<:Number}
    #    function LinearMixedModel_wrapper(form,data::Array{<:Union{Missing,TData},1},Xs;wts = []) where {TData<:Number}

    # XXX Push this to utilities zeropad
    # Make sure X & y are the same size
    m = size(Xs[1])[1]

    if m != size(data)[1]
        Xs = changeMatSize!(size(data)[1],Xs[1],Xs[2:end])
    end

    y = (reshape(float(data), (:, 1)))

    MixedModels.LinearMixedModel(y, Xs, form, wts)
 end

 function MixedModels.LinearMixedModel(y, Xs, form::Array, wts)

    form_combined = form[1]
    for f =form[2:end]

        form_combined = form_combined.lhs ~ MatrixTerm(form_combined.rhs[1] + f.rhs[1]) + form_combined.rhs[2:end] + f.rhs[2:end]
    end
    MixedModels.LinearMixedModel(y, Xs, form_combined, wts)
 end


text:
const MAXNUMPLTS = 20

type NetProgressPlotter
  net::NeuralNet
  stats::SolverStats
  fields::Vector{Symbol}
  subplt::Plots.Subplot
end

# setup
function NetProgressPlotter(net::NeuralNet, stats::SolverStats, fields::Vector{Symbol} = Symbol[:x, :xhat, :y, :Σ, :a]; kw...)
  n = length(net.layers)
  m = length(fields)

  # initialize the plots
  plts = Plots.Plot{typeof(backend())}[]
  for field in fields
    for (i,layer) in enumerate(net.layers)
      arr = getfield(layer, field)
      push!(plts, scatter(min(length(arr), MAXNUMPLTS); title = "Layer $i: $field", leg=false, show=false, xticks=nothing, subplot=true, kw...))
    end
  end

  # create the subplot
  subplt = subplot(plts...; nc = length(net.layers), show=false, kw...)

  NetProgressPlotter(net, stats, fields, subplt)
end

function track_progress(net::NeuralNet; stats::SolverStats = SolverStats(), fields::AVec{Symbol} = Symbol[:x, :xhat, :y, :Σ, :a], kw...)
  NetProgressPlotter(net, stats, fields; kw...)
end

function OnlineStats.fit!(plotter::NetProgressPlotter, updateiter = false; show=false)
  if updateiter
    plotter.stats.numiter += 1
  end

  for (i, field) in enumerate(plotter.fields)
    for (j, layer) in enumerate(plotter.net.layers)

      # get the values for each node in this layer
      arr = getfield(layer, field)

      # get the correct plot from the grid
      plt = plotter.subplt[i,j]

      # add the data
      data = vec(arr)[1:min(length(arr),MAXNUMPLTS)]
      push!(plt, plotter.stats.numiter, data)
    end
  end

  if show
    gui(plotter.subplt)
  end

  # todo: do something with other stats...
end

# # --------------------------------------------------------------------------

@require Qwt begin

  type LayerViz
    layer::NeuralNetLayer
    circles::Vector{Qwt.Ellipse}
    wgtlines::Matrix{Qwt.Line}
    biaslines::Vector{Qwt.Line}
    vals::Vector{Qwt.SceneText}
  end

  function updateLine(line::Qwt.Line, val::Float64, bigval::Float64)
    color = val > 0.0 ? :green : (val < 0.0 ? :red : :black)
    sz = max(1.0, 10.0 * abs(val) / bigval)
    Qwt.pen!(line, sz, color)
  end

  # updates the color of the circles, and color/size of the weight connections
  function OnlineStats.fit!(viz::LayerViz, bigweight::Float64, bigbias::Float64)
    # change the colors of the line widths (weights)
    nout, nin = size(viz.wgtlines)
    l = viz.layer

    for i in 1:nout
      for j in 1:nin
        updateLine(viz.wgtlines[i,j], l.w[i,j], bigweight)
      end
      updateLine(viz.biaslines[i], l.b[i], bigbias)
    end

    # if we dropped out this node, color it black
    for i in 1:nout
      Qwt.brush!(viz.circles[i], l.nextr[i] == 0.0 ? :black : :white)
    end

    # add Σ and activation values inside the circles of the nodes
    for i in 1:nout
      s = @sprintf("Σ: %1.3f\na: %1.3f", l.Σ[i], forward(l.activation, l.Σ[i]))
      Qwt.settext(viz.vals[i], s)
      Qwt.position!(viz.vals[i], Qwt.position(viz.circles[i]))
    end

  end

  # ----------------------

  # getCenters(n, sz, radius) = getLinspace(n, sz / 2 - radius - 20)
  getCenters(n, sz) = getLinspace(n+2, sz / 2)[2:end-1]

  function visualize(net::NeuralNet)
    scene = Qwt.Scene(Qwt.P2(0,0), Qwt.P2(1200,800))
    L = length(net.layers)
    maxnodes = maximum(map(x->max(x.nin,x.nout), net.layers))
    W, H = size(scene)

    # figure out the biggest circle radius we can use
    maxHorizontalRadius = W / (L+2) / 2.5
    # maxVerticalRadius = H / (maxnodes+1) / 3
    # radius = min(maxHorizontalRadius, maxVerticalRadius)

    # get the x positions (first Qwt.position is the inputs... won't need circles there)
    xs = getCenters(L+1, W)

    # create the layer visualizations
    Qwt.defaultBrush!(:white)
    z = -500 # anything negative is fine... use this to put the wgtlines behind the circles

    # add the nodes and weight connections
    vizs = LayerViz[]
    for (i, layer) in enumerate(net.layers)

      # get the xs/ys for the inputs and nodes for this layer
      xIn, xOut = xs[i:i+1]
      ysIn, ysOut = map(n->getCenters(n, H), (layer.nin, layer.nout))

      radius = min(maxHorizontalRadius, H / (layer.nout+1) / 2.5)
      circles = [Qwt.circle!(radius, Qwt.P2(xOut, ysOut[iOut])) for iOut in 1:layer.nout]
      wgtlines = [Qwt.line!(circles[iOut], Qwt.P3(xIn, ysIn[iIn], z)) for iOut in 1:layer.nout, iIn in 1:layer.nin]

      biaspos = Qwt.P2((xIn + xOut)/2, -H/2)
      biaslines = [Qwt.line!(circles[iOut], biaspos) for iOut in 1:layer.nout]

      vals = [Qwt.text!("") for iOut in 1:layer.nout]

      push!(vizs, LayerViz(layer, circles, wgtlines, biaslines, vals))
    end

    # add the input textboxes
    nin = net.layers[1].nin
    ys = getCenters(nin, H)
    inputvalx = (-W/2 + xs[1]) / 2
    inputvals = [Qwt.text!("", Qwt.P2(inputvalx, ys[i])) for i in 1:nin]
    # for i in 1:nin
    #   Qwt.line!(Qwt.P3(-W/2, ys[i], z), Qwt.P3(xs[1], ys[i], z))
    # end

    viz = NetViz(net, vizs, inputvals)
    fit!(viz)
    viz
  end

  type NetViz
    net::NeuralNet
    layervizs::Vector{LayerViz}
    inputvals::Vector
  end

  # find the biggest weight and bias in absolute value within the network... used for determining the line width
  biggestabswgt(net::NeuralNet) = maximum([maximum(abs(l.w)) for l in net.layers])
  biggestabsbias(net::NeuralNet) = maximum([maximum(abs(l.b)) for l in net.layers])

  function OnlineStats.fit!(viz::NetViz; bigweight::Float64 = biggestabswgt(viz.net), bigbias::Float64 = biggestabsbias(viz.net))

    # update the layers
    for lv in viz.layervizs
      fit!(lv, bigweight, bigbias)
    end

    # update the input val textboxes
    l = viz.net.layers[1]
    for i in 1:l.nin
      iv = viz.inputvals[i]
      Qwt.settext(iv, @sprintf("Input: %1.3f", l.x[i]))
    end

  end

end


text:
module TensorFlowBuilder

include("CoreTypes.jl")
using .CoreTypes

include("PyInspector.jl")
using .PyInspector

include("TFParser.jl")
using .TFParser

end # module


text:
# using Dates

@testset "Ase.jl" begin
    filename = "data\\test.ase"
    df = DDR2import.Ase.read(filename)
    @test df[1,:FLIGHTCOUNT] == 0.0
    @test df[1,:SEGMENTPARITY] == 1
    @test df[1,:SEGMENTTYPE] == 41
    @test df[1,:LATBEGINSEGMENT_DEG] ≈ 48.7580 atol = 0.0001
    @test df[1,:LONBEGINSEGMENT_DEG] ≈ 10.8534 atol = 0.0001
    @test df[1,:LATENDSEGMENT_DEG] ≈ 48.6956 atol = 0.0001
    @test df[1,:LONENDSEGMENT_DEG] ≈ 10.9489 atol = 0.0001
    @test df[1,:SEGMENTNAME] == "%%BRU_BURAM"
    #LAST TEST
    @test df[3,:SEGMENTNAME] == "%%EDH_WSN"
end


text:
@testset "dynamic programming" begin
    T = Tuple{Int32, Int32}
    windows = 4
    haplotype_set = [T[] for i in 1:windows]
    push!(haplotype_set[1], (1, 4))
    push!(haplotype_set[1], (1, 5))
    push!(haplotype_set[1], (1, 6))
    push!(haplotype_set[1], (2, 4))
    push!(haplotype_set[1], (2, 5))
    push!(haplotype_set[1], (2, 6))
    push!(haplotype_set[1], (3, 4))
    push!(haplotype_set[1], (3, 5))
    push!(haplotype_set[1], (3, 6))

    push!(haplotype_set[2], (1, 5))
    push!(haplotype_set[2], (1, 7))
    push!(haplotype_set[2], (1, 8))
    push!(haplotype_set[2], (2, 5))
    push!(haplotype_set[2], (2, 7))
    push!(haplotype_set[2], (2, 8))
    push!(haplotype_set[2], (6, 5))
    push!(haplotype_set[2], (6, 7))
    push!(haplotype_set[2], (6, 8))

    push!(haplotype_set[3], (1, 1))
    push!(haplotype_set[3], (1, 5))
    push!(haplotype_set[3], (3, 1))
    push!(haplotype_set[3], (3, 5))

    push!(haplotype_set[4], (4, 4))
    push!(haplotype_set[4], (4, 6))
    push!(haplotype_set[4], (4, 8))
    push!(haplotype_set[4], (5, 4))
    push!(haplotype_set[4], (5, 6))
    push!(haplotype_set[4], (5, 8))
    push!(haplotype_set[4], (8, 4))
    push!(haplotype_set[4], (8, 6))
    push!(haplotype_set[4], (8, 8))

    sol_path, next_pair, subtree_err, best_err =
        MendelImpute.connect_happairs!(haplotype_set)

    @test best_err == 1.0
    @test sol_path == [(1, 5), (1, 5), (1, 5), (5, 4)]
    @test all(subtree_err[1] .== [2.0; 1.0; 2.0; 3.0; 2.0; 3.0; 5.0; 2.0; 3.0])
    @test all(subtree_err[2] .== [1.0; 2.0; 2.0; 2.0; 5.0; 5.0; 2.0; 5.0; 5.0])
    @test all(subtree_err[3] .== [4.0; 1.0; 4.0; 1.0])
    @test all(subtree_err[end] .== 0.0)
    @test all(next_pair[1] .== [1; 1; 1; 4; 1; 4; 1; 1; 7])
    @test all(next_pair[2] .== 2)
    @test all(next_pair[3] .== [1; 4; 1; 4])
    @test all(next_pair[4] .== 0)
end