text:
# Creates a _prettified_ version of a CST.

@enum(
    PLeaf,
    NEWLINE,
    SEMICOLON,
    WHITESPACE,
    PLACEHOLDER,
    NOTCODE,
    INLINECOMMENT,
    TRAILINGCOMMA,
)

mutable struct PTree
    typ::Union{CSTParser.Head,PLeaf}
    startline::Int
    endline::Int
    indent::Int
    len::Int
    val::Union{Nothing,AbstractString}
    nodes::Union{Nothing,Vector{PTree}}
    ref::Union{Nothing,Ref{CSTParser.EXPR}}
    force_nest::Bool
end

PTree(x::CSTParser.EXPR, indent::Int) =
    PTree(x.typ, -1, -1, indent, 0, nothing, PTree[], Ref(x), false)

function PTree(x::CSTParser.EXPR, startline::Int, endline::Int, val::AbstractString)
    PTree(x.typ, startline, endline, 0, length(val), val, nothing, Ref(x), false)
end

function PTree(x::CSTParser.Head, startline::Int, endline::Int, val::AbstractString)
    PTree(x, startline, endline, 0, length(val), val, nothing, nothing, false)
end

Newline() = PTree(NEWLINE, -1, -1, 0, 0, "\n", nothing, nothing, false)
Semicolon() = PTree(SEMICOLON, -1, -1, 0, 1, ";", nothing, nothing, false)
TrailingComma() = PTree(TRAILINGCOMMA, -1, -1, 0, 0, "", nothing, nothing, false)
Whitespace(n) = PTree(WHITESPACE, -1, -1, 0, n, " "^n, nothing, nothing, false)
Placeholder(n) = PTree(PLACEHOLDER, -1, -1, 0, n, " "^n, nothing, nothing, false)
Notcode(startline, endline) =
    PTree(NOTCODE, startline, endline, 0, 0, "", nothing, nothing, false)
InlineComment(line) = PTree(INLINECOMMENT, line, line, 0, 0, "", nothing, nothing, false)

Base.length(x::PTree) = x.len

is_leaf(x::PTree) = x.nodes === nothing
empty_start(x::PTree) = x.startline == 1 && x.endline == 1 && x.val == ""

is_punc(x) = CSTParser.ispunctuation(x)
is_end(x) = x.typ === CSTParser.KEYWORD && x.val == "end"
is_colon(x) = x.typ === CSTParser.OPERATOR && x.val == ":"
is_comma(x::PTree) =
    (x.typ === CSTParser.PUNCTUATION && x.val == ",") || x.typ === TRAILINGCOMMA
is_comment(x::PTree) = x.typ === INLINECOMMENT || x.typ === NOTCODE

is_colon_op(x) =
    (x.typ === CSTParser.BinaryOpCall && x.args[2].kind === Tokens.COLON) ||
    x.typ === CSTParser.ColonOpCall

function parent_is(x, typs...)
    p = x.parent
    p === nothing && return false
    while p !== nothing && p.typ === CSTParser.InvisBrackets
        p = p.parent
    end

    for typ in typs
        p.typ === typ && return true
    end
    false
end

function add_node!(t::PTree, n::PTree, s::State; join_lines = false, max_padding = -1)
    if n.typ === SEMICOLON
        join_lines = true
        loc = s.offset > length(s.doc.text) && t.typ === CSTParser.TopLevel ?
              loc = cursor_loc(s, s.offset - 1) : cursor_loc(s)
        for l = t.endline:loc[1]
            if has_semicolon(s.doc, l)
                # @info "found semicolon" l
                n.startline = l
                n.endline = l
                break
            end
        end
        # @info "" t.endline n.endline loc[1]

        # If there's no semicolon, treat it
        # as a PLeaf
        if n.startline == -1
            t.len += length(n)
            n.startline = t.startline
            n.endline = t.endline
            push!(t.nodes, n)
            return
        end
    elseif n.typ === TRAILINGCOMMA && is_comma(t.nodes[end])
        t.nodes[end] = n
        return
    elseif n.typ === NOTCODE || n.typ === INLINECOMMENT
        push!(t.nodes, n)
        return
    elseif n.typ isa PLeaf
        t.len += length(n)
        n.startline = t.startline
        n.endline = t.endline
        push!(t.nodes, n)
        return
    end

    if n.typ === CSTParser.Block && length(n) == 0
        return
    elseif n.typ === CSTParser.Parameters
        add_node!(t, Semicolon(), s)
        add_node!(t, Placeholder(1), s)
    end

    if length(t.nodes) == 0
        t.startline = n.startline
        t.endline = n.endline
        t.len += length(n)
        push!(t.nodes, n)
        return
    end

    if !is_prev_newline(t.nodes[end])
        current_line = t.endline
        notcode_startline = current_line + 1
        notcode_endline = n.startline - 1
        nt = t.nodes[end].typ

        if notcode_startline <= notcode_endline && n.typ !== CSTParser.LITERAL
            # If there are comments in between node elements
            # nesting is forced in an effort to preserve them.
            t.force_nest = true

            hs = hascomment(s.doc, current_line)
            hs && add_node!(t, InlineComment(current_line), s)
            if nt !== PLACEHOLDER
                add_node!(t, Newline(), s)
            elseif hs && nt === PLACEHOLDER
                # swap PLACEHOLDER (will be NEWLINE) with INLINECOMMENT node
                idx = length(t.nodes)
                t.nodes[idx-1], t.nodes[idx] = t.nodes[idx], t.nodes[idx-1]
            end
            add_node!(t, Notcode(notcode_startline, notcode_endline), s)
            add_node!(t, Newline(), s)
        elseif !join_lines
            hascomment(s.doc, current_line) && add_node!(t, InlineComment(current_line), s)
            add_node!(t, Newline(), s)
        elseif nt === PLACEHOLDER &&
               current_line != n.startline && hascomment(s.doc, current_line)
            t.force_nest = true
            add_node!(t, InlineComment(current_line), s)
            # swap PLACEHOLDER (will be NEWLINE) with INLINECOMMENT node
            idx = length(t.nodes)
            t.nodes[idx-1], t.nodes[idx] = t.nodes[idx], t.nodes[idx-1]
        end

        if n.typ === CSTParser.Parameters && n.force_nest == true
            t.force_nest = true
        end
    end

    if n.startline < t.startline || t.startline == -1
        t.startline = n.startline
    end
    if n.endline > t.endline || t.endline == -1
        t.endline = n.endline
    end

    if !join_lines && is_end(n)
        # end keyword isn't useful w.r.t margin lengths
    elseif t.typ === CSTParser.StringH
        # The length of this node is the length of
        # the longest string
        t.len = max(t.len, length(n))
    elseif max_padding >= 0
        t.len = max(t.len, length(n) + max_padding)
    else
        t.len += length(n)
    end
    push!(t.nodes, n)
    nothing
end

function is_prev_newline(x::PTree)
    if x.typ === NEWLINE
        return true
    elseif is_leaf(x) || length(x.nodes) == 0
        return false
    end
    is_prev_newline(x.nodes[end])
end

"""
`length_to_newline` returns the length to the next NEWLINE or PLACEHOLDER node

based off the `start` index.
"""
function length_to_newline(x::PTree, start = 1)
    x.typ === NEWLINE && (return 0, true)
    x.typ === PLACEHOLDER && (return 0, true)
    is_leaf(x) && (return length(x), false)
    len = 0
    for i = start:length(x.nodes)
        n = x.nodes[i]
        ln, nl = length_to_newline(n)
        len += ln
        nl && (return len, nl)
    end
    return len, false
end

is_closer(x::PTree) =
    x.typ === CSTParser.PUNCTUATION && (x.val == "}" || x.val == ")" || x.val == "]")
is_closer(x::CSTParser.EXPR) =
    x.kind === Tokens.RBRACE || x.kind === Tokens.RPAREN || x.kind === Tokens.RSQUARE

is_opener(x::PTree) =
    x.typ === CSTParser.PUNCTUATION && (x.val == "{" || x.val == "(" || x.val == "[")
is_opener(x::CSTParser.EXPR) =
    x.kind === Tokens.LBRACE || x.kind === Tokens.LPAREN || x.kind === Tokens.LSQUARE

function pretty(x::CSTParser.EXPR, s::State)
    # @debug "" x.typ x.args
    if x.typ === CSTParser.IDENTIFIER
        return p_identifier(x, s)
    elseif x.typ === CSTParser.OPERATOR
        return p_operator(x, s)
    elseif x.typ === CSTParser.PUNCTUATION
        return p_punctuation(x, s)
    elseif x.typ === CSTParser.KEYWORD
        return p_keyword(x, s)
    elseif x.typ === CSTParser.LITERAL
        return p_literal(x, s)
    elseif x.typ === CSTParser.StringH
        return p_stringh(x, s)
    elseif x.typ === CSTParser.Block
        return p_block(x, s)
    elseif x.typ === CSTParser.ModuleH
        return p_module(x, s)
    elseif x.typ === CSTParser.BareModule
        return p_module(x, s)
    elseif x.typ === CSTParser.FunctionDef
        return p_function(x, s)
    elseif x.typ === CSTParser.Macro
        return p_function(x, s)
    elseif x.typ === CSTParser.Primitive
        return p_primitive(x, s)
    elseif x.typ === CSTParser.Struct
        return p_struct(x, s)
    elseif x.typ === CSTParser.Mutable
        return p_mutable(x, s)
    elseif x.typ === CSTParser.Abstract
        return p_abstract(x, s)
    elseif x.typ === CSTParser.Primitive
        return p_primitive(x, s)
    elseif x.typ === CSTParser.For
        return p_loop(x, s)
    elseif x.typ === CSTParser.While
        return p_loop(x, s)
    elseif x.typ === CSTParser.Do
        return p_do(x, s)
    elseif x.typ === CSTParser.If
        return p_if(x, s)
    elseif x.typ === CSTParser.Try
        return p_try(x, s)
    elseif x.typ === CSTParser.TopLevel
        return p_toplevel(x, s)
    elseif x.typ === CSTParser.Begin
        return p_begin(x, s)
    elseif x.typ === CSTParser.Quote
        return p_quote(x, s)
    elseif x.typ === CSTParser.Let
        return p_let(x, s)
    elseif x.typ === CSTParser.Vect
        return p_vect(x, s)
    elseif x.typ === CSTParser.Braces
        return p_braces(x, s)
    elseif x.typ === CSTParser.TupleH
        return p_tuple(x, s)
    elseif x.typ === CSTParser.InvisBrackets
        return p_invisbrackets(x, s)
    elseif x.typ === CSTParser.Curly
        return p_curly(x, s)
    elseif x.typ === CSTParser.Call
        return p_call(x, s)
    elseif x.typ === CSTParser.MacroCall
        return p_macrocall(x, s)
    elseif x.typ === CSTParser.WhereOpCall
        return p_wherecall(x, s)
    elseif x.typ === CSTParser.ConditionalOpCall
        return p_condcall(x, s)
    elseif x.typ === CSTParser.BinaryOpCall
        return p_binarycall(x, s)
    elseif x.typ === CSTParser.UnaryOpCall
        return p_unarycall(x, s)
    elseif x.typ === CSTParser.ChainOpCall
        return p_chaincall(x, s)
    elseif x.typ === CSTParser.Comparison
        return p_chaincall(x, s)
    elseif x.typ === CSTParser.Kw
        return p_kw(x, s)
    elseif x.typ === CSTParser.Parameters
        return p_params(x, s)
    elseif x.typ === CSTParser.Local
        return p_vardef(x, s)
    elseif x.typ === CSTParser.Global
        return p_vardef(x, s)
    elseif x.typ === CSTParser.Const
        return p_vardef(x, s)
    elseif x.typ === CSTParser.Return
        return p_vardef(x, s)
    elseif x.typ === CSTParser.Import
        return p_import(x, s)
    elseif x.typ === CSTParser.ImportAll
        return p_import(x, s)
    elseif x.typ === CSTParser.Export
        return p_import(x, s)
    elseif x.typ === CSTParser.Using
        return p_import(x, s)
    elseif x.typ === CSTParser.Row
        return p_row(x, s)
    elseif x.typ === CSTParser.Vcat
        return p_vcat(x, s)
    elseif x.typ === CSTParser.TypedVcat
        return p_vcat(x, s)
    elseif x.typ === CSTParser.Hcat
        return p_hcat(x, s)
    elseif x.typ === CSTParser.TypedHcat
        return p_hcat(x, s)
    elseif x.typ === CSTParser.Ref
        return p_ref(x, s)
    elseif x.typ === CSTParser.Comprehension
        return p_comprehension(x, s)
    elseif x.typ === CSTParser.Generator
        return p_comprehension(x, s)
    elseif x.typ === CSTParser.Filter
        return p_comprehension(x, s)
    end

    t = PTree(x, nspaces(s))
    is_fileh = x.typ === CSTParser.FileH
    for a in x
        if a.kind === Tokens.NOTHING
            s.offset += a.fullspan
            continue
        end
        # @debug "" a a.typ
        add_node!(
            t,
            pretty(a, s),
            s,
            join_lines = !is_fileh,
            max_padding = is_fileh ? 0 : -1,
        )
    end
    t
end

function p_identifier(x, s)
    loc = cursor_loc(s)
    s.offset += x.fullspan
    PTree(x, loc[1], loc[1], x.val)
end

function p_operator(x, s)
    loc = cursor_loc(s)
    val = string(CSTParser.Expr(x))
    s.offset += x.fullspan
    PTree(x, loc[1], loc[1], val)
end

function p_keyword(x, s)
    loc = cursor_loc(s)
    val = x.kind === Tokens.ABSTRACT ? "abstract" :
          x.kind === Tokens.BAREMODULE ? "baremodule" :
          x.kind === Tokens.BEGIN ? "begin" :
          x.kind === Tokens.BREAK ? "break" :
          x.kind === Tokens.CATCH ? "catch" :
          x.kind === Tokens.CONST ? "const" :
          x.kind === Tokens.CONTINUE ? "continue" :
          x.kind === Tokens.NEW ? "new" :
          x.kind === Tokens.DO ? "do" :
          x.kind === Tokens.IF ? "if" :
          x.kind === Tokens.ELSEIF ? "elseif" :
          x.kind === Tokens.ELSE ? "else" :
          x.kind === Tokens.END ? "end" :
          x.kind === Tokens.EXPORT ? "export" :
          x.kind === Tokens.FINALLY ? "finally" :
          x.kind === Tokens.FOR ? "for" :
          x.kind === Tokens.FUNCTION ? "function" :
          x.kind === Tokens.GLOBAL ? "global" :
          x.kind === Tokens.IMPORT ? "import" :
          x.kind === Tokens.IMPORTALL ? "importall" :
          x.kind === Tokens.LET ? "let" :
          x.kind === Tokens.LOCAL ? "local" :
          x.kind === Tokens.MACRO ? "macro" :
          x.kind === Tokens.MODULE ? "module" :
          x.kind === Tokens.MUTABLE ? "mutable" :
          x.kind === Tokens.OUTER ? "outer " :
          x.kind === Tokens.PRIMITIVE ? "primitive" :
          x.kind === Tokens.QUOTE ? "quote" :
          x.kind === Tokens.RETURN ? "return" :
          x.kind === Tokens.STRUCT ? "struct" :
          x.kind === Tokens.TYPE ? "type" :
          x.kind === Tokens.TRY ? "try" :
          x.kind === Tokens.USING ? "using" : x.kind === Tokens.WHILE ? "while" : ""
    s.offset += x.fullspan
    PTree(x, loc[1], loc[1], val)
end

function p_punctuation(x, s)
    loc = cursor_loc(s)
    val = x.kind === Tokens.LPAREN ? "(" :
          x.kind === Tokens.LBRACE ? "{" :
          x.kind === Tokens.LSQUARE ? "[" :
          x.kind === Tokens.RPAREN ? ")" :
          x.kind === Tokens.RBRACE ? "}" :
          x.kind === Tokens.RSQUARE ? "]" :
          x.kind === Tokens.COMMA ? "," :
          x.kind === Tokens.SEMICOLON ? ";" :
          x.kind === Tokens.AT_SIGN ? "@" : x.kind === Tokens.DOT ? "." : ""
    s.offset += x.fullspan
    PTree(x, loc[1], loc[1], val)
end

function p_literal(x, s)
    loc = cursor_loc(s)
    if !is_str_or_cmd(x.kind)
        s.offset += x.fullspan
        return PTree(x, loc[1], loc[1], x.val)
    end

    # Strings are unescaped by CSTParser
    # to mimic Meta.parse which makes reproducing
    # the correct output from the LITERAL value problematic.
    # So we'll just look at the source directly!
    str_info = get(s.doc.lit_strings, s.offset - 1, nothing)

    # Tokenize treats the `ix` part of r"^(=?[^=]+)=(.*)$"ix as an
    # IDENTIFIER where as CSTParser parses it as a LITERAL.
    # An IDENTIFIER won't show up in the string literal lookup table.
    if str_info === nothing && x.parent.typ === CSTParser.x_Str
        s.offset += x.fullspan
        return PTree(x, loc[1], loc[1], x.val)
    end

    startline, endline, str = str_info
    # @debug "" loc startline endline str

    s.offset += x.fullspan

    lines = split(str, "\n")

    if length(lines) == 1
        return PTree(x, loc[1], loc[1], lines[1])
    end

    sidx = loc[2]
    for l in lines[2:end]
        fc = findfirst(c -> !isspace(c), l)
        if fc !== nothing
            sidx = min(sidx, fc)
        end
    end

    # @debug "" lines x.val loc loc[2] sidx

    t = PTree(CSTParser.StringH, -1, -1, loc[2], 0, nothing, PTree[], Ref(x), false)
    for (i, l) in enumerate(lines)
        ln = startline + i - 1
        l = i == 1 ? l : l[sidx:end]
        tt = PTree(CSTParser.LITERAL, ln, ln, sidx, length(l), l, nothing, nothing, false)
        add_node!(t, tt, s)
    end
    t
end

# StringH
function p_stringh(x, s)
    loc = cursor_loc(s)
    startline, endline, str = s.doc.lit_strings[s.offset-1]

    s.offset += x.fullspan

    lines = split(str, "\n")

    if length(lines) == 1
        t = PTree(x, startline, startline, lines[1])
        t.typ = CSTParser.LITERAL
        return t
    end

    sidx = loc[2]
    for l in lines[2:end]
        fc = findfirst(c -> !isspace(c), l)
        if fc !== nothing
            sidx = min(sidx, fc)
        end
    end

    # @debug "" lines x.val loc loc[2] sidx

    t = PTree(x, loc[2])
    for (i, l) in enumerate(lines)
        ln = startline + i - 1
        l = i == 1 ? l : l[sidx:end]
        tt = PTree(CSTParser.LITERAL, ln, ln, sidx, length(l), l, nothing, nothing, false)
        add_node!(t, tt, s)
    end
    t
end

# MacroCall
function p_macrocall(x, s)
    t = PTree(x, nspaces(s))
    if x.args[1].typ === CSTParser.GlobalRefDoc
        loc = cursor_loc(s)
        # x.args[1] is empty and fullspan is 0 so we can skip it
        if x.args[2].typ === CSTParser.LITERAL
            add_node!(t, p_literal(x.args[2], s), s, max_padding = 0)
        elseif x.args[2].typ == CSTParser.StringH
            add_node!(t, p_stringh(x.args[2], s), s)
        end
        loc = cursor_loc(s)
        add_node!(t, pretty(x.args[3], s), s, max_padding = 0)
        return t
    end

    multi_arg = length(x) > 5 && CSTParser.is_lparen(x.args[2]) ? true : false

    has_closer = is_closer(x.args[end])

    # @info "" has_closer

    # same as CSTParser.Call but whitespace sensitive
    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if a.typ === CSTParser.MacroName
            if a.fullspan - a.span > 0 && length(x) > 1
                add_node!(t, n, s, join_lines = true)
                add_node!(t, Whitespace(1), s)
            else
                # assumes the next argument is a brace of some sort
                add_node!(t, n, s, join_lines = true)
            end
        elseif is_opener(n)
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif is_closer(n)
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, n, s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        elseif a.fullspan - a.span > 0
            if has_closer && i < length(x) - 1
                add_node!(t, n, s, join_lines = true)
                add_node!(t, Whitespace(1), s)
            elseif !has_closer && i < length(x)
                add_node!(t, n, s, join_lines = true)
                add_node!(t, Whitespace(1), s)
            else
                add_node!(t, n, s, join_lines = true)
            end
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Block
# length Block is the length of the longest expr
function p_block(x, s; ignore_single_line = false, from_quote = false)
    t = PTree(x, nspaces(s))
    single_line = ignore_single_line ? false :
                  cursor_loc(s)[1] == cursor_loc(s, s.offset + x.span - 1)[1]
    # @debug "" from_quote single_line ignore_single_line
    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if from_quote && !single_line
            if i == 1 || CSTParser.is_comma(a)
                add_node!(t, n, s, join_lines = true)
            elseif CSTParser.is_comma(x.args[i-1])
                add_node!(t, Whitespace(1), s)
                add_node!(t, n, s, join_lines = true)
            else
                add_node!(t, Semicolon(), s)
                add_node!(t, n, s, max_padding = 0)
            end
        elseif single_line
            if i == 1 || CSTParser.is_comma(a)
                add_node!(t, n, s, join_lines = true)
            elseif CSTParser.is_comma(x.args[i-1])
                add_node!(t, Whitespace(1), s)
                add_node!(t, n, s, join_lines = true)
            else
                add_node!(t, Semicolon(), s)
                add_node!(t, Whitespace(1), s)
                add_node!(t, n, s, join_lines = true)
            end
        else
            if i < length(x) && CSTParser.is_comma(a) && is_punc(x.args[i+1])
                add_node!(t, n, s, join_lines = true)
            elseif CSTParser.is_comma(a) && i != length(x)
                add_node!(t, n, s, join_lines = true)
            else
                add_node!(t, n, s, max_padding = 0)
            end
        end
    end
    # @info "" t.typ length(t)
    t
end

# Abstract
function p_abstract(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[4], s), s, join_lines = true)
    t
end

# Primitive
function p_primitive(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), join_lines = true)
    add_node!(t, Whitespace(1))
    add_node!(t, pretty(x.args[3], s), join_lines = true)
    add_node!(t, Whitespace(1))
    add_node!(t, pretty(x.args[4], s), join_lines = true)
    add_node!(t, Whitespace(1))
    add_node!(t, pretty(x.args[5], s), join_lines = true)
    t
end

# FunctionDef/Macro
function p_function(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    if length(x) > 3
        if x.args[3].fullspan == 0
            add_node!(t, Whitespace(1), s)
            add_node!(t, pretty(x.args[4], s), s, join_lines = true)
        else
            s.indent += s.indent_size
            add_node!(
                t,
                p_block(x.args[3], s, ignore_single_line = true),
                s,
                max_padding = s.indent_size,
            )
            s.indent -= s.indent_size
            add_node!(t, pretty(x.args[4], s), s)
        end
    else
        # function stub, i.e. "function foo end"
        # this should be on one line
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    end
    t
end

# Struct
function p_struct(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    if x.args[3].fullspan == 0
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[4], s), s, join_lines = true)
    else
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[3], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size
        add_node!(t, pretty(x.args[4], s), s)
    end
    t
end

# Mutable
function p_mutable(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    if x.args[4].fullspan == 0
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[5], s), s, join_lines = true)
    else
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[4], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size
        add_node!(t, pretty(x.args[5], s), s)
    end
    t
end

# ModuleH/BareModule
function p_module(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    if x.args[3].fullspan == 0
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[4], s), s, join_lines = true)
    else
        add_node!(t, pretty(x.args[3], s), s, max_padding = 0)
        add_node!(t, pretty(x.args[4], s), s)
    end
    t
end

# Const/Local/Global/Return
function p_vardef(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    if x.args[2].fullspan != 0
        for a in x.args[2:end]
            add_node!(t, Whitespace(1), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# TopLevel
function p_toplevel(x, s)
    t = PTree(x, nspaces(s))
    for a in x.args
        if a.kind === Tokens.NOTHING
            s.offset += a.fullspan
            continue
        end
        add_node!(t, pretty(a, s), s, max_padding = s.indent_size)
        add_node!(t, Semicolon(), s)
    end
    t
end

# Begin
function p_begin(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    if x.args[2].fullspan == 0
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    else
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[2], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size
        add_node!(t, pretty(x.args[3], s), s)
    end
    t
end

# Quote
function p_quote(x, s)
    t = PTree(x, nspaces(s))
    if x.args[1].typ === CSTParser.KEYWORD && x.args[1].kind === Tokens.QUOTE
        add_node!(t, pretty(x.args[1], s), s)
        if x.args[2].fullspan == 0
            add_node!(t, Whitespace(1), s)
            add_node!(t, pretty(x.args[3], s), s, join_lines = true)
        else
            s.indent += s.indent_size
            add_node!(
                t,
                p_block(x.args[2], s, ignore_single_line = true),
                s,
                max_padding = s.indent_size,
            )
            s.indent -= s.indent_size
            add_node!(t, pretty(x.args[3], s), s)
        end
    else
        for a in x.args
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# Let
#
# two forms:
#
# let var1 = value1, var2
#     body
# end
#
# y, back = let
#     body
# end
function p_let(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    if length(x.args) > 3
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[2], s), s, join_lines = true)
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[3], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size
        add_node!(t, pretty(x.args[end], s), s)
    else
        s.indent += s.indent_size
        add_node!(t, p_block(x.args[2], s, ignore_single_line = true), s)
        s.indent -= s.indent_size
        add_node!(t, pretty(x.args[end], s), s)
    end
    t
end

# Transforms
#
# for i = iter body end
#
# =>
#
# for i in iter body end
#
# AND
#
# for i in 1:10 body end
#
# =>
#
# for i = 1:10 body end
#
# https://github.com/domluna/JuliaFormatter.jl/issues/34
function eq_to_in_normalization!(x)
    if x.typ === CSTParser.BinaryOpCall
        op = x.args[2]
        arg2 = x.args[3]
        if op.kind === Tokens.EQ && !is_colon_op(arg2)
            x.args[2].kind = Tokens.IN
        elseif op.kind === Tokens.IN && is_colon_op(arg2)
            x.args[2].kind = Tokens.EQ
        end
    elseif x.typ === CSTParser.Block || x.typ === CSTParser.InvisBrackets
        for a in x.args
            eq_to_in_normalization!(a)
        end
    end
end

# For/While
function p_loop(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    if x.args[1].kind === Tokens.FOR
        eq_to_in_normalization!(x.args[2])
    end
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    s.indent += s.indent_size
    add_node!(
        t,
        p_block(x.args[3], s, ignore_single_line = true),
        s,
        max_padding = s.indent_size,
    )
    s.indent -= s.indent_size
    add_node!(t, pretty(x.args[4], s), s)
    t
end

# Do
function p_do(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    if x.args[3].fullspan != 0
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    end
    if x.args[4].typ === CSTParser.Block
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[4], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size
    end
    add_node!(t, pretty(x.args[end], s), s)
    t
end

# Try
function p_try(x, s)
    t = PTree(x, nspaces(s))
    for a in x.args
        if a.fullspan == 0
        elseif a.typ === CSTParser.KEYWORD
            add_node!(t, pretty(a, s), s, max_padding = 0)
        elseif a.typ === CSTParser.Block
            s.indent += s.indent_size
            add_node!(
                t,
                p_block(a, s, ignore_single_line = true),
                s,
                max_padding = s.indent_size,
            )
            s.indent -= s.indent_size
        else
            len = length(t)
            add_node!(t, Whitespace(1), s)
            n = pretty(a, s)
            # "catch n"
            t.len = max(len, 5 + 1 + length(n))
            add_node!(t, n, s, join_lines = true, max_padding = 0)
        end
    end
    t
end

# If
function p_if(x, s)
    t = PTree(x, nspaces(s))
    if x.args[1].typ === CSTParser.KEYWORD && x.args[1].kind === Tokens.IF
        add_node!(t, pretty(x.args[1], s), s)
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(x.args[2], s), s, join_lines = true)
        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[3], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size

        len = length(t)
        if length(x.args) > 4
            add_node!(t, pretty(x.args[4], s), s, max_padding = 0)
            if x.args[4].kind === Tokens.ELSEIF
                add_node!(t, Whitespace(1), s)
                n = pretty(x.args[5], s)
                add_node!(t, n, s, join_lines = true)
                # "elseif n"
                t.len = max(len, length(n))
            else
                # ELSE KEYWORD
                s.indent += s.indent_size
                add_node!(
                    t,
                    p_block(x.args[5], s, ignore_single_line = true),
                    s,
                    max_padding = s.indent_size,
                )
                s.indent -= s.indent_size
            end
        end
        # END KEYWORD
        add_node!(t, pretty(x.args[end], s), s)
    else
        # "cond" part of "elseif cond"
        t.len += 7
        add_node!(t, pretty(x.args[1], s), s)

        s.indent += s.indent_size
        add_node!(
            t,
            p_block(x.args[2], s, ignore_single_line = true),
            s,
            max_padding = s.indent_size,
        )
        s.indent -= s.indent_size

        len = length(t)
        if length(x.args) > 2
            # this either else or elseif keyword
            add_node!(t, pretty(x.args[3], s), s, max_padding = 0)

            if x.args[3].kind === Tokens.ELSEIF
                add_node!(t, Whitespace(1), s)
                n = pretty(x.args[4], s)
                add_node!(t, n, s, join_lines = true)
                # "elseif n"
                t.len = max(len, length(n))
            else
                s.indent += s.indent_size
                add_node!(
                    t,
                    p_block(x.args[4], s, ignore_single_line = true),
                    s,
                    max_padding = s.indent_size,
                )
                s.indent -= s.indent_size
            end
        end
    end
    t
end

# ChainOpCall/Comparison
function p_chaincall(x, s)
    t = PTree(x, nspaces(s))
    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if a.typ === CSTParser.OPERATOR
            add_node!(t, Whitespace(1), s)
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        elseif i == length(x) - 1 && is_punc(a) && is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
        elseif CSTParser.is_comma(a) && i != length(x)
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# CSTParser.Kw
function p_kw(x, s)
    t = PTree(x, nspaces(s))
    for a in x
        if a.kind === Tokens.EQ
            add_node!(t, Whitespace(1), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# TODO: think of a better name?
block_type(x::CSTParser.EXPR) =
    x.typ === CSTParser.If ||
    x.typ === CSTParser.Do ||
    x.typ === CSTParser.Begin ||
    (x.typ === CSTParser.Quote && x.args[1].kind === Tokens.QUOTE) ||
    x.typ === CSTParser.Try ||
    x.typ === CSTParser.For || x.typ === CSTParser.While || x.typ === CSTParser.Let

nest_assignment(x::CSTParser.EXPR) = CSTParser.is_assignment(x) && block_type(x.args[3])

function nestable(x::CSTParser.EXPR)
    CSTParser.defines_function(x) && (return true)
    CSTParser.is_assignment(x) && (return block_type(x.args[3]))

    op = x.args[2]
    op.kind === Tokens.ANON_FUNC && (return false)
    op.kind === Tokens.PAIR_ARROW && (return false)
    CSTParser.precedence(op) in (1, 6) && (return false)
    if op.kind == Tokens.LAZY_AND || op.kind == Tokens.LAZY_OR
        arg = x.args[1]
        while arg.typ === CSTParser.InvisBrackets
            arg = arg.args[2]
        end
        if arg.typ === CSTParser.BinaryOpCall
            op = arg.args[2].kind
            (op == Tokens.LAZY_AND || op == Tokens.LAZY_OR) && (return true)
        end

        arg = x.args[3]
        while arg.typ === CSTParser.InvisBrackets
            arg = arg.args[2]
        end
        if arg.typ === CSTParser.BinaryOpCall
            op = arg.args[2].kind
            (op == Tokens.LAZY_AND || op == Tokens.LAZY_OR) && (return true)
        end

        return parent_is(x, CSTParser.If, CSTParser.BinaryOpCall, CSTParser.While)
    end
    true
end

function nest_arg2(x::CSTParser.EXPR)
    if CSTParser.defines_function(x)
        arg2 = x.args[3]
        arg2.typ === CSTParser.Block && (arg2 = arg2.args[1])
        return block_type(arg2)
    end
    false
end

# BinaryOpCall
function p_binarycall(x, s; nonest = false, nospace = false)
    t = PTree(x, nspaces(s))
    op = x.args[2]
    nonest = nonest || op.kind === Tokens.COLON
    if x.parent.typ === CSTParser.Curly && op.kind in (Tokens.ISSUBTYPE, Tokens.ISSUPERTYPE)
        nospace = true
    elseif op.kind === Tokens.COLON
        nospace = true
    end

    if x.args[1].typ === CSTParser.BinaryOpCall
        add_node!(t, p_binarycall(x.args[1], s, nonest = nonest, nospace = nospace), s)
    elseif x.args[1].typ === CSTParser.InvisBrackets
        add_node!(t, p_invisbrackets(x.args[1], s, nonest = nonest, nospace = nospace), s)
    else
        add_node!(t, pretty(x.args[1], s), s)
    end

    narg2 = nest_arg2(x)
    # @info "" nestable(x) !nonest narg2

    narg2 && (t.force_nest = true)
    nest = (nestable(x) && !nonest) || narg2

    if op.fullspan == 0 && x.args[3].typ === CSTParser.IDENTIFIER
        # do nothing
    elseif (CSTParser.precedence(op) in (8, 13, 14, 16) && op.kind !== Tokens.ANON_FUNC) ||
           nospace
        add_node!(t, pretty(op, s), s, join_lines = true)
    elseif op.kind === Tokens.EX_OR
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(op, s), s, join_lines = true)
    elseif nest
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(op, s), s, join_lines = true)
        # for newline
        add_node!(t, Placeholder(1), s)
    else
        add_node!(t, Whitespace(1), s)
        add_node!(t, pretty(op, s), s, join_lines = true)
        add_node!(t, Whitespace(1), s)
    end

    if x.args[3].typ === CSTParser.BinaryOpCall
        n = p_binarycall(x.args[3], s, nonest = nonest, nospace = nospace)
    elseif x.args[3].typ === CSTParser.InvisBrackets
        n = p_invisbrackets(x.args[3], s, nonest = nonest, nospace = nospace)
    else
        n = pretty(x.args[3], s)
    end
    add_node!(t, n, s, join_lines = true)

    if nest
        # for indent, will be converted to `indent_size` if needed
        insert!(t.nodes, length(t.nodes), Placeholder(0))
    end
    t
end

# WhereOpCall
# A where B
function p_wherecall(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)

    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    add_node!(t, Whitespace(1), s)

    # Used to mark where `B` starts.
    add_node!(t, Placeholder(0), s)

    multi_arg = length(CSTParser.get_where_params(x)) > 1
    add_braces = !CSTParser.is_lbrace(x.args[3]) &&
                 x.parent.typ !== CSTParser.Curly && x.args[3].typ !== CSTParser.Curly
    add_braces && add_node!(
        t,
        PTree(CSTParser.PUNCTUATION, t.endline, t.endline, "{"),
        s,
        join_lines = true,
    )

    # @debug "" multi_arg in_braces x.args[3].val == "{" x.args[end].val
    for (i, a) in enumerate(x.args[3:end])
        if is_opener(a) && multi_arg
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(0), s)
            s.indent += s.indent_size
        elseif is_closer(a) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
            s.indent -= s.indent_size
        elseif CSTParser.is_comma(a) && !is_punc(x.args[i+3])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif a.typ === CSTParser.BinaryOpCall
            add_node!(t, p_binarycall(a, s, nospace = true), s, join_lines = true)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    add_braces && add_node!(
        t,
        PTree(CSTParser.PUNCTUATION, t.endline, t.endline, "}"),
        s,
        join_lines = true,
    )
    t
end

# Conditional
function p_condcall(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    add_node!(t, Placeholder(1), s)

    add_node!(t, pretty(x.args[3], s), s, join_lines = true)
    add_node!(t, Whitespace(1), s)
    add_node!(t, pretty(x.args[4], s), s, join_lines = true)
    add_node!(t, Placeholder(1), s)

    add_node!(t, pretty(x.args[5], s), s, join_lines = true)
    t
end

# UnaryOpCall
function p_unarycall(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)
    t
end

function p_curly(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)

    multi_arg = length(CSTParser.get_curly_params(x)) > 1

    if multi_arg
        add_node!(t, Placeholder(0), s)
    end

    for (i, a) in enumerate(x.args[3:end])
        if i + 2 == length(x) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) - 3 && !is_punc(x.args[i+3])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

function p_call(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, pretty(x.args[2], s), s, join_lines = true)

    multi_arg = length(CSTParser.get_args(x)) > 1

    if multi_arg
        add_node!(t, Placeholder(0), s)
    end

    for (i, a) in enumerate(x.args[3:end])
        if i + 2 == length(x) && multi_arg
            pn = x.args[i+1]
            if pn.typ !== CSTParser.Parameters
                add_node!(t, TrailingComma(), s)
            elseif pn.typ === CSTParser.Parameters && !CSTParser.is_comma(pn.args[end])
                add_node!(t, TrailingComma(), s)
            end
            add_node!(t, Placeholder(0), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) - 3 && !is_punc(x.args[i+3])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# InvisBrackets
function p_invisbrackets(x, s; nonest = false, nospace = false)
    t = PTree(x, nspaces(s))
    multi_arg = length(x) > 4

    for (i, a) in enumerate(x)
        if a.typ === CSTParser.Block
            add_node!(t, p_block(a, s, from_quote = true), s, join_lines = true)
        elseif a.typ === CSTParser.BinaryOpCall
            add_node!(
                t,
                p_binarycall(a, s, nonest = nonest, nospace = nospace),
                s,
                join_lines = true,
            )
        elseif a.typ === CSTParser.InvisBrackets
            add_node!(
                t,
                p_invisbrackets(a, s, nonest = nonest, nospace = nospace),
                s,
                join_lines = true,
            )
        elseif is_opener(a) && multi_arg
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif is_closer(a) && multi_arg
            add_node!(t, Placeholder(0), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# TupleH
function p_tuple(x, s)
    t = PTree(x, nspaces(s))
    multi_arg = false
    if CSTParser.is_lparen(x.args[1]) && length(x) > 4
        multi_arg = true
    elseif !CSTParser.is_lparen(x.args[1]) && length(x) > 2
        multi_arg = true
    end

    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if is_opener(n) && multi_arg
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif is_closer(n) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, n, s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Braces
function p_braces(x, s)
    t = PTree(x, nspaces(s))
    # {a,b}
    multi_arg = length(x) > 4
    # @debug "" multi_arg typeof(x)

    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if i == 1 && multi_arg
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif i == length(x) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, n, s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Vect
function p_vect(x, s)
    t = PTree(x, nspaces(s))
    # [a,b]
    multi_arg = length(x) > 4

    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if i == 1 && multi_arg
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif i == length(x) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, n, s, join_lines = true)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Parameters
function p_params(x, s)
    t = PTree(x, nspaces(s))
    for (i, a) in enumerate(x)
        n = pretty(a, s)
        if CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        else
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Import, Export, Using, ImportAll
function p_import(x, s)
    t = PTree(x, nspaces(s))
    add_node!(t, pretty(x.args[1], s), s)
    add_node!(t, Whitespace(1), s)
    for (i, a) in enumerate(x.args[2:end])
        if CSTParser.is_comma(a)
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        elseif CSTParser.is_colon(a)
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# Ref
function p_ref(x, s)
    t = PTree(x, nspaces(s))
    multi_arg = length(x) > 5
    for (i, a) in enumerate(x)
        if is_closer(a) && multi_arg
            add_node!(t, TrailingComma(), s)
            add_node!(t, Placeholder(0), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
        elseif is_opener(a) && multi_arg
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Placeholder(1), s)
        elseif a.typ === CSTParser.BinaryOpCall
            add_node!(
                t,
                p_binarycall(a, s, nonest = true, nospace = true),
                s,
                join_lines = true,
            )
        elseif a.typ === CSTParser.InvisBrackets
            add_node!(
                t,
                p_invisbrackets(a, s, nonest = true, nospace = true),
                s,
                join_lines = true,
            )
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# Vcat/TypedVcat
function p_vcat(x, s)
    t = PTree(x, nspaces(s))
    st = x.typ === CSTParser.Vcat ? 1 : 2
    multi_arg = length(x) > st + 2

    for (i, a) in enumerate(x)
        n = pretty(a, s)
        diff_line = t.endline != t.startline
        if is_opener(a) && multi_arg
            add_node!(t, n, s, join_lines = true)
            add_node!(t, Placeholder(0), s)
        elseif !is_closer(a) && i > st
            add_node!(t, n, s, join_lines = true)
            if i != length(x) - 1
                add_node!(t, Semicolon(), s)
                add_node!(t, Placeholder(1), s)
            # Keep trailing semicolon if there's only one arg
            elseif !multi_arg
                add_node!(t, Semicolon(), s)
                add_node!(t, Placeholder(0), s)
            else
                add_node!(t, Placeholder(0), s)
            end
        else
            # If arguments are on different force nest
            diff_line && (t.force_nest = true)
            add_node!(t, n, s, join_lines = true)
        end
    end
    t
end

# Hcat/TypedHcat
function p_hcat(x, s)
    t = PTree(x, nspaces(s))
    st = x.typ === CSTParser.Hcat ? 1 : 2
    for (i, a) in enumerate(x)
        if i > st && i < length(x) - 1
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# Row
function p_row(x, s)
    t = PTree(x, nspaces(s))
    for (i, a) in enumerate(x)
        if i < length(x)
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end

# Comprehension/Generator/Filter
function p_comprehension(x, s)
    t = PTree(x, nspaces(s))
    for (i, a) in enumerate(x)
        if a.typ === CSTParser.KEYWORD
            add_node!(t, Whitespace(1), s)
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
            if a.kind === Tokens.FOR
                for j = i+1:length(x)
                    eq_to_in_normalization!(x.args[j])
                end
            end
        elseif CSTParser.is_comma(a) && i < length(x) && !is_punc(x.args[i+1])
            add_node!(t, pretty(a, s), s, join_lines = true)
            add_node!(t, Whitespace(1), s)
        else
            add_node!(t, pretty(a, s), s, join_lines = true)
        end
    end
    t
end


text:
# We work with Rydberg units, length scale Bohr radius a_0, energy scale: Ry
using StaticArrays

###### constants ###########
const e0 = sqrt(2)  # electric charge
const me = 0.5  # electron mass
const dim = 3    # dimension (D=2 or 3, doesn't work for other D!!!)
const spin = 2  # number of spins

const rs = 1.0
const kF = (dim == 3) ? (9π / (2spin))^(1 / 3) / rs : sqrt(4 / spin) / rs
const EF = kF^2 / (2me)
const beta = 0.05
const β = beta/ EF
const mass2 = 0.01

const Weight = SVector{2,Float64}
const Base.abs(w::Weight) = abs(w[1]) + abs(w[2]) # define abs(Weight)
const INL, OUTL, INR, OUTR = 1, 2, 3, 4
# const Nf = (D==3) ?

println("rs=$rs, β=$β, kF=$kF, EF=$EF, mass2=$mass2")

ext: jl


text:
const TupleWithDDF = Union{Tuple{<:DateDataFrame}, Tuple{<:DateDataFrame, <:Any}, Tuple{<:Any, <:DateDataFrame}}

Base.broadcastable(ddf::DateDataFrame) = ddf

Base.ndims(::Type{<:DateDataFrame{<:Any, <:AbstractVector}}) = 1
Base.ndims(::Type{<:DateDataFrame}) = 2

translate_arg(x) = x
translate_arg(x::DateDataFrame) = x.df
translate_arg(x::Base.Broadcast.Broadcasted) = translate_arg(Base.materialize(x)) # TODO: implement "true" fusing
# TODO: This implementation will not work for `ddf .* ddf .+ ddf .* ddf`, however I don't have time to fix it at this time.

function Base.materialize!(ddf::DateDataFrame, bc::Base.Broadcast.Broadcasted{<:Base.Broadcast.DefaultArrayStyle, <:Nothing, <:Any, <:TupleWithDDF})
    broadcast!(bc.f, ddf.df, translate_arg.(bc.args)...)
    return ddf
end

function Base.materialize!(ddf::DateDataFrame, bc::Base.Broadcast.Broadcasted{<:Base.Broadcast.DefaultArrayStyle, <:Nothing})
    broadcast!(bc.f, ddf.df, bc.args...)
    return ddf
end

# TODO: Very ugly hack...

function Base.materialize(bc::Base.Broadcast.Broadcasted{<:Base.Broadcast.DefaultArrayStyle, <:Nothing, <:Any, <:Tuple{<:DateDataFrame, <:Any}})
    ddf = bc.args[1]
    df = broadcast(bc.f, translate_arg.(bc.args)...)
    @assert length(ddf.timestamp) == size(df, 1)
    return DateDataFrame(ddf.timestamp, df)
end

function Base.materialize(bc::Base.Broadcast.Broadcasted{<:Base.Broadcast.DefaultArrayStyle, <:Nothing, <:Any, <:Tuple{<:Any, <:DateDataFrame}})
    ddf = bc.args[2]
    df = broadcast(bc.f, translate_arg.(bc.args)...)
    @assert length(ddf.timestamp) == size(df, 1)
    return DateDataFrame(ddf.timestamp, df)
end

function Base.materialize(bc::Base.Broadcast.Broadcasted{<:Base.Broadcast.DefaultArrayStyle, <:Nothing, <:Any, <:Tuple{<:DateDataFrame, <:DateDataFrame}})
    ddf = bc.args[1]
    df = broadcast(bc.f, translate_arg.(bc.args)...)
    @assert length(ddf.timestamp) == size(df, 1)
    return DateDataFrame(ddf.timestamp, df)
end

function Base.dotview(ddf::DateDataFrame, row_idx)
    t_idx = translate_idx(ddf.timestamp, row_idx)
    return DateDataFrame((@view ddf.timestamp[t_idx]), (@view ddf.df[t_idx]))
end


text:
abstract FPTree

type FPEmptyNode <: FPTree end

type FPTreeNode <: FPTree
    key::Int64 # I thought Generic Types should be used here, but it is much more harder to implement than I expected
    name::Int64
    cnt::Int64
    parent::Int64
    children::Array{Int64}

    function FPTreeNode(k,v)
        node = new(k,v,1,0)
        node.children = Array(Int64,0)
        node
    end

    function FPTreeNode(k,v,p)
        node = new(k,v,1,p)
        node.children = Array(Int64,0)
        node
    end

    function FPTreeNode(k,v,p,c)
        node = new(k,v,1,p)
        if typeof(c) <: Array{Int64}
            node.children = c
        else
            node.children = Array(Int64,0)
        end
        node
    end
end

function haschild(idx,childname,N)
    length(getnode(idx,N).children) < 1 && return false
    for c in getnode(idx,N).children
        getnode(c,N).name == childname && return true
    end
    return false
end

function getchildindex(idx,childname,N)
    for c in getnode(idx,N).children
        getnode(c,N).name == childname && return c
    end
end

haschild(t) = ifelse(length(t.children) < 1,false,true)

function growchild!(idx, childlist::AbstractArray, N::AbstractArray)
    nchild = length(childlist)
    if nchild < 1 return
    else
        child_idx = length(N)
        child_name = childlist[1]
        deleteat!(childlist,1)
        n = FPTreeNode(child_idx, child_name, idx)
        filter!(x -> !isempty(x), getnode(idx,N).children)
        push!(getnode(idx,N).children, child_idx)
        push!(N, n)
    end
    growchild!(child_idx, childlist,N)
end

function climb_grow(idx, childlist, N)
    isempty(childlist) && return
    childname = childlist[1]
    if haschild(idx, childname, N)
        childindex = getchildindex(idx, childname,N)
        getnode(childindex,N).cnt += 1
        deleteat!(childlist,1)
        climb_grow(childindex, childlist, N)
    else growchild!(idx, childlist,N)
    end
end

# Climb down from tree node to root, and record the path
function climb_down!(idx, N, path)
    push!(path, idx)
    idx == 0 && return sort!(path)
    climb_down!(getnode(idx,N).parent, N, path)
end

function remove_node(idx, N)
    _N = Array(FPTree,0)
    for n in N
        n.key == idx && continue
        n.parent == idx && (n.parent = 0) # if parent node is going to be removed, set root(null) node as parent
        if !isempty(n.children)
            _children = Array(Int64, 0)
            for i in 1:length(n.children)
                if n.children[i] != idx
                    push!(_children, n.children[i])
                end
            end
            n.children = _children
        end
        push!(_N, n)
    end
    _N
end

# Get a certain node by its key
function getnode(idx::Int64,N::Array{FPTree})
    for n in N
        if n.key == idx return n
        end
    end
    throw(BoundsError())
end


text:
# SPD-License-Identifier: MIT

using ArgParse
using AutoGrad
using CUDA
using Dates
using HDF5
using JLD2
using Knet
using MarkovModels
using Random
using TOML

# Data loader to load batch of utterances.
include("dataloader.jl")

# Differentiable LF-MMI objective function.
include("lfmmi_loss.jl")

function train!(model, batchloader, denfsm, opts, θ, use_gpu, getlengths)
    trainmode!(model)

    acc_loss::Float64 = 0
    N = 0

    for (i, (batch_data, batch_nums, inlengths)) in enumerate(batchloader)
        seqlengths = getlengths(inlengths)
        batch_dens = union([denfsm for i in 1:size(batch_data, 3)]...)
        if use_gpu
            batch_data = CuArray(batch_data)
            batch_nums = MarkovModels.gpu(batch_nums)
        end

        L = @diff lfmmi_loss(model(batch_data), batch_nums, batch_dens, seqlengths)

        update!(value.(θ), grad.([L], θ), opts)

        acc_loss += value(L) / sum(seqlengths)
        N += 1

        # Remove the variable from the scope to free the memory.
        L = nothing
    end

    acc_loss / N
end

function test!(model, batchloader, denfsm, use_gpu, getlengths)
    testmode!(model)

    acc_loss::Float64 = 0
    N = 0

    for (i, (batch_data, batch_nums, inlengths)) in enumerate(batchloader)
        seqlengths = getlengths(inlengths)

        batch_dens = union([denfsm for i in 1:size(batch_data, 3)]...)

        if use_gpu
            batch_data = CuArray(batch_data)
            batch_nums = MarkovModels.gpu(batch_nums)
        end

        L = lfmmi_loss(model(batch_data), batch_nums, batch_dens, seqlengths)

        acc_loss += L
        N += sum(seqlengths)
    end

    acc_loss / N
end

function main(args)
    conf = TOML.parsefile(args["config"])
    epochs = conf["epochs"]
    mbsize = conf["mini_batch_size"]
    use_gpu = args["use-gpu"]
    cdir = args["checkpoint-dir"]
    ckptpath = args["from-checkpoint"]

    ckpt = if ckptpath != ""
        @debug "Using checkpoint: $ckptpath."
        load(ckptpath)
    else
        nothing
    end

    getlengths_fn = s -> getlengths(TOML.parsefile(args["modelconfig"]), s)

    train_numfsms = load(args["train_numfsms"])
    dev_numfsms = load(args["dev_numfsms"])
    denfsm = load(args["denfsm"])["fsm"]

    model = isnothing(ckpt) ? load(args["model"])["model"] : ckpt["model"]

    @debug "model = $model"

    if args["use-gpu"]
        model = model |> gpu
        denfsm = denfsm |> MarkovModels.gpu
    end

    θ = params(model)
    opts = isnothing(ckpt) ? [getoptimizer(conf) for p in θ] : ckpt["optimizers"]
    scheduler = isnothing(ckpt) ? getscheduler(conf) : ckpt["scheduler"]
    best_loss = isnothing(ckpt) ? Float32(Inf) : ckpt["best_loss"]
    startepoch = isnothing(ckpt) ? 1 : ckpt["epoch"]+1
    for epoch in startepoch:epochs
        t₀ = now()

        train_loss = 0
        dev_loss = 0
        shuffledata = epoch > conf["curriculum"] ? true : false
        if ! shuffledata
            @debug "Loading utterances sorted by length (i.e. curriculum training)."
        end

        h5open(args["train"], "r") do trainfea
            trainbl = BatchLoader(trainfea, train_numfsms, mbsize; shuffledata)
            train_loss = train!(model, trainbl, denfsm, opts, θ, use_gpu,
                                getlengths_fn)
        end

        h5open(args["dev"], "r") do devfea
            devbl = BatchLoader(devfea, dev_numfsms, mbsize)
            dev_loss = test!(model, devbl, denfsm, use_gpu, getlengths_fn)
        end

        update_scheduler!(scheduler, opts, dev_loss)

        checkpoint = Dict(
            "model" => model |> cpu,
            "optimizers" => opts,
            "epoch" => epoch,
            "best_loss" => best_loss,
            "scheduler" => scheduler
        )
        ckpt_path = joinpath(cdir, "last.jld2")
        @debug "Saving checkpoint to $ckpt_path."
        save(ckpt_path, checkpoint)

        if dev_loss < best_loss
            best_loss = dev_loss
            ckpt_path = joinpath(cdir, "best.jld2")
            @debug "New best lost is $best_loss; model saved to $ckpt_path."
            save(ckpt_path, checkpoint)
        end

        t₁ = now()
        println("epoch=$epoch/$epochs " *
                "train_loss=$(round(train_loss, digits = 4)) " *
                "dev_loss=$(round(dev_loss, digits = 4)) " *
                "epoch_duration=$((t₁ - t₀).value/1000)")
    end
end

function parse_commandline()
    s = ArgParseSettings()
    @add_arg_table s begin
        "--checkpoint-dir", "-c"
            default = "checkpoints"
            help = "directory where to save training checkpoint"
        "--from-checkpoint", "-f"
            default = ""
            help = "start training from the provided checkpoint"
        "--use-gpu", "-g"
            action = :store_true
            help = "train the model on gpu"
        "modelfile"
            required = true
            help = "model definition file"
        "config"
            required = true
            help = "training configuration file (TOML)"
        "modelconfig"
            required = true
            help = "model configuration file (TOML)"
        "train"
            required = true
            help = "training set"
        "dev"
            required = true
            help = "development set"
        "train_numfsms"
            required = true
            help = "numerator fsms (i.e. alignment graphs) for the training set"
        "dev_numfsms"
            required = true
            help = "numerator fsms (i.e. alignment graphs) for the dev set"
        "denfsm"
            required = true
            help = "denominator graph"
        "model"
            required = true
            help = "input model saved in JLD2 format"
    end
    s.description = """
    Train a neural network using the LF-MMI objective function.
    of utterances to use is read from the standard input.
    """
    parse_args(s)
end

args = parse_commandline()

# This file should define:
#   * `getlengths`
#   * `getoptimizer`
#   * `getscheduler`
include(args["modelfile"])
main(args)

max_stars_repo_name: MartinKocour/SpeechLab

max_issues_repo_name: MartinKocour/SpeechLab

max_forks_repo_name: MartinKocour/SpeechLab


text:
# Thrift types are supported
# All Hive types (complex types) are not supported yet

const JTYPES = Dict(
  Int32(0) => Bool,                 # BOOLEAN
  Int32(1) => Int8,                 # TINYINT
  Int32(2) => Int16,                # SMALLINT
  Int32(3) => Int32,                # INT
  Int32(4) => Int64,                # BIGINT
  Int32(5) => Float64,              # FLOAT
  Int32(6) => Float64,              # DOUBLE
  Int32(7) => Compat.UTF8String,    # STRING
  Int32(8) => Int64,                # TIMESTAMP
  Int32(9) => Compat.UTF8String,    # BINARY
  Int32(10) => Compat.UTF8String,   # ARRAY
  Int32(11) => Compat.UTF8String,   # MAP
  Int32(12) => Compat.UTF8String,   # STRUCT
  Int32(13) => Compat.UTF8String,   # UNIONTYPE
  Int32(15) => Compat.UTF8String,   # DECIMAL
  Int32(16) => Compat.UTF8String,   # NULL
  Int32(17) => Compat.UTF8String,   # DATE
  Int32(18) => Compat.UTF8String,   # VARCHAR
  Int32(19) => Compat.UTF8String,   # CHAR
  Int32(20) => Compat.UTF8String,   # INTERVAL_YEAR_MONTH
  Int32(21) => Compat.UTF8String    # INTERVAL_DAY_TIME
)

##
# map column types to Julia types
function julia_type(hs2type::TTypeDesc, typeentry::TPrimitiveTypeEntry)
    JTYPES[typeentry._type]
end

function julia_type(hs2type::TTypeDesc, typeentry::TArrayTypeEntry)
    typeptr = typeentry.objectTypePtr
    T = julia_type(hs2type, hs2type.types[typeptr])
    Array{T,1}
end

function julia_type(hs2type::TTypeDesc, typeentry::TMapTypeEntry)
    keytypeptr = typeentry.keyTypePtr
    valuetypeptr = typeentry.valueTypePtr
    K = julia_type(hs2type, hs2type.types[keytypeptr])
    V = julia_type(hs2type, hs2type.types[valuetypeptr])
    Dict{K,V}
end

function julia_type(hs2type::TTypeDesc, typeentry::TStructTypeEntry)
    Any
end

function julia_type(hs2type::TTypeDesc, typeentry::TUnionTypeEntry)
    Any
end

function julia_type(hs2type::TTypeDesc, typeentry::TUserDefinedTypeEntry)
    Any
end

function julia_type(hs2type::TTypeDesc, typeentry::TTypeEntry)
    for fld in fieldnames(TTypeEntry)
        isfilled(typeentry, fld) || continue
        return julia_type(hs2type, getfield(typeentry, fld))
    end
end

function julia_type(hs2type::TTypeDesc)
    # The "top" type is always the first element of the list.
    # If the top type is an ARRAY, MAP, STRUCT, or UNIONTYPE type, then subsequent elements represent nested types.
    toptype = hs2type.types[1]
    julia_type(hs2type, toptype)
end

##
# map column types to Julia type names
function julia_type_name(hs2type::TTypeDesc, typeentry::TPrimitiveTypeEntry)
    TYPE_NAMES[typeentry._type]
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TArrayTypeEntry)
    typeptr = typeentry.objectTypePtr
    typename = julia_type_name(hs2type, hs2type.types[typeptr])
    "Array{$typename}"
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TMapTypeEntry)
    keytypeptr = typeentry.keyTypePtr
    valuetypeptr = typeentry.valueTypePtr
    keytypename = julia_type_name(hs2type, hs2type.types[keytypeptr])
    valuetypename = julia_type_name(hs2type, hs2type.types[valuetypeptr])
    "Dict{$keytypename, $valuetypename}"
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TStructTypeEntry)
    "Struct"
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TUnionTypeEntry)
    "Union"
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TUserDefinedTypeEntry)
    "UserDefined"
end

function julia_type_name(hs2type::TTypeDesc, typeentry::TTypeEntry)
    for fld in fieldnames(TTypeEntry)
        isfilled(typeentry, fld) || continue
        return julia_type_name(hs2type, getfield(typeentry, fld))
    end
end

function julia_type_name(hs2type::TTypeDesc)
    # The "top" type is always the first element of the list.
    # If the top type is an ARRAY, MAP, STRUCT, or UNIONTYPE type, then subsequent elements represent nested types.
    toptype = hs2type.types[1]
    julia_type_name(hs2type, toptype)
end

##
# map column values to Julia types
# map columns into DataVectors
typealias ColValue Union{TBoolValue, TByteValue, TI16Value, TI32Value, TI64Value, TDoubleValue, TStringValue}
typealias Col  Union{TBoolColumn, TByteColumn, TI16Column, TI32Column, TI64Column, TDoubleColumn, TStringColumn, TBinaryColumn}

function julia_type{T<:ColValue}(colval::T)
    isfilled(colval, :value) ? getfield(colval, :value) : NA
end

function julia_type(colval::TColumnValue)
    for fld in fieldnames(TColumnValue)
        isfilled(colval, fld) || continue
        return julia_type(getfield(colval, fld))
    end
    NA
end

function julia_type{T<:Col}(col::T)
    if (length(col.nulls) < 2) || isempty(col.values)
        DataArray(col.values)
    else
        values = col.values
        nulls = bitset_to_bools(col.nulls, length(values))
        DataArray(values, nulls)
    end
end

function julia_type(col::TColumn)
    for fld in fieldnames(TColumn)
        isfilled(col, fld) || continue
        return julia_type(getfield(col, fld))
    end
    NA
end


text:
module Day25

using AdventOfCode2019
using Combinatorics

function day25(input::String = readInput(joinpath(@__DIR__, "..", "data", "day25.txt")))
    program = parse.(Int, split(input, ","))
    return solve(program)
end

# This last puzzle was an interactive puzzle.
# It has been solved by manually exploring the map and collecting all the
# items which do not lead the game to end.
# After having collected all the items it uses simple brute-force to enter
# the "Pressure-Sensitive Floor" and collecting the password.
#
# This process has been hardcoded in that `solve` method, so it will only
# work for my particular input!
function solve(program::Array{Int,1})
    inp = Channel{Int}(100)
    out = Channel{Int}(1000)
    waiting = Channel{Bool}(1)
    @async AdventOfCode2019.IntCode.run_program!(copy(program), inp, out, nothing; waitingForInput = waiting)

    items = ["mutex", "whirled peas", "cake", "space law space brochure", "loom", "hologram", "manifold", "easter egg"]
    command("north", inp, out)
    command("take mutex", inp, out)
    command("east", inp, out)
    command("east", inp, out)
    command("east", inp, out)
    command("take whirled peas", inp, out)
    command("west", inp, out)
    command("west", inp, out)
    command("west", inp, out)
    command("south", inp, out)
    command("south", inp, out)
    command("take cake", inp, out)
    command("north", inp, out)
    command("west", inp, out)
    command("take space law space brochure", inp, out)
    command("north", inp, out)
    command("take loom", inp, out)
    command("south", inp, out)
    command("south", inp, out)
    command("take hologram", inp, out)
    command("west", inp, out)
    command("take manifold", inp, out)
    command("east", inp, out)
    command("north", inp, out)
    command("east", inp, out)
    command("south", inp, out)
    command("west", inp, out)
    command("south", inp, out)
    command("take easter egg", inp, out)
    command("south", inp, out)

    for combi in Combinatorics.combinations(items)
        drop_all(items, inp, out)
        take_items(combi, inp, out)
        command("south", inp, out; clear_output = false)
        while !isready(out)
            yield()
        end
        os = get_output(out)
        if occursin("heavier", os) || occursin("lighter", os)
            continue
        end
        # println(os)
        m = match(r"\d+", os)
        return parse(Int, m.match)
        break
    end

    # while true
    #     print("> "); n = readline()
    #     if n == "read"
    #         o = Int[]
    #         while isready(out)
    #             push!(o, take!(out))
    #         end
    #         print(join(Char.(o)))
    #         continue
    #     end
    #     if n == "exit" || n == "quit" || n == "q"
    #         return
    #     end
    #     input = push!([Int(x) for x in n], 10)
    #     for x in input
    #         put!(inp, x)
    #     end
    # end
end

function get_output(out::Channel{Int})
    o = Int[]
    while isready(out)
        push!(o, take!(out))
    end
    return join(Char.(o))
end

function command(c::String, inp::Channel{Int}, out::Channel{Int}; clear_output = true)
    commandInts = push!([Int(x) for x in c], 10)
    for x in commandInts
        put!(inp, x)
    end
    if clear_output
        while !isready(out)
            yield()
        end
        while isready(out)
            take!(out)
        end
    end
end

function drop_all(items::Array{String,1}, inp::Channel{Int}, out::Channel{Int})
    for item in items
        c = "drop " * item
        command(c, inp, out)
    end
end

function take_items(items::Array{String,1}, inp::Channel{Int}, out::Channel{Int})
    for item in items
        c = "take " * item
        command(c, inp, out)
    end
end

end # module


text:
# Define Order Fill mode and utilities
"""
    OrderTraits(allornone::Bool,immediateorcancel::Bool,allow_cross::Bool)

`OrderTraits` specify order traits which modify execution logic.

An instance can be initialized by using the keyword intializer or by using the exported constants
`VANILLA_FILLTYPE`, `IMMEDIATEORCANCEL_FILLTYPE`, `FILLORKILL_FILLTYPE`.

The default execution logic is represented by `VANILLA_FILLTYPE`.

__Note:__ This feature is not well supported yet.
Other than the constants described above, use non-vanilla modes with caution.

"""
Base.@kwdef struct OrderTraits
    allornone::Bool = false
    immediateorcancel::Bool = false
    allow_cross::Bool = true
end

isallornone(mode::OrderTraits) = mode.allornone
isimmediateorcancel(mode::OrderTraits) = mode.immediateorcancel
isfillorkill(mode::OrderTraits) = isallornone(mode)&&isimmediateorcancel(mode)
allows_book_insert(mode::OrderTraits) = !isimmediateorcancel(mode)
allows_partial_fill(mode::OrderTraits) = !isallornone(mode)
allows_cross(mode::OrderTraits) = mode.allow_cross

const VANILLA_FILLTYPE = OrderTraits(false,false,true)
const FILLORKILL_FILLTYPE = OrderTraits(true,true,true)
const IMMEDIATEORCANCEL_FILLTYPE = OrderTraits(false,true,true)
# const ALLORNONE_ORDER = OrderTraits(true,false,true)

Base.string(x::OrderTraits) =
    @sprintf("OrderTraits(allornone=%s,immediateorcancel=%s,allow_cross=%s)",x.allornone,x.immediateorcancel,x.allow_cross)
Base.print(io::IO, x::OrderTraits) = print(io, string(x))
Base.show(io::IO, ::MIME"text/plain", x::OrderTraits) = print(io, string(x))

# Define Order Side and utilities
"""
    OrderSide(is_buy::Bool)

Type representing whether an order is a buy order or sell order.
New instance can be generated with `OrderSide(::Bool)` or by using exported constants
`BUY_ORDER` and `SELL_ORDER`
"""
struct OrderSide
    is_buy::Bool
end

Base.string(x::OrderSide) = x.is_buy ? "OrderSide(Buy)" : "OrderSide(Sell)"
Base.print(io::IO, x::OrderSide) = print(io, string(x))
Base.show(io::IO, ::MIME"text/plain", x::OrderSide) = print(io, string(x))
Base.show(io::IO, x::OrderSide) = print(io, string(x))

isbuy(x::OrderSide) = x.is_buy
issell(x::OrderSide) = !x.is_buy

const BUY_ORDER = OrderSide(true)
const SELL_ORDER = OrderSide(false)

# Define Order, OrderQueue objects and relavant methods.
"""
    Order{Sz<:Real,Px<:Real,Oid<:Integer,Aid<:Integer}

Type representing a limit order.

An `Order{Sz<:Real,Px<:Real,Oid<:Integer,Aid<:Integer}` is a struct representing a resting Limit Order which contains
 - `side::OrderSide`, the side of the book the order will rest in. See [`OrderSide`](@ref) for more info.
 - `size::Sz`, the order size
 - `price::Px`, the price the order is set at
 - `orderid::Oid`, a unique Order ID
 - (optional) `acctid::Union{Aid,Nothing}`, which is set to nothing if the account is unknown or irrelevant.

One can create a new `Order` as

```
Order{Sz,Px,Pid,Aid}(side, size, price, orderid, order_mode [,acctid=nothing])
```

where the types of `size` and `price` will be cast to the correct types.
The `orderid` and `acctid` types will not be cast in order to avoid ambiguity.
"""
struct Order{Sz<:Real,Px<:Real,Oid<:Integer,Aid<:Integer}
    side::OrderSide
    size::Sz
    price::Px
    orderid::Oid
    acctid::Union{Aid,Nothing}
    function Order{Sz,Px,Oid,Aid}(
        side::OrderSide,
        size::Real,
        price::Real,
        orderid::Oid,
        acctid::Union{Aid,Nothing} = nothing,
    ) where {Sz<:Real,Px<:Real,Oid<:Integer,Aid<:Integer}
        new{Sz,Px,Oid,Aid}(side, Sz(size), Px(price), orderid, acctid) # cast price and size to correct types
    end
end

# Order utility functions
has_acct(o::Order) = isnothing(o.acctid)
isbuy(o::Order) = o.side.is_buy

function Base.show(io::IO,o::Order{Sz,Px,Oid,Aid}) where {Sz,Px,Oid,Aid}
    str_lst = [
        "Order{$Sz,$Px,$Oid,$Aid}(",
        "side=$(o.side),",
        "size=$(o.size),",
        "price=$(o.price),",
        "orderid=$(o.orderid),",
        "acctid=$(o.acctid)",
        ")"]
    join(io,str_lst," ")
end

function Base.print(io::IO,o::Order{Sz,Px,Oid,Aid}) where {Sz,Px,Oid,Aid}
    str_lst = [
        "Order{$Sz,$Px,$Oid,$Aid}(",
        "side=$(o.side),",
        "size=$(o.size),",
        "price=$(o.price),",
        "orderid=$(o.orderid),",
        "acctid=$(o.acctid)",
        ")"]
    join(io,str_lst)
end

# Orderbook State Saving Methods
function _order_to_csv(o::Order)
    @sprintf "LMT,%i,%s,%f,%f,%i" o.orderid o.side o.size o.price o.acctid
end

"Return new order with size modified"
copy_modify_size(o::Order{Sz,Px,Oid,Aid}, new_size::Sz) where {Sz,Px,Oid,Aid} =
    Order{Sz,Px,Oid,Aid}(o.side, new_size::Sz, o.price, o.orderid, o.acctid)

""""
OrderQueue is a queue of orders at a fixed price, implemented as a Deque/Vector.

OrderQueue.queue is a Vector (interpreted as double ended queue). Orders are added and removed via FIFO logic.
OrderQueue also keeps track of its contained volume in shares and orders

OrderQueue(price) Initializes an empty order queue at price
"""
struct OrderQueue{Sz<:Real,Px<:Real,Oid<:Integer,Aid<:Integer}
    price::Px # price at which queue is located
    queue::Vector{Order{Sz,Px,Oid,Aid}} # queue of orders as vector
    total_volume::Base.RefValue{Sz} # total volume in queue
    num_orders::Base.RefValue{Int64} # total size of queue
    # Initialize empty OrderQueue
    function OrderQueue{Sz,Px,Oid,Aid}(price::Px) where {Sz,Px,Oid,Aid}
        new{Sz,Px,Oid,Aid}(
            price,
            Vector{Order{Sz,Px,Oid,Aid}}(),
            Base.RefValue{Sz}(0),
            Base.RefValue{Int64}(0),
        )
    end
end

Base.length(q::OrderQueue,) = length(q.queue)
Base.iterate(q::OrderQueue,i=1) = iterate(q.queue,i)

# Insert, delete, push, pop orders into/out of OrderQueue
function Base.push!(
    oq::OrderQueue{Sz,Px,Oid,Aid},
    ord::Order{Sz,Px,Oid,Aid},
) where {Sz,Px,Oid,Aid}
    push!(oq.queue, ord)
    oq.total_volume[] += ord.size::Sz
    oq.num_orders[] += 1
end

function Base.pushfirst!(
    oq::OrderQueue{Sz,Px,Oid,Aid},
    ord::Order{Sz,Px,Oid,Aid},
) where {Sz,Px,Oid,Aid}
    pushfirst!(oq.queue, ord)
    oq.total_volume[] += ord.size::Sz
    oq.num_orders[] += 1
end

isequal_orderid(o::Order{<:Real,<:Real,Oid,<:Real}, this_id::Oid) where {Oid<:Integer} =
    o.orderid == this_id
order_id_match(order_id) = Base.Fix2(isequal_orderid, order_id)

@inline function _popat_orderid_internal!(
    oq::OrderQueue{Sz,Px,Oid,Aid},
    pop_id::Oid,
) where {Sz,Px,Oid,Aid}
    ret_ix = findfirst(order_id_match(pop_id), oq.queue)::Union{Int64,Nothing}
    return (
        isnothing(ret_ix) ? ret_ix::Nothing : popat!(oq.queue, ret_ix)::Order{Sz,Px,Oid,Aid}
    )::Union{Order{Sz,Px,Oid,Aid},Nothing}
end

"""
    popat_orderid!(oq::OrderQueue, orderid::Integer)

Pop Order with orderid from oq::OrderQueue.

Returns eiter
    popped order, updates queue statistics.
    `nothing` if orderid not found.

"""
function popat_orderid!(oq::OrderQueue{Sz,Px,Oid,Aid}, orderid::Oid) where {Sz,Px,Oid,Aid}
    ord = _popat_orderid_internal!(oq, orderid)
    if !isnothing(ord) # if order is returned, track stats
        oq.total_volume[] -= ord.size
        oq.num_orders[] -= 1
    end
    return ord::Union{Order{Sz,Px,Oid,Aid},Nothing}
end

function Base.popfirst!(oq::OrderQueue{Sz,Px,Oid,Aid}) where {Sz,Px,Oid,Aid}
    if isempty(oq)
        return nothing
    else
        ord = Base.popfirst!(oq.queue)
        oq.total_volume[] -= ord.size
        oq.num_orders[] -= 1
        return ord
    end
end

Base.isempty(oq::OrderQueue) = isempty(oq.queue)

function Base.print(io::IO, oq::OrderQueue)
    write(io::IO, "OrderQueue at price=$(oq.price):", "\n")
    for ord in oq.queue
        write(io::IO, " ")
        print(io::IO, ord)
        write(io::IO, "\n")
    end
end


text:
#=
# Tests vector unknown capability with a simple Poisson-like problem.
=#
if !@isdefined(Femshop)
    include("../Femshop.jl");
    using .Femshop
end
init_femshop("vector");

# Try making an optional log
useLog("vectorlog")

# Set up the configuration
domain(2)
functionSpace(order = 3)

# Specify the problem
mesh(QUADMESH, elsperdim=20)

u = variable("u", VECTOR)
p = variable("p")

testSymbol("v", VECTOR)
testSymbol("w")

boundary(u, 1, DIRICHLET, [0, 0])
boundary(p, 1, DIRICHLET, 0)

# Write the weak form
coefficient("f", ["-25*pi*pi*sin(pi*x)*sin(2*pi*y)", "-125*pi*pi*sin(3*pi*x)*sin(4*pi*y)"], VECTOR)
coefficient("g", "-2*pi*pi*sin(pi*x)*sin(pi*y)")
coefficient("a", 5)

weakForm([u,p], ["-a*inner(grad(u), grad(v)) - dot(f,v)", "-dot(grad(p), grad(w)) - g*w"])

solve([u,p]);

# # exact solution is [sin(pi*x)*sin(2*pi*y), sin(3*pi*x)*sin(4*pi*y)]
# # check error
erroru = zeros(size(u.values));
maxerru = 0
maxerrp = 0
exactu1(x,y) = sin(pi*x)*sin(2*pi*y);
exactu2(x,y) = sin(3*pi*x)*sin(4*pi*y);
exactp(x,y) = sin(pi*x)*sin(pi*y);

for i=1:size(Femshop.grid_data.allnodes,2)
    x = Femshop.grid_data.allnodes[1,i];
    y = Femshop.grid_data.allnodes[2,i];
    exac = [exactu1(x,y), exactu2(x,y)];
    for j=1:size(Femshop.grid_data.allnodes,1)
        erroru[j,i] = u.values[j,i] - exac[j];
        global maxerru;
        maxerru = max(abs(erroru[j,i]),maxerru);
    end
    global maxerrp;
    maxerrp = max(abs(p.values[i] - exactp(x,y)),maxerrp);
end
println("u max error = "*string(maxerru));
println("p max error = "*string(maxerrp));

# using Plots
# pyplot();
# display(plot(Femshop.grid_data.allnodes[1,:], Femshop.grid_data.allnodes[2,:], u.values[1,:], st=:surface))

# check
# log_dump_config();
# log_dump_prob();

output_values([u,p], "vector2d", format="vtk");

finalize_femshop();


text:
using QuNet
using LightGraphs, SimpleWeightedGraphs, GraphPlot, MetaGraphs, Plots, Colors, Statistics
using LaTeXStrings
using JLD
using Parameters

datafile = "data/bandwidth_wtih_memory"

function get_bandwidth(i)
    # Params
    num_trials = 5::Int64
    grid_size = 10::Int64
    time_depth = 8::Int64
    num_pairs = 10::Int64
    asynchronus_weight = 100::Int64
    raw_data = []

    # Generate ixi graph and extend it in time
    G = GridNetwork(grid_size, grid_size)

    # Extend graph in time without memory
    T = QuNet.TemporalGraph(G, time_depth, memory_prob=0.0)

    for j in 1:num_trials
    # Make a network with some probability of memory
        T_mem = QuNet.TemporalGraph(G, time_depth, memory_prob=i, memory_costs = unit_costvector())
        # Make a copy of the network without memory
        T_copy = deepcopy(T)

        # Get i random userpairs with asynchronus src and dst nodes.
        # mem_user_pairs = make_user_pairs(T_mem, num_pairs, src_layer=-1, dst_layer=-1)
        user_pairs = make_user_pairs(T, num_pairs, src_layer=-1, dst_layer=-1)

        # Add async nodes
        QuNet.add_async_nodes!(T_mem, user_pairs, ϵ=asynchronus_weight)
        QuNet.add_async_nodes!(T_copy, user_pairs, ϵ=asynchronus_weight)

        # Get pathset data
        # NOTE: the line below used to have mem_user_pairs as an argument.
        pathset_mem, dum1, dum2 = QuNet.greedy_multi_path!(T_mem, QuNet.purify, user_pairs, 1)
        pathset, dum1, dum2 = QuNet.greedy_multi_path!(T_copy, QuNet.purify, user_pairs, 1)
        # Pathset is an array of vectors containing edges describing paths between end-user pairs
        # Objective: find the largest timedepth used in the pathsets

        max_depth_mem = QuNet.max_timedepth(pathset_mem, T_mem)
        max_depth = QuNet.max_timedepth(pathset, T_copy)

        println(max_depth_mem)
        println(max_depth)

        # Get the bandwidth of this quantity
        push!(raw_data, max_depth / max_depth_mem)
    end
    return mean(raw_data)
end

println("collecting for 1.")
println(get_bandwidth(1.))

max_stars_repo_name: FalafelGood/QuNet

max_issues_repo_name: FalafelGood/QuNet

max_forks_repo_name: FalafelGood/QuNet


text:
random_stackvector(L) =  StackVector(rand(UInt) & (1 << (L&63) - 1), L)

@testset "Construction" begin
    @test_throws DomainError StackVector(zero(UInt), 65)

    @test StackVector([1, 0, 1, 1, 0, 1]) !== false

    @test StackVector([0, 1, 1, 0]) isa StackVector
    @test StackVector() === StackVector()
    toolong = rand(Bool, 65)
    @test_throws DomainError StackVector(toolong)
end

@testset "Basic" begin
    v_ = [0, 1, 1, 0, 1, 1, 1, 0, 0, 1]
    v = StackVector(v_)
    @test length(v) == 10
    @test size(v) == (10,)

    @test !isempty(v_)
    @test isempty(StackVector())
end

@testset "Get/set index" begin
    # Get index
    v_ = [0, 1, 1, 0, 1, 1, 1, 0, 0, 1]
    v = StackVector(v_)
    @test !v[1]
    @test v[2]
    @test !v[4]
    @test collect(v) == v_

    # Set index
    v2 = setindex(v, true, 4)
    @test v2[4]
    @test length(v) == length(v2)
end

@testset "Push/pop" begin
    v = StackVector()
    @test_throws ArgumentError pop(v)
    v = StackVector(fill(true, 64))
    @test_throws DomainError push(v, true)

    for L in [1, 3, 50, 63]
        for i in 1:3
            v = random_stackvector(L)
            v´ = collect(v)
            @test collect(pop(v)) == (v´´ = copy(v); pop!(v´´); v´´)
            @test collect(push(v, true)) == push!(copy(v´), true)
            @test collect(push(v, 0)) == push!(copy(v´), 0)
        end
    end
end

@testset "Iteration" begin
    for L in [0, 1, 3, 50]
        for i in 1:3
            v = random_stackvector(L)
            @test Bool[v[j] for j in 1:L] == Bool[j for j in v]
        end
    end
end

@testset "Misc" begin
    # In
    v = StackVector([true, 1, 0x01, 0x0000])
    @test (true in v)
    @test (false in v)

    v = StackVector([true, true, true, true])
    @test (true in v)
    @test !(false in v)

    # Inversion
    @test ~StackVector([0, 1, 1, 0]) == StackVector([1, 0, 0, 1])
    @test ~StackVector([0, 1, 0]) == StackVector([1, 0, 1])
end

@testset "Sum" begin
    for L in [0, 1, 3, 9, 64]
        for i in 1:3
            v = random_stackvector(L)
            @test sum(v) == sum(collect(v))
        end
    end
end

@testset "Min/max" begin
    for F in [argmin, argmax, minimum, maximum, findfirst,
             x -> findfirst(~, x), x -> findfirst(y -> true, x)]
        if F in [argmin, argmax, minimum, maximum]
            @test_throws ArgumentError F(StackVector())
        end
        for L in [1, 2, 3, 11, 64]
            for i in 1:10
                v = random_stackvector(L)
                @test F(v) == F(collect(v))
            end
        end
    end
end

@testset "Empty transformations" begin
    for f in [reverse, (x -> circshift(x, 3)), (x -> circshift(x, -301)), ~]
        @test f(StackVector()) === StackVector()
    end
end

@testset "Transformations" begin
    for F in [reverse, x -> circshift(x, 0), x -> circshift(x, -3),
             x -> circshift(x, -102), x -> circshift(x, 69), x -> circshift(x, 4)]
        for L in [1, 3, 10]
            for i in 1:3
                v = random_stackvector(L)
                @test collect(F(v)) == F(collect(v))
            end
        end
    end
end

@testset "Convert to bitvector" begin
    for L in [1, 2, 3, 11, 64]
        for i in 1:10
            v = random_stackvector(L)
            @test convert(BitVector, v) == BitVector(collect(v))
        end
    end
end


max_stars_repo_name: gabriel-doriath-dohler/pjulia

max_issues_repo_name: gabriel-doriath-dohler/pjulia

max_forks_repo_name: gabriel-doriath-dohler/pjulia


text:
@testset " Classic model forecasting " begin

    import DifferentialEquations: ODEProblem, solve, Tsit5

    σ = 10.0
    ρ = 28.0
    β = 8.0 / 3

    dt_integration = 0.01
    dt_states = 1
    dt_obs = 8
    parameters = [σ, ρ, β]
    var_obs = [1]
    nb_loop_train = 10^2
    nb_loop_test = 10
    sigma2_catalog = 0.0
    sigma2_obs = 2.0

    ssm = StateSpaceModel(
        lorenz63,
        dt_integration,
        dt_states,
        dt_obs,
        parameters,
        var_obs,
        nb_loop_train,
        nb_loop_test,
        sigma2_catalog,
        sigma2_obs,
    )

    # compute u0 at time = 5 to be in the attractor space

    u0 = [8.0; 0.0; 30.0]
    tspan = (0.0, 5.0)
    prob = ODEProblem(lorenz63, u0, tspan, parameters)
    u0 = last(solve(prob, Tsit5(), reltol = 1e-6, save_everystep = false))

    xt, yo, catalog = generate_data(ssm, u0)

    DA = DataAssimilation(ssm, xt)

    x̂ = forecast(DA, yo, AnEnKS(100), progress = false)

    rmse = RMSE(xt, x̂)

    println(" model forecasting RMSE : $rmse ")

    @test rmse < 1.0

end


text:
# ---------------------------------------------------------------------------- #
#=
 Testing computational speed of various Fast JL transforms
  Feb 11 2019
Y = S*X, X is M x N and S is m x M

Code by: Stephen Becker
Julia modifications: Jake Knigge
=#
# ---------------------------------------------------------------------------- #
# load packages
using Random, LinearAlgebra, Plots, Statistics, Hadamard, FFTW, SparseArrays
# ---------------------------------------------------------------------------- #
function Hadamard_teaching_code(x)
# Hadamard_teaching_code(x)
#   applies the Hadamard transform to x.
#   If x has more than one column, the transform is applied
#   to each column.
# This code is not fast, but it shows you how to exploit
#   the structure of the transform.
# Note: this code does not do any sub-sampling.
# Reference: https://en.wikipedia.org/wiki/Fast_Walsh–Hadamard_transform
  m = size(x,1); n = size(x,2);
  if nextpow(2,m) != m
      print("ERROR: The number of rows of x must be a power of 2.")
      return
  end
  y = copy(x)
    for bit in 1:log2(m)
        k   = Int64(2^bit);      # e.g., 2, 4, ..., m
        k2  = Int64(2^(bit-1));  # e.g., 1, 2, ..., m/2
        y   = reshape(y, k, :, n);
        tmp = y[1:k2,:,:];
        y[1:k2,:,:]       = y[1:k2,:,:] + y[k2+1:k,:,:];
        y[(k2+1):k,:,:]   = tmp         - y[k2+1:k,:,:];
        y   = reshape(y, m, n)
    end # loop
    return y
end # function
# ---------------------------------------------------------------------------- #
function slowCountSketch( DX, targetRows )
# slow version of count sketch
    m   = length( targetRows );
    Y   = zeros(m, size(DX,2) );
    for j in 1:size(DX,1)
        i  = targetRows[j];
        Y[i,:] = Y[i,:] + DX[j,:];
    end # loop
end # function
# ---------------------------------------------------------------------------- #
# test to generate an error
x = randn(65,5);
y = Hadamard_teaching_code(x)

# test to generate an error
x = randn(64,5);
y = Hadamard_teaching_code(x);
y_alt = 64*fwht_natural(x, 1);
norm(y - y_alt)
# Check Implementations for correctness
X = randn(2^13,100); M, N = size(X); m = M/4;

# == Hadamard Code ==
# Check for normalization and ordering
Y1 = Hadamard_teaching_code(Matrix{Float64}(I, 4, 4))
Y2 = 4*fwht_natural(Matrix{Float64}(I, 4, 4), 1)
norm(Y1 - Y2)
# Compare times---run more than once b/c of Julia's "just-in-time" compiler!
@time Y1 = Hadamard_teaching_code(X);
@time Y2 = M*fwht_natural(X, 1);
norm(Y1 - Y2)
# ---------------------------------------------------------------------------- #
# Test speed
N = 100;
M_list  = 2 .^(11:14)# (^).(2,10:13); # defined using "broadcast" operation
nTrials     = 10; # get some averages
nAlgos      = 6; # change to 7 if/when CountSketch is included
Times       = zeros(nAlgos,length(M_list),nTrials);
Times_setup = Times;
ALGO_NAMES = ["Gaussian","FJLT - DCT","FJLT - Hadamard", #"Count",
              "Very Very sparse","Very sparse","Sparse"];
rng = Random.seed!(9);
# ---------------------------------------------------------------------------- #
for Mi in 1:length( M_list )
  println("Mi is ", Mi, " of ", length(M_list), ".");
  for trial = 1:nTrials
    M   = M_list[Mi];
    m   = round(M/4);
    X   = randn(M,N);

    ALGO = 1; # Gaussian sketch
    tic = time();
    S = randn(Int64(m),M);
    Times_setup[ALGO,Mi,trial] = time() - tic;
    tic = time();
    Y   = S*X;
    Times[ALGO,Mi,trial] = time() - tic;

    ALGO = 2; # Fast JL, DCT
    tic     = time();
    D = spdiagm(0 => broadcast(sign,randn(M)) );
    ind     = rand(1:M, Int64(m));
    Times_setup[ALGO,Mi,trial] = time() - tic;
    tic = time();
    Y       = dct( D*X );
    Y       = Y[ind,:];
    Times[ALGO,Mi,trial] = time() - tic;

    ALGO = 3;  # Fast JL, Hadamard
    tic     = time();
    D = spdiagm(0 => broadcast(sign,randn(M)) );
    ind     = rand(1:M, Int64(m));
    Times_setup[ALGO,Mi,trial] = time() - tic;
    tic = time();
    Y       = M*fwht_natural( D*X, 1 );
    Y       = Y[ind,:];
    Times[ALGO,Mi,trial] = time() - tic;

    # ALGO = 4; # Count
    # tic     = time();
    # D = spdiagm(0 => broadcast(sign,randn(M)) );
    # indx_map        = Int64.(rand(1:m,M));
    # Times_setup[ALGO,Mi,trial] = time() - tic;
    # tic = time();
    # Y = slowCountSketch(X'*D,indx_map);
    # Times[ALGO,Mi,trial] = time() - tic;

    let ALGO = ALGO
    # Sparse. We can normalize later. Does that help speed?
      for Sparsity = 1:3
          ALGO        = ALGO + 1;
          s           = 2.0^(1-Sparsity)*sqrt(M);
          density     = 1/(2*s);
          ALGO_NAMES[ALGO] = string(round(100*density,digits=2), "% sparse");
          tic = time();
          S   = sprandn(Int64(m),M,density); # this takes longer than the multiply!
          S   = broadcast(sign, S);
          Times_setup[ALGO,Mi,trial]     = time() - tic;
          tic = time();
          Y   = sqrt(s)*(S*X);
          Times[ALGO,Mi,trial] = time() - tic;
      end # for loop for sparsity
    end # let block
  end # for loop for trials
end # for loop M list
# ---------------------------------------------------------------------------- #
# Plots
Data = Times;
mn  = reshape(mean(Data, dims = 3), nAlgos, length(M_list));
plot(M_list, mn', yscale = :log10, xscale = :log10, legend = :topleft,
        label = ALGO_NAMES, title = "times to apply sketch",
        titlefontsize = 10)
xlabel!("size M")
ylabel!("times in seconds")
y   = M_list/M_list[1];
plot!(M_list, mn[1].*y.^2, label = "M^2", linecolor = :black)
plot!(M_list, mn[3,1].*(M_list.*log.(M_list)/(M_list[1]*log.(M_list[1]))),
      label = "M log M", linecolor = :black, linestyle = :dash)
plot!(M_list, minimum(mn[:,1])*y, label = "M", linecolor = :black,
      linestyle = :dot)

# ---------------------------------------------------------------------------- #
Data = Times_setup + Times;
mn   = reshape(mean(Data,dims=3),nAlgos, length(M_list));;
plot(M_list, mn', yscale = :log10, xscale = :log10, legend = :topleft,
        label = ALGO_NAMES, title = "times to apply sketch with setup",
        titlefontsize = 10)
xlabel!("size M")
ylabel!("times in seconds")
y   = M_list/M_list[1];
plot!(M_list, mn[1].*y.^2, label = "M^2", linecolor = :black)
plot!(M_list, mn[3,1].*(M_list.*log.(M_list)/(M_list[1]*log.(M_list[1]))),
      label = "M log M", linecolor = :black, linestyle = :dash)
plot!(M_list, minimum(mn[:,1])*y, label = "M", linecolor = :black,
      linestyle = :dot)

# ---------------------------------------------------------------------------- #

max_stars_repo_name: jakeknigge/randomized-algorithm-class

max_issues_repo_name: jakeknigge/randomized-algorithm-class

max_forks_repo_name: jakeknigge/randomized-algorithm-class


text:
module HTbase

# using PyCall
# using PyPlot

include("directory.jl")
include("plot.jl")
greet() = print("Hello World!")

end # module


text:
module ThresholdSampling
  using MLLibrary
  using ..DataParser
  function getThresholdDataSetIndicies(file,size,threshold,isTest)
    number_remaining =size
    isTest ? multiplier = threshold : multiplier=1-threshold
    number_needed =convert(Int,round(number_remaining*(multiplier)))
    #  println(number_needed)
    dataset= zeros(Int,number_needed,1)
    probability = number_needed/number_remaining
    line=0;
    index =0;
    stream = DataParser.DataStream.getDataSet(file)
    for i in enumerate(eachline(stream))
      #println(i)
      if number_needed==0
        break;
      end

      line+=1
      number_remaining-=1
      random = rand()
      if isTest? random <= (probability): random > (1-probability)
        index+=1
        dataset[index]=line
        number_needed-=1
      end

      probability = number_needed/number_remaining

    end
    DataParser.DataStream.closeDataSet(stream)
    return dataset

  end
  #assume that srand() will be called! this is important
  function getThresholdTrainingSetIndicies(file,size,threshold)
    getThresholdDataSetIndicies(file,size,threshold,false)
  end

  function getThresholdTestSetIndices(file,size, threshold)
    getThresholdDataSetIndicies(file,size,threshold,true)
  end

  function getThresholdDataSetInMemory(file,indices,datatype)
  size = DataParser.DataStream.getDataSetCount(file)
  #indices = datasetFunction(file,size,threshold)
  featureCount = DataParser.DataStream.getFeatureCount(file);
  stream = DataParser.DataStream.getDataSet(file)
  dataset = cell(length(indices),featureCount)
  index=0;
  loc=1

  for line in eachline(stream)
    index+=1
    if in(index,indices)
      dataset[loc,:]= split(strip(line,'\n'),",")
      loc+=1
    end

  end
  #println(size(testset))

  closeDataSet(stream)

  return dataset
end

function getTestSet(file,delimiter,threshold,datatype=Any)
  count = MLLibrary.DataParser.DataStream.getDataSetCount(file)
  indices =getThresholdTestSetIndices(file,count,threshold)

  return MLLibrary.DataSampling.DataReader.getDataSetInMemoryFromIndices(file,delimiter,indices)

end

function getTrainingSet(file,delimiter,threshold,datatype=Any)
  count = MLLibrary.DataParser.DataStream.getDataSetCount(file)
  indices = getThresholdTrainingSetIndicies(file,count,threshold)
  return MLLibrary.DataSampling.DataReader.getDataSetInMemoryFromIndices(file,delimiter,indices)
end

  function getSets(file,delimiter,threshold,seed=1,datatype=Any)
    srand(seed)
    trainingset = getTrainingSet(file,delimiter,threshold,datatype)
    srand(seed)
    testset = getTestSet(file,delimiter,threshold,datatype)
    return trainingset, testset
  end

end

max_stars_repo_name: ajermaky/CustomMLLibrary

max_issues_repo_name: ajermaky/CustomMLLibrary

max_forks_repo_name: ajermaky/CustomMLLibrary


text: # test integral function inside [0,1] ball


text:
export addto!, broadcast_addto!

function addto!(dest::Array{T}, doffs::Int, src::Array{T}, soffs::Int, n::Int) where T
    @assert doffs > 0 && soffs > 0
    axpy!(n, T(1), pointer(src,soffs), 1, pointer(dest,doffs), 1)
    dest
end

function addto!(dest::CuArray{T}, doffs::Int, src::CuArray{T}, soffs::Int, n::Int) where T
    @assert doffs > 0 && soffs > 0
    axpy!(n, T(1), cupointer(src,soffs), 1, cupointer(dest,doffs), 1)
    dest
end

function addto!(dest::UniArray{T}, src::UniArray{T}) where T
    @assert length(dest) == length(src)
    addto!(dest, 1, src, 1, length(dest))
end

function addto!(dest::Array{T}, src::SubArray{T}) where T
    dest[:] = dest + src
    dest
end
function addto!(dest::SubArray{T}, src::Array{T}) where T
    dest[:] = dest + src
    dest
end

broadcast_addto!(dest::Array{T}, src::Array{T}) where T = dest .+= src
broadcast_addto!(dest::CuArray, src::CuArray) = CUDA.broadcast_addto!(dest, src)

@generated function addto!(dest::CuArray{T}, src::CuSubArray{T,N}) where {T,N}
    Ct = cstring(T)
    k = Kernel("""
    __global__ void addto($Ct *dest, Array<$Ct,$N> src) {
        int idx = blockIdx.x * blockDim.x + threadIdx.x;
        if (idx >= src.length()) return;
        dest[idx] += src(idx);
    }
    """)
    quote
        @assert length(dest) == length(src)
        src.linear && return addto!(dest, 1, src.parent, src.offset+1, length(dest))
        gdims, bdims = cudims(length(dest))
        $k(gdims, bdims, pointer(dest), src)
        dest
    end
end

@generated function addto!(dest::CuSubArray{T,N}, src::CuArray{T}) where {T,N}
    Ct = cstring(T)
    k = Kernel("""
    __global__ void addto(Array<$Ct,$N> dest, $Ct *src) {
        int idx = blockIdx.x * blockDim.x + threadIdx.x;
        if (idx >= dest.length()) return;
        dest(idx) += src[idx];
    }
    """)
    quote
        @assert length(dest) == length(src)
        dest.linear && return addto!(dest.parent, dest.offset+1, src, 1, length(src))
        gdims, bdims = cudims(length(dest))
        $k(gdims, bdims, dest, pointer(src))
        dest
    end
end