text:
export getVariableOrder, calcCliquesRecycled
export getCliquePotentials
export getClique, getCliques, getCliqueIds, getCliqueData
export hasClique
export setCliqueDrawColor!, getCliqueDrawColor
export appendSeparatorToClique!

export buildTreeFromOrdering! # TODO make internal and deprecate external use to only `buildTreeReset!``

"""
    $SIGNATURES

Set the color of a cliq in the Bayes (Junction) tree.
"""
function setCliqueDrawColor!(cliq::TreeClique, fillcolor::String)::Nothing
  cliq.attributes["fillcolor"] = fillcolor
  cliq.attributes["style"] = "filled"
  nothing
end

function getCliqueDrawColor(cliq::TreeClique)
  haskey(cliq.attributes, "fillcolor") ? cliq.attributes["fillcolor"] : nothing
end

"""
    $SIGNATURES

Get the frontal variable IDs `::Int` for a given clique in a Bayes (Junction) tree.
"""
getCliqFrontalVarIds(cliqdata::Union{BayesTreeNodeData, PackedBayesTreeNodeData})::Vector{Symbol} = cliqdata.frontalIDs
getCliqFrontalVarIds(cliq::TreeClique)::Vector{Symbol} = getCliqFrontalVarIds(getCliqueData(cliq))

"""
    $SIGNATURES

Get the frontal variable IDs `::Int` for a given clique in a Bayes (Junction) tree.
"""
getFrontals(cliqd::Union{TreeClique,BayesTreeNodeData})::Vector{Symbol} = getCliqFrontalVarIds(cliqd)

"""
    $SIGNATURES

Create a new clique.
"""
function addClique!(bt::MetaBayesTree,
                    dfg::AbstractDFG,
                    varID::Symbol,
                    condIDs::Array{Symbol}=Symbol[]  )
  bt.btid += 1 #increment cliqueID counter

  cId = CliqueId(bt.btid)

  clq = TreeClique(cId)
  setLabel!(clq, "")

  #TODO addClique!(bt, clq), can't we already have the parent here
  # if isa(bt.bt,GenericIncidenceList)
  #   Graphs.add_vertex!(bt.bt, clq)
  #   bt.cliques[bt.btid] = clq
  #   clId = bt.btid
  # if isa(bt.bt, MetaDiGraph)
  #   @assert MetaGraphs.add_vertex!(bt.bt, :clique, clq) "add_vertex! failed"
  #   clId = MetaGraphs.nv(bt.bt)
  #   MetaGraphs.set_indexing_prop!(bt.bt, clId, :cliqId, bt.btid)
  # else
  #   error("Oops, something went wrong when adding a clique to the tree")
  # end
  @assert MetaGraphs.add_vertex!(bt.bt, :clique, clq) "Error trying to addClique! - add_vertex! failed"
  MetaGraphs.set_indexing_prop!(bt.bt, MetaGraphs.nv(bt.bt), :cliqId, cId)

  appendClique!(bt, cId, dfg, varID, condIDs)
  return clq
end

"""
    $(SIGNATURES)

Return the TreeClique node object that represents a clique in the Bayes
(Junction) tree, as defined by one of the frontal variables `frt<:AbstractString`.

Notes
- Frontal variables only occur once in a clique per tree, therefore is a unique identifier.

Related:

getCliq, getTreeAllFrontalSyms
"""
getClique(tree::AbstractBayesTree, cId::Int) = tree.cliques[cId]
getClique(bt::AbstractBayesTree, frt::Symbol) = getClique(bt, bt.frontals[frt])
getClique(tree::MetaBayesTree, cId::CliqueId) = MetaGraphs.get_prop(tree.bt, tree.bt[:cliqId][cId], :clique)
getClique(tree::MetaBayesTree, cIndex::Int) = MetaGraphs.get_prop(tree.bt, cIndex, :clique)

Base.getindex(tr::AbstractBayesTree, afrontal::Union{Symbol,Int,CliqueId}) = getClique(tr, afrontal)

function Base.show(io::IO, cliq::TreeClique)
  printstyled(io, "TreeClique (id=$(cliq.id))\n", color=:blue)
  println(io, "  frontals:    ", getFrontals(cliq))
  println(io, "  separator:   ", getCliqSeparatorVarIds(cliq))
  println(io, "  status:      ", cliq.data.status)
  println(io, "  allmarginld: ", cliq.data.allmarginalized)
  println(io, "  potentials:  ", cliq.data.potentials)
  printstyled(io, bold=true, " messages\n")
  cliq.data.messages.upTx isa Nothing ? nothing : println(io, "    .upTx:     ", cliq.data.messages.upTx.belief |> keys)
  if 0 != length(cliq.data.messages.upRx)
    print(io, "    .upRx:     ")
    for (id, msg) in cliq.data.messages.upRx
      print(io, id, "=>", msg.belief |> keys)
    end
    println(io,)
  end
  cliq.data.messages.downTx isa Nothing ? nothing : println(io, "    .downTx:   ", cliq.data.messages.downTx.belief |> keys)
  cliq.data.messages.downRx isa Nothing ? nothing : println(io, "    .downRx:   ", cliq.data.messages.downRx.belief |> keys)
  nothing
end

Base.show(io::IO, ::MIME"text/plain", cliq::TreeClique) = show(io, cliq)

function DFG.ls(tr::AbstractBayesTree)
  ids = keys(tr.cliques) |> collect |> sort
  ret = Vector{Pair{Int, Vector{Symbol}}}(undef, length(ids))
  for (idx,id) in enumerate(ids)
    ret[idx] = (id=>getFrontals(tr.cliques[id]))
  end
  ret
end

function DFG.ls(tr::AbstractBayesTree, id::Union{Symbol, Int, CliqueId})
  cliq = getClique(tr, id)
  prnt = getParent(tr, cliq)
  chld = getChildren(tr, cliq)

  # build children list
  chll = Vector{Pair{Int, Vector{Symbol}}}()
  cp = (x->x.id.value).(chld) |> sortperm
  for ch in chld[cp]
    push!(chll, ch.id.value=>getFrontals(ch))
  end

  # NOTE prnt is Vector{TreeClique}
  prll = Vector{Pair{Int, Vector{Symbol}}}()
  for pr in prnt
    push!(prll, pr.id.value=>getFrontals(pr))
  end

  # experimental, return a NamedTuple for tree around specific cliques
  return (;parent=prll, children=chll)
end

function Base.show(io::IO, ntl::NamedTuple{(:parent, :children), Tuple{Vector{Pair{Int64, Vector{Symbol}}}, Vector{Pair{Int64, Vector{Symbol}}}}})
  printstyled(io, "IIF.show(::NamedTuple{..}) for Bayes tree\n", color=:blue)
  println(io, " (parent   = ", ntl.parent)
  println(io, "  children = ", ntl.children, ")")
  nothing
end

Base.show(io::IO, ::MIME"text/plain", ntl::NamedTuple{(:parent, :children), Tuple{Vector{Pair{Int64, Vector{Symbol}}}, Vector{Pair{Int64, Vector{Symbol}}}}}) = show(io, ntl)

"""
    $(SIGNATURES)
"""
function deleteClique!(tree::MetaBayesTree, clique::TreeClique)
  cId = clique.id
  @assert MetaGraphs.rem_vertex!(tree.bt, tree.bt[:cliqId][cId]) "rem_vertex! failed"
  foreach(frt->delete!(tree.frontals,frt), getFrontals(clique))
  return clique
end

deleteClique!(tree::MetaBayesTree, cId::CliqueId) = deleteClique!(tree, getClique(tree, cId))

isRoot(tree::MetaBayesTree, cliq::TreeClique) = isRoot(tree, cliq.id)
function isRoot(tree::MetaBayesTree, cliqId::CliqueId)
  length(MetaGraphs.inneighbors(tree.bt, tree.bt[:cliqId][cliqId])) == 0
end

"""
    $SIGNATURES

Return boolean on whether the frontal variable `frt::Symbol` exists somewhere in the `::BayesTree`.
"""
hasClique(bt::AbstractBayesTree, frt::Symbol) = haskey(bt.frontals, frt)

"""
    $SIGNATURES

Return depth in tree as `::Int`, with root as depth=0.

Related

getCliq
"""
function getCliqDepth(tree, cliq)::Int
  prnt = getParent(tree, cliq)
  if length(prnt) == 0
    return 0
  end
  return getCliqDepth(tree, prnt[1]) + 1
end
getCliqDepth(tree::AbstractBayesTree, sym::Symbol)::Int = getCliqDepth(tree, getClique(tree, sym))

getCliques(tree::AbstractBayesTree) = tree.cliques

function getCliques(tree::MetaBayesTree)
  d = Dict{Int,Any}()
  for (k,v) in tree.bt.vprops
    d[k] = v[:clique]
  end
  return d
end

getCliqueIds(tree::AbstractBayesTree) = keys(getCliques(tree))

function getCliqueIds(tree::MetaBayesTree)
  # MetaGraphs.vertices(tree.bt)
  keys(tree.bt.metaindex[:cliqId])
end

"""
    $SIGNATURES

Return reference to the clique data container.
"""
getCliqueData(cliq::TreeClique) = cliq.data
getCliqueData(tree::AbstractBayesTree, cId::Union{CliqueId, Int}) = getClique(tree, cId) |> getCliqueData

"""
    $SIGNATURES

Set the clique data container to a new object `data`.
"""
setCliqueData!(cliq::TreeClique, data::Union{PackedBayesTreeNodeData, BayesTreeNodeData}) = cliq.data = data
setCliqueData!(tree::AbstractBayesTree, cId::Int, data::Union{PackedBayesTreeNodeData, BayesTreeNodeData}) = setCliqueData!(getClique(tree, cId), data)

# #TODO
# addClique!(tree::AbstractBayesTree, parentCliqId::Int, cliq::TreeClique) = error("addClique!(tree::AbstractBayesTree, parentCliqId::Int, cliq::TreeClique) not implemented")
# updateClique!(tree::AbstractBayesTree, cliq::TreeClique) = error("updateClique!(tree::AbstractBayesTree, cliq::TreeClique)::Bool not implemented")
# deleteClique!(tree::AbstractBayesTree, cId::Int) =  error("deleteClique!(tree::AbstractBayesTree, cId::Int)::TreeClique not implemented")

"""
    $SIGNATURES

Generate the label for particular clique (used by graphviz for visualization).
"""
function makeCliqueLabel(dfg::G, bt::AbstractBayesTree, clqID::CliqueId)::String where G <: AbstractDFG
  clq = getClique(bt, clqID)
  flbl = ""
  clbl = ""
  for fr in getCliqueData(clq).frontalIDs
    flbl = string(flbl, DFG.getVariable(dfg,fr).label, ",")
  end
  for sepr in getCliqueData(clq).separatorIDs
    clbl = string(clbl, DFG.getVariable(dfg,sepr).label, ",")
  end
  setLabel!(clq, string(clqID,"| ", flbl, ": ", clbl))
end

"""
    $SIGNATURES

Add the separator for the newly created clique.
"""
function appendSeparatorToClique!(bt::AbstractBayesTree,
                                  clqID::CliqueId,
                                  seprIDs::Array{Symbol,1} )
  #
  union!(getCliqueData(bt, clqID).separatorIDs, seprIDs)
  nothing
end

"""
    $SIGNATURES

Add a new frontal variable to clique.

DevNotes
- TODO, define what "conditionals" are CLEARLY!!
"""
function appendClique!( bt::AbstractBayesTree,
                        clqID::CliqueId,
                        dfg::AbstractDFG,
                        varID::Symbol,
                        seprIDs::Array{Symbol,1}=Symbol[] )
  #
  clq = getClique(bt, clqID)
  var = DFG.getVariable(dfg, varID)

  # add frontal variable
  push!(getCliqueData(clq).frontalIDs, varID)

  # total dictionary of frontals for easy access
  bt.frontals[varID] = clqID

  # TODO - confirm this, append to cliq separator ??
  # @info "going for: appendSeparatorToClique on (clqID, seprIDs)=($clqID, $seprIDs)"
  appendSeparatorToClique!(bt, clqID, seprIDs)
  makeCliqueLabel(dfg, bt, clqID)
  return nothing
end

"""
    $SIGNATURES

Instantiate a new child clique in the tree.
"""
function newChildClique!( bt::AbstractBayesTree,
                          dfg::AbstractDFG,
                          CpID::CliqueId,
                          varID::Symbol,
                          Sepj::Array{Symbol,1})
  #
  # physically create the new clique
  chclq = addClique!(bt, dfg, varID, Sepj)
  parent = getClique(bt, CpID)

  if isa(bt.bt, MetaDiGraph)
    # TODO EDGE properties here
    @assert MetaGraphs.add_edge!(bt.bt, bt.bt[:cliqId][CpID], bt.bt[:cliqId][chclq.id]) "Add edge failed"
  else
    error("Oops, something went wrong when adding a new child clique to the tree")
  end

  return chclq
end

"""
    $SIGNATURES

Post order tree traversal and build potential functions
"""
function findCliqueFromFrontal(bt::AbstractBayesTree, frtlID::Int)
  for cliqPair in getCliques(bt, cliques)
    id = cliqPair[1]
    cliq = cliqPair[2]
    for frtl in getFrontals(cliq)
      if frtl == frtlID
        return cliq
      end
    end
  end
  error("Clique with desired frontal ID not found")
end

"""
    $SIGNATURES

Find parent clique Cp that containts the first eliminated variable of Sj as frontal.
"""
function identifyFirstEliminatedSeparator(dfg::AbstractDFG,
                                          elimorder::Vector{Symbol},
                                          firvert::DFGVariable,
                                          Sj=getSolverData(firvert).separator)::DFGVariable
  #
  firstelim = (2^(Sys.WORD_SIZE-1)-1)
  for s in Sj
    temp = something(findfirst(isequal(s), elimorder), 0) # findfirst(p, s)
    if (temp < firstelim)
      firstelim = temp
    end
  end
  DFG.getVariable(dfg, elimorder[firstelim])
end

"""
    $SIGNATURES

Eliminate a variable and add to tree cliques accordingly.

Dev Notes
- `p` should be elimination order.
- `var` is next variable to be added to the tree.
- TODO, make sure this works for disjoint graphs
  - Check laplacian, check eigen == 1 is disjoint sets

References
Kaess et al.: Bayes Tree, WAFR, 2010, [Alg. 2]
Kaess et al.: iSAM2, IJRR, 2011, [Alg. 3]
Fourie, D.: mmisam, PhD thesis, 2017. [Chpt. 5]
"""
function newPotential(tree::AbstractBayesTree, dfg::G, var::Symbol, elimorder::Array{Symbol,1}) where G <: AbstractDFG
  firvert = DFG.getVariable(dfg,var)
  # no parent
  if (length(getSolverData(firvert).separator) == 0)
    # if (length(getCliques(tree)) == 0)
      # create new root
      addClique!(tree, dfg, var)
    # else
    #   # add to root
    #   @warn "root append clique is happening"
    #   appendClique!(tree, 1, dfg, var)
    # end
  else
    # find parent clique Cp that containts the first eliminated variable of Sj as frontal
    Sj = getSolverData(firvert).separator
    felbl = identifyFirstEliminatedSeparator(dfg, elimorder, firvert, Sj).label
    # get clique id of first eliminated frontal
    CpID = tree.frontals[felbl]
    # look to add this conditional to the tree
    cliq = getClique(tree, CpID)
    # clique of the first eliminated frontal
    unFC = union(getCliqFrontalVarIds(cliq), getCliqSeparatorVarIds(cliq))
    # if the separator of this new variable is identical to the (entire) clique of the firstly eliminated frontal.
    if (sort(unFC) == sort(Sj))
      # just add new variable as frontal to this clique
      # insert conditional (p(var|sepr)) into clique CpID -- i.e. just adding a frontal
      # @info "adding new frontal $var to existing clique $CpID"
      appendClique!(tree, CpID, dfg, var)
    else
      # a new child clique is required here (this becomes parent)
      # @info "adding new child clique with parent separator."
      newChildClique!(tree, dfg, CpID, var, Sj)
    end
  end
end

"""
    $SIGNATURES

Build the whole tree in batch format.
"""
function buildTree!(tree::AbstractBayesTree,
                    dfg::AbstractDFG,
                    elimorder::AbstractVector{Symbol}  )
  #
  revorder = reverse(elimorder,dims=1) # fixing #499
  for var in revorder
    @debug "Adding $var to tree..."
    newPotential(tree, dfg, var, elimorder)
    prevVar = var
  end

  return tree
end

"""
    $SIGNATURES

Open view to see the graphviz exported Bayes tree, assuming default location and
viewer app.  See keyword arguments for more details.
"""
function showTree(; filepath::String="/tmp/caesar/bt.dot",
                    viewerapp::String="xdot"  )
  #
  try
    @async run(`$(viewerapp) $(filepath)`)
  catch ex
    @warn "not able to show via $(viewerapp) $(filepath)"
    @show ex
    @show stacktrace()
  end
end

# A replacement for _to_dot that saves only plotting attributes
function savedot_attributes(io::IO, g::MetaDiGraph)
    write(io, "digraph G {\n")
    for p in props(g)
        write(io, "$(p[1])=$(p[2]);\n")
    end

    for v in MetaGraphs.vertices(g)
        write(io, "$v")
        if length(props(g, v)) > 0
            write(io, " [ ")
        end
        for p in props(g, v)
            # key = p[1]
            # write(io, "$key=\"$(p[2])\",")
            for (k,v) in p[2]
              write(io, "\"$k\"=\"$v\",")
            end
        end
        if length(props(g, v)) > 0
            write(io, "];")
        end
        write(io, "\n")
    end

    for e in MetaGraphs.edges(g)
        write(io, "$(MetaGraphs.src(e)) -> $(MetaGraphs.dst(e)) [ ")
        if MetaGraphs.has_prop(g, e, :downMsg) && MetaGraphs.has_prop(g, e, :upMsg)
          if isready(MetaGraphs.get_prop(g, e, :downMsg))
            write(io, "color=red")
          elseif isready(MetaGraphs.get_prop(g, e, :upMsg))
            write(io, "color=orange")
          else
            write(io, "color=black")
          end
        end
        write(io, "]\n")
    end
    write(io, "}\n")
end

function _to_dot(mdigraph::MetaDiGraph)
  g = deepcopy(mdigraph)
  for (i,val) in g.vprops
    push!(g.vprops[i],:attributes=>val[:clique].attributes)
    delete!(g.vprops[i],:clique)
    delete!(g.vprops[i],:cliqId)
  end
  m = PipeBuffer()
  savedot_attributes(m, g)
  data = take!(m)
  close(m)
  return String(data)
end

"""
    $SIGNATURES

Draw the Bayes (Junction) tree by means of graphviz `.dot` files.  Ensure Linux packages
are installed `sudo apt-get install graphviz xdot`.

Notes
- `xlabels` is optional `cliqid=>xlabel`.
"""
function drawTree(treel::AbstractBayesTree;
                  show::Bool=false,                  # must remain false for stability and automated use in solver
                  suffix::AbstractString="_"*(split(string(uuid1()),'-')[1]),
                  filepath::String="/tmp/caesar/random/bt$(suffix).dot",
                  xlabels::Dict{Int,String}=Dict{Int,String}(),
                  dpi::Int=200,
                  viewerapp::String="xdot",
                  imgs::Bool=false )
  #
  fext = filepath[(end-2):end] #split(filepath, '.')[end]
  fpwoext = filepath[1:(end-4)]# split(filepath, '.')[end-1]
  path = mkpath(dirname(fpwoext))

  # modify a deepcopy
  btc = deepcopy(treel)
  for (cid, cliq) in getCliques(btc)
    if imgs
      firstlabel = split(getLabel(cliq),',')[1]
      spyCliqMat(cliq, suppressprint=true) |> exportimg(path*"/$firstlabel$suffix.png")
      cliq.attributes["image"] = path*"/$firstlabel$suffix.png"
      setLabel!(cliq, "")
    end
    delete!(cliq.attributes, "data")
  end

  # add any xlabel info
  for (cliqid, xlabel) in xlabels
    btc.bt.vertices[cliqid].attributes["xlabel"] = xlabel
  end

  fid = IOStream("")
  try
    fid = open("$(fpwoext).dot","w+")
    write(fid,_to_dot(btc.bt))
    close(fid)
    if string(fext) == "png"
      run(`dot $(fpwoext).dot -T $(fext) -Gdpi=$dpi -o $(fpwoext).pdf`)
    else
      run(`dot $(fpwoext).dot -T $(fext) -o $(fpwoext).pdf`)
    end
  catch ex
    @warn ex
    @show stacktrace()
  finally
    close(fid)
  end

  show ? showTree(viewerapp=viewerapp, filepath=filepath) : nothing
end

"""
    $SIGNATURES

If opt.drawtree then start an async task to draw tree in a loop according to opt.drawtreerate.

Notes
- wont draw if opt.drawtree=false, just skips back to caller.
- Currently @async
- use `opt.showtree::Bool`
- Does not work too well when opt.async during solveTree! call, but user can use this function separately.

DevNotes
- TODO, use Threads.@spawn instead.

Related

drawTree, drawGraph
"""
function drawTreeAsyncLoop( tree::AbstractBayesTree,
                            opt::SolverParams;
                            filepath=joinLogPath(opt,"bt.dot"),
                            dotreedraw = Int[1;]  )
  #
  # single drawtreerate
  treetask = if opt.drawtree
    @async begin
      xlabels = Dict{Int,String}()
      @info("Solve is drawing the Bayes tree")
      while dotreedraw[1] == 1 && 0 < opt.drawtreerate
        # actually draw the tree
        drawTree(tree,show=false,filepath=filepath)
        sleep(1/opt.drawtreerate)
      end
      drawTree(tree,show=opt.showtree,filepath=filepath)
    end
  end
  return treetask, dotreedraw
end

"""
    $SIGNATURES

Draw the Bayes (junction) tree with LaTeX labels by means of `.dot` and `.tex`
files.

Notes
- Uses system install of graphviz.org.
- Uses external python `dot2tex` tool (`pip install dot2tex`).

Related:

drawTree
"""
function generateTexTree( treel::AbstractBayesTree;
                          filepath::String="/tmp/caesar/bayes/bt")
  #
  btc = deepcopy(treel)
  for (cid, cliq) in getCliques(btc)
    label = getLabel(cliq)

    # Get frontals and separator, and split into elements.
    frt, sep = split(label,':')
    efrt = split(frt,',')
    esep = split(sep,',')

    # Transform frontals into latex.
    newfrontals = ""
    for l in efrt
      # Split into symbol and subindex (letter and number).
      parts = split(l, r"[^a-z0-9]+|(?<=[a-z])(?=[0-9])|(?<=[0-9])(?=[a-z])")
      if size(parts)[1] == 2
        newfrontals = string(newfrontals, "\\bm{", parts[1], "}_{", parts[2], "}, ")
      elseif size(parts)[1] == 3
        newfrontals = string(newfrontals, "\\bm{", parts[2], "}_{", parts[3], "}, ")
      end
    end
    # Remove the trailing comma.
    newfrontals = newfrontals[1:end-2]

    # Transform separator into latex.
    newseparator = ""
    if length(sep) > 1
      for l in esep
        # Split into symbol and subindex.
        parts = split(l, r"[^a-z0-9]+|(?<=[a-z])(?=[0-9])|(?<=[0-9])(?=[a-z])")
        if size(parts)[1] == 2
          newseparator = string(newseparator, "\\bm{", parts[1], "}_{", parts[2], "}, ")
        elseif size(parts)[1] == 3
          newseparator = string(newseparator, "\\bm{", parts[2], "}_{", parts[3], "}, ")
        end
      end
    end
    # Remove the trailing comma.
    newseparator = newseparator[1:end-2]
    # Create full label and replace the old one.
    newlabel = string(newfrontals, ":", newseparator)
    setLabel!(cliq, newlabel)
  end

  # Use new labels to produce `.dot` and `.tex` files.
  fid = IOStream("")
  try
    mkpath(joinpath((split(filepath,'/')[1:(end-1)])...) )
    fid = open("$(filepath).dot","w+")
    write(fid, _to_dot(btc.bt))
    close(fid)
    # All in one command.
    run(`dot2tex -tmath --preproc $(filepath).dot -o $(filepath)proc.dot`)
    run(`dot2tex $(filepath)proc.dot -o $(filepath).tex`)
  catch ex
    @warn ex
    @show stacktrace()
  finally
    close(fid)
  end

  return btc
end

"""
    $SIGNATURES

Build Bayes/Junction/Elimination tree from a given variable ordering.

DevNotes
- TODO use `solvable` filter during local graph copy step
- TODO review `buildCliquePotentials` and rather incorporate into CSM, see #1083

Related

[`buildTreeReset!`](@ref)
"""
function buildTreeFromOrdering!(dfg::DFG.AbstractDFG,
                                elimOrder::Vector{Symbol};
                                drawbayesnet::Bool=false,
                                solvable::Int=1  )
  #
  @debug "Building Bayes tree with local DFG copy"
  t0 =time_ns()
  fge = LocalDFG(solverParams=getSolverParams(dfg))

  #TODO JT - I think an optional solvable filter is needed in buildTreeFromOrdering!
  # copy required for both remote and local graphs
  DFG.deepcopyGraph!(fge, dfg)

  println("Building Bayes net...")
  buildBayesNet!(fge, elimOrder, solvable=solvable)

  tree = BayesTree()
  tree.eliminationOrder = elimOrder
  buildTree!(tree, fge, elimOrder)

  if drawbayesnet
    println("Bayes Net")
    sleep(0.1)
    fid = open("bn.dot","w+")
    write(fid,_to_dot(fge.bn))
    close(fid)
  end

  println("Find potential functions for each clique")
  for cliqIds in getCliqueIds(tree)
    # start at the root, of which there could be multiple disconnected trees
    if isRoot(tree, cliqIds)
      cliq = getClique(tree, cliqIds)
      # fg does not have the marginals as fge does
      buildCliquePotentials(dfg, tree, cliq, solvable=solvable)
    end
  end

  # also store the build time
  tree.buildTime = (time_ns()-t0)*1e-9

  return tree
end

"""
    $SIGNATURES

Build Bayes/Junction/Elimination tree.

Notes
- Default to free qr factorization for variable elimination order.

DevNotes
- TODO deprecate and update to better name than `drawpdf`
"""
function prepBatchTreeOLD!( dfg::AbstractDFG;
                            eliminationOrder::Union{Nothing, Vector{Symbol}}=nothing,
                            variableOrder::Union{Nothing, Vector{Symbol}}=nothing,
                            eliminationConstraints::Vector{Symbol}=Symbol[],
                            variableConstraints::Union{Nothing, Vector{Symbol}}=nothing,
                            ordering::Symbol= 0==length(variableConstraints) ? :qr : :ccolamd,
                            drawpdf::Bool=false,
                            show::Bool=false,
                            filepath::String="/tmp/caesar/random/bt.dot",
                            viewerapp::String="xdot",
                            imgs::Bool=false )
                            # drawbayesnet::Bool=false )
  #

  # deprecations on keywords
  if variableOrder !== nothing
    @warn "variableOrder keyword is deprecated, use buildTreeReset!(dfg, vo; kwargs...) instead."
    eliminationOrder = variableOrder
  end
  if variableConstraints !== nothing
    @warn "variableConstraints keyword is deprecated, use eliminationConstraints instead."
    eliminationConstraints = variableConstraints
  end

  p = eliminationOrder !== nothing ? eliminationOrder : getEliminationOrder(dfg, ordering=ordering, constraints=eliminationConstraints)

  # for debuggin , its useful to have the elimination ordering
  if drawpdf
    ispath(getLogPath(dfg)) ? nothing : Base.mkpath(getLogPath(dfg))
    open(joinLogPath(dfg,"eliminationOrder.txt"), "a") do io
      writedlm(io, string.(reshape(p,1,:)), ',')
    end
  end

  tree = buildTreeFromOrdering!(dfg, p, drawbayesnet=false) # drawbayesnet

  @info "Bayes Tree Complete"
  if drawpdf
    drawTree(tree, show=show, filepath=filepath, viewerapp=viewerapp, imgs=imgs)
  end

  return tree
end

"""
    $SIGNATURES

Partial reset of basic data fields in `::VariableNodeData` of `::FunctionNode` structures.
"""
function resetData!(vdata::VariableNodeData)
  vdata.eliminated = false
  vdata.BayesNetOutVertIDs = Symbol[]
  # vdata.BayesNetVertID = :_null # TODO dont use nothing, see DFG issue #16
  vdata.separator = Symbol[]
  nothing
end

function resetData!(vdata::FunctionNodeData)
  vdata.eliminated = false
  vdata.potentialused = false
  nothing
end

"""
    $SIGNATURES

Wipe data from `dfg` object so that a completely fresh Bayes/Junction/Elimination tree
can be constructed.
"""
function resetFactorGraphNewTree!(dfg::AbstractDFG)
  for v in DFG.getVariables(dfg)
    resetData!(getSolverData(v))
  end
  for f in DFG.getFactors(dfg)
    resetData!(getSolverData(f))
  end
  nothing
end

"""
    $(SIGNATURES)

Build a completely new Bayes (Junction) tree, after first wiping clean all
temporary state in fg from a possibly pre-existing tree.

DevNotes
- replaces `resetBuildTreeFromOrder!`

Related:

buildTreeFromOrdering!,
"""
function buildTreeReset!( dfg::AbstractDFG,
                          eliminationOrder::Union{Nothing, <:AbstractVector{Symbol}}=nothing;
                          variableOrder::Union{Nothing, <:AbstractVector{Symbol}}=nothing,
                          ordering::Symbol=:qr,
                          drawpdf::Bool=false,
                          show::Bool=false,
                          filepath::String="/tmp/caesar/random/bt.dot",
                          viewerapp::String="xdot",
                          imgs::Bool=false,
                          ensureSolvable::Bool=true,
                          eliminationConstraints::AbstractVector{Symbol}=Symbol[],
                          variableConstraints=nothing  )
  #
  if ensureSolvable
    ensureSolvable!(dfg)
  end
  # depcrecation
  if variableOrder !== nothing
    @warn "variableOrder keyword is deprecated, use buildTreeReset!(dfg, vo; kwargs...) instead."
    eliminationOrder = variableOrder
  end
  if variableConstraints !== nothing
    @warn "variableConstraints keyword is deprecated, use eliminationConstraints instead."
    eliminationConstraints = variableConstraints
  end

  resetFactorGraphNewTree!(dfg);
  return prepBatchTreeOLD!(dfg, eliminationOrder=eliminationOrder, ordering=ordering, drawpdf=drawpdf, show=show, filepath=filepath, viewerapp=viewerapp, imgs=imgs, eliminationConstraints=eliminationConstraints);
end

"""
    $(SIGNATURES)
Experimental create and initialize tree message channels
"""
function initTreeMessageChannels!(tree::MetaBayesTree)
  for e = MetaGraphs.edges(tree.bt)
    set_props!(tree.bt, e, Dict{Symbol,Any}(:upMsg=>Channel{LikelihoodMessage}(0),:downMsg=>Channel{LikelihoodMessage}(0)))
    # push!(tree.messageChannels, e=>(upMsg=Channel{LikelihoodMessage}(0),downMsg=Channel{LikelihoodMessage}(0)))
  end
  return nothing
end

"""
    $SIGNATURES

Returns state of Bayes tree clique `.initialized` flag.

Notes:
- used by Bayes tree clique logic.
- similar method in DFG
"""
isInitialized(cliq::TreeClique) = getSolverData(cliq).initialized

function appendUseFcts!(usefcts,
                        lblid::Symbol,
                        fct::DFGFactor )
                        # fid::Symbol )
  #
  union!(usefcts, Symbol(fct.label))
  nothing
end

"""
    $SIGNATURES

Get all factors connected to frontal variables

Dev Notes
- Why not just do this, `ffcs = union( map(x->ls(fgl, x), frtl) )`?
"""
function getCliqFactorsFromFrontals(fgl::G,
                                    cliq::TreeClique,
                                    varlist::Vector{Symbol};
                                    inseparator::Bool=true,
                                    unused::Bool=true,
                                    solvable::Int=1  ) where {G <: AbstractDFG}
  #
  frtls = getCliqueData(cliq).frontalIDs
  seprs = getCliqueData(cliq).separatorIDs
  allids = [frtls;seprs]

  usefcts = Symbol[]
  for frsym in frtls
    # usefcts = Int[]
    for fctid in ls(fgl, frsym)
        fct = getFactor(fgl, fctid)
        if !unused || !getSolverData(fct).potentialused
            loutn = ls(fgl, fctid, solvable=solvable)
            # deal with unary factors
            if length(loutn)==1
                union!(usefcts, Symbol[Symbol(fct.label);])
                # appendUseFcts!(usefcts, loutn, fct) # , frsym)
                getSolverData(fct).potentialused = true
            end
            # deal with n-ary factors
            for sep in loutn
                if sep == frsym
                    continue # skip the frsym itself
                end
                insep = sep in allids
                if !inseparator || insep
                    union!(usefcts, Symbol[Symbol(fct.label);])
                    getSolverData(fct).potentialused = true
                    if !insep
                        @debug "cliq=$(cliq.id) adding factor that is not in separator, $sep"
                    end
                end
            end
        end
    end
  end

  return usefcts
end

"""
    $SIGNATURES

Return `::Bool` on whether factor is a partial constraint.
"""
isPartial(fcf::T) where {T <: AbstractFactor} = :partial in fieldnames(T)
isPartial(ccw::CommonConvWrapper) = ccw.usrfnc! |> isPartial
isPartial(fct::DFGFactor) = _getCCW(fct) |> isPartial

"""
    $SIGNATURES

Determine and set the potentials for a particular `cliq` in the Bayes (Junction) tree.
"""
function setCliqPotentials!(dfg::G,
                            bt::AbstractBayesTree,
                            cliq::TreeClique;
                            solvable::Int=1  ) where G <: AbstractDFG
  #
  varlist = getCliqVarIdsAll(cliq)

  @debug "using all factors connected to frontals and attached to separator"
  fctsyms = getFactorsAmongVariablesOnly(dfg, varlist, unused=true )
  # filter only factors connected to frontals (for upward)
  frtfcts = union(map(x->ls(dfg, x), getCliqFrontalVarIds(cliq))...)
  fctsyms = intersect(fctsyms, frtfcts)

  getCliqueData(cliq).potentials = fctsyms
  getCliqueData(cliq).partialpotential = Vector{Bool}(undef, length(fctsyms))
  fcts = map(x->getFactor(dfg, x), fctsyms)
  getCliqueData(cliq).partialpotential = map(x->isPartial(x), fcts)
  for fct in fcts
    getSolverData(fct).potentialused = true
  end

  @debug "finding all frontals for down WIP"
  ffctsyms = getCliqFactorsFromFrontals(dfg, cliq, Symbol[], inseparator=false, unused=false, solvable=solvable)
  # fnsyms = getCliqVarsWithFrontalNeighbors(csmc.dfg, csmc.cliq)
  getCliqueData(cliq).dwnPotentials = ffctsyms
  getCliqueData(cliq).dwnPartialPotential = map(x->isPartial(getFactor(dfg,x)), ffctsyms )

  nothing
end

getCliquePotentials(cliq::TreeClique) = getCliqueData(cliq).potentials

function cliqPotentialIDs(cliq::TreeClique)
  potIDs = Symbol[]
  for idfct in getCliqueData(cliq).potentials
    push!(potIDs, idfct)
  end
  return potIDs
end

"""
    $SIGNATURES

Collect and returl all child clique separator variables.
"""
function collectSeparators(bt::AbstractBayesTree, cliq::TreeClique)::Vector{Symbol}
  allseps = Symbol[]
  for child in childCliqs(bt, cliq)#tree
      allseps = [allseps; getCliqueData(child).separatorIDs]
  end
  return allseps
end

"""
    $SIGNATURES

Return boolean matrix of factor by variable (row by column) associations within
clique, corresponds to order presented by `getCliqFactorIds` and `getCliqAllVarIds`.
"""
function getCliqAssocMat(cliq::TreeClique)
  getCliqueData(cliq).cliqAssocMat
end

"""
    $SIGNATURES

Return boolean matrix of upward message singletons (i.e. marginal priors) from
child cliques.  Variable order corresponds to `getCliqAllVarIds`.
"""
getCliqMsgMat(cliq::TreeClique) = getCliqueData(cliq).cliqMsgMat

"""
    $SIGNATURES

Return boolean matrix of factor variable associations for a clique, optionally
including (`showmsg::Bool=true`) the upward message singletons.  Variable order
corresponds to `getCliqAllVarIds`.
"""
function getCliqMat(cliq::TreeClique; showmsg::Bool=true)
  assocMat = getCliqAssocMat(cliq)
  msgMat = getCliqMsgMat(cliq)
  mat = showmsg ? [assocMat;msgMat] : assocMat
  return mat
end

"""
    $SIGNATURES

Get `cliq` separator (a.k.a. conditional) variable ids`::Symbol`.
"""
getCliqSeparatorVarIds(cliqdata::Union{BayesTreeNodeData, PackedBayesTreeNodeData})::Vector{Symbol} = cliqdata.separatorIDs
getCliqSeparatorVarIds(cliq::TreeClique)::Vector{Symbol} = getCliqSeparatorVarIds(getCliqueData(cliq))

"""
    $SIGNATURES

Get `cliq` potentials (factors) ids`::Int`.
"""
getCliqFactorIds(cliqdata::BayesTreeNodeData)::Vector{Symbol} = cliqdata.potentials
getCliqFactorIds(cliq::TreeClique)::Vector{Symbol} = getCliqFactorIds(getCliqueData(cliq))

"""
    $SIGNATURES

Get all `cliq` variable ids`::Symbol`.

Related

getCliqVarIdsAll, getCliqAllFactIds, getCliqVarsWithFrontalNeighbors
"""
function getCliqAllVarIds(cliq::TreeClique)::Vector{Symbol}
  frtl = getCliqFrontalVarIds(cliq)
  cond = getCliqSeparatorVarIds(cliq)
  union(frtl,cond)
end

"""
    $SIGNATURES

Return all variables (including frontal factor neighbors).

Dev Notes
- TODO needs to be refactored and optimized.

Related

getCliqAllVarIds
"""
function getCliqVarsWithFrontalNeighbors(fgl::G,
                                         cliq::TreeClique;
                                         solvable::Int=1  ) where {G <: AbstractDFG}
  #
  frtl = getCliqFrontalVarIds(cliq)
  cond = getCliqSeparatorVarIds(cliq)
  syms = Symbol[]
  union!(syms,Symbol.(frtl))
  union!(syms,Symbol.(cond))

  # TODO Can we trust factors are frontal connected?
  ffcs = union( map(x->ls(fgl, x, solvable=solvable), frtl)... )
  # @show ffcs = getCliqueData(cliq).potentials
  neig = union( map(x->ls(fgl, x, solvable=solvable), ffcs)... )
  union!(syms, Symbol.(neig))
  return syms
end

"""
    $SIGNATURES

Get all `cliq` variable ids`::Symbol`.

Related

getCliqAllVarIds, getCliqAllFactIds
"""
getCliqVarIdsAll(cliq::TreeClique)::Vector{Symbol} = getCliqAllVarIds(cliq::TreeClique)

"""
    $SIGNATURES

Get all `cliq` factor ids`::Symbol`.

DEPRECATED, use getCliqFactorIdsAll instead.

Related

getCliqVarIdsAll, getCliqFactors
"""
getCliqFactorIdsAll(cliqd::BayesTreeNodeData) = cliqd.potentials
getCliqFactorIdsAll(cliq::TreeClique) = getCliqFactorIdsAll(getCliqueData(cliq))
getCliqFactorIdsAll(treel::AbstractBayesTree, frtl::Symbol) = getCliqFactorIdsAll(getClique(treel, frtl))

const getCliqFactors = getCliqFactorIdsAll

"""
    $SIGNATURES

Get all `cliq` factor ids`::Symbol`.

DEPRECATED, use getCliqFactorIdsAll instead.

Related

getCliqVarIdsAll
"""
function getCliqAllFactIds(cliqd::BayesTreeNodeData)
    @warn "getCliqAllFactIds deprecated, use getCliqFactorIdsAll instead."
    return getCliqFactorIdsAll(cliqd)
end

function getCliqAllFactIds(cliq::TreeClique)
    @warn "getCliqAllFactIds deprecated, use getCliqFactorIdsAll instead."
    return getCliqFactorIdsAll(getCliqueData(cliq))
end

"""
    $SIGNATURES

Get all `cliq` variable labels as `::Symbol`.
"""
function getCliqAllVarSyms(dfg::G, cliq::TreeClique)::Vector{Symbol} where G <: AbstractDFG
  # Symbol[getSym(dfg, varid) for varid in getCliqAllVarIds(cliq)]
  @warn "deprecated getCliqAllVarSyms, use getCliqAllVarIds instead."
  getCliqAllVarIds(cliq) # not doing all frontals
end

"""
    $SIGNATURES

Return the number of factors associated with each variable in `cliq`.
"""
getCliqNumAssocFactorsPerVar(cliq::TreeClique)::Vector{Int} = sum(getCliqAssocMat(cliq), dims=1)[:]

"""
    $SIGNATURES

Get variable ids`::Int` with prior factors associated with this `cliq`.

Notes:
- does not include any singleton messages from upward or downward message passing.
"""
function getCliqVarIdsPriors( cliq::TreeClique,
                              allids::Vector{Symbol}=getCliqAllVarIds(cliq),
                              partials::Bool=true  )
  # get ids with prior factors associated with this cliq
  amat = getCliqAssocMat(cliq)
  prfcts = sum(amat, dims=2) .== 1

  # remove partials as per request
  !partials ? nothing : (prfcts .&= getCliqueData(cliq).partialpotential)

  # return variable ids in `mask`
  mask = sum(amat[prfcts[:],:], dims=1)[:] .> 0
  return allids[mask]::Vector{Symbol}
end

"""
    $SIGNATURES

Get `cliq` variable IDs with singleton factors -- i.e. both in clique priors and up messages.
"""
function getCliqVarSingletons(cliq::TreeClique,
                              allids::Vector{Symbol}=getCliqAllVarIds(cliq),
                              partials::Bool=true  )
  # get incoming upward messages (known singletons)
  mask = sum(getCliqMsgMat(cliq),dims=1)[:] .>= 1
  upmsgids = allids[mask]

  # get ids with prior factors associated with this cliq
  prids = getCliqVarIdsPriors(cliq, getCliqAllVarIds(cliq), partials)

  # return union of both lists
  return union(upmsgids, prids)::Vector{Symbol}
end

"""
    $SIGNATURES

Get each clique subgraph association matrix.
"""
function compCliqAssocMatrices!(dfg::G, bt::AbstractBayesTree, cliq::TreeClique) where G <: AbstractDFG
  frtl = getCliqFrontalVarIds(cliq)
  cond = getCliqSeparatorVarIds(cliq)
  inmsgIDs = collectSeparators(bt, cliq)
  potIDs = cliqPotentialIDs(cliq)
  # Construct associations matrix here
  # matrix has variables are columns, and messages/constraints as rows
  cols = [frtl;cond]
  getCliqueData(cliq).inmsgIDs = inmsgIDs
  getCliqueData(cliq).potIDs = potIDs

  @debug "Building cliqAssocMat" cliq
  @debug "Building cliqAssocMat" cliq.id string(inmsgIDs) string(potIDs)

  cliqAssocMat = Array{Bool,2}(undef, length(potIDs), length(cols))
  cliqMsgMat = Array{Bool,2}(undef, length(inmsgIDs), length(cols))
  fill!(cliqAssocMat, false)
  fill!(cliqMsgMat, false)
  for j in 1:length(cols)
    for i in 1:length(inmsgIDs)
      if cols[j] == inmsgIDs[i]
        cliqMsgMat[i,j] = true
      end
    end
    for i in 1:length(potIDs)
      idfct = getCliqueData(cliq).potentials[i]
      if idfct == potIDs[i] # sanity check on clique potentials ordering
        # TODO int and symbol compare is no good
        for vertidx in getVariableOrder(DFG.getFactor(dfg, idfct))
          if vertidx == cols[j]
            cliqAssocMat[i,j] = true
          end
        end
      else
        @error("compCliqAssocMatrices! -- potential ID ordering was lost")
      end
    end
  end
  @debug "Final cliqAssocMat" cliq.id cliqAssocMat
  getCliqueData(cliq).cliqAssocMat = cliqAssocMat
  getCliqueData(cliq).cliqMsgMat = cliqMsgMat
  nothing
end

function countSkips(bt::AbstractBayesTree)
  skps = 0
  for cliq in getCliques(bt)
    m = getCliqMat(cliq[2])
    mi = map(Int,m)
    skps += sum(map(Int,sum(mi, dims=1) .== 1))
  end
  return skps
end

function skipThroughMsgsIDs(cliq::TreeClique)
  cliqdata = getCliqueData(cliq)
  numfrtl1 = floor(Int,length(cliqdata.frontalIDs)+1)
  condAssocMat = cliqdata.cliqAssocMat[:,numfrtl1:end]
  condMsgMat = cliqdata.cliqMsgMat[:,numfrtl1:end]
  mat = [condAssocMat;condMsgMat];
  mab = sum(map(Int,mat),dims=1) .== 1
  mabM = sum(map(Int,condMsgMat),dims=1) .== 1
  mab = mab .& mabM
  # rang = 1:size(condMsgMat,2)
  msgidx = cliqdata.separatorIDs[vec(collect(mab))]
  return msgidx
end

function directPriorMsgIDs(cliq::TreeClique)
  frtl = getCliqueData(cliq).frontalIDs
  sepr = getCliqueData(cliq).separatorIDs
  cols = [frtl;sepr]
  mat = getCliqMat(cliq, showmsg=true)
  singr = sum(map(Int,mat),dims=2) .== 1
  rerows = collect(1:length(singr))
  b = vec(collect(singr))
  rerows2 = rerows[b]
  sumsrAc = sum(map(Int,mat[rerows2,:]),dims=1)
  sumc = sum(map(Int,mat),dims=1)
  pmSkipCols = (sumsrAc - sumc) .== 0
  return cols[vec(collect(pmSkipCols))]
end

function directFrtlMsgIDs(cliq::TreeClique)
  numfrtl = length(getCliqueData(cliq).frontalIDs)
  frntAssocMat = getCliqueData(cliq).cliqAssocMat[:,1:numfrtl]
  frtlMsgMat = getCliqueData(cliq).cliqMsgMat[:,1:numfrtl]
  mat = [frntAssocMat; frtlMsgMat];
  mab = sum(map(Int,mat),dims=1) .== 1
  mabM = sum(map(Int,frtlMsgMat),dims=1) .== 1
  mab = mab .& mabM
  return getCliqueData(cliq).frontalIDs[vec(collect(mab))]
end

function directAssignmentIDs(cliq::TreeClique)
  # NOTE -- old version been included in iterated variable stack
  assocMat = getCliqueData(cliq).cliqAssocMat
  msgMat = getCliqueData(cliq).cliqMsgMat
  mat = [assocMat;msgMat];
  mab = sum(map(Int,mat),dims=1) .== 1
  mabA = sum(map(Int,assocMat),dims=1) .== 1
  mab = mab .& mabA
  # TODO -- use proper accessor methods
  frtl = getCliqueData(cliq).frontalIDs
  sepr = getCliqueData(cliq).separatorIDs
  cols = [frtl;sepr]
  return cols[vec(collect(mab))]
  # also calculate how which are conditionals
end

function mcmcIterationIDs(cliq::TreeClique)
  @debug "mcmcIterationIDs\n" cliq.id getCliqFrontalVarIds(cliq) getCliqSeparatorVarIds(cliq)
  mat = getCliqMat(cliq)
  @debug "getCliqMat" mat
  # assocMat = getCliqueData(cliq).cliqAssocMat
  # msgMat = getCliqueData(cliq).cliqMsgMat
  # mat = [assocMat;msgMat];
  sum(sum(map(Int,mat),dims=1)) == 0 ? error("mcmcIterationIDs -- unaccounted variables") : nothing
  mab = 1 .< sum(map(Int,mat),dims=1)
  cols = getCliqAllVarIds(cliq)

  # must also include "direct variables" connected through projection only
  directvars = directAssignmentIDs(cliq)
  usset = union(directvars, cols[vec(collect(mab))])
  # NOTE -- fix direct vs itervar issue, DirectVarIDs against Iters should also Iter
  # NOTE -- using direct then mcmcIter ordering to prioritize non-msg vars first
  return setdiff(usset, getCliqueData(cliq).directPriorMsgIDs)
end

function getCliqMatVarIdx(cliq::TreeClique, varid::Symbol, allids=getCliqAllVarIds(cliq) )
  len = length(allids)
  [1:len;][allids .== varid][1]
end

"""
    $SIGNATURES

Determine and return order list of variable ids required for minibatch Gibbs iteration inside `cliq`.

Notes
- Singleton factors (priors and up messages) back of the list
- least number of associated factor variables earlier in list
- Same as getCliqVarSolveOrderUp
"""
function mcmcIterationIdsOrdered(cliq::TreeClique)
  # get unordered iter list
  alliter = mcmcIterationIDs(cliq)

  # get all singletons
  allsings = getCliqVarSingletons(cliq)
  singletonvars = intersect(alliter, allsings)

  # get all non-singleton iters
  nonsinglvars = setdiff(alliter, singletonvars)

  # sort nonsingletons ascending number of factors
  mat = getCliqMat(cliq)
  lenfcts = sum(mat, dims=1)
  nonslen = zeros(length(nonsinglvars))
  for i in 1:length(nonsinglvars)
    varid = nonsinglvars[i]
    varidx = getCliqMatVarIdx(cliq, varid)
    nonslen[i] = lenfcts[varidx]
  end
  p = sortperm(nonslen)
  ascnons = nonsinglvars[p]

  # sort singleton vars ascending number of factors
  singslen = zeros(length(singletonvars))
  for i in 1:length(singletonvars)
    varid = singletonvars[i]
    varidx = getCliqMatVarIdx(cliq, varid)
    singslen[i] = lenfcts[varidx]
  end
  p = sortperm(singslen)
  ascsing = singletonvars[p]

  return [ascnons; ascsing]
end

"""
    $SIGNATURES

Determine and return order list of variable ids required for minibatch Gibbs iteration inside `cliq`.

Notes
- Singleton factors (priors and up messages) back of the list
- least number of associated factor variables earlier in list
- Same as mcmcIterationIdsOrdered
"""
function getCliqVarSolveOrderUp(cliq::TreeClique)
  return mcmcIterationIdsOrdered(cliq)
end

"""
    $(SIGNATURES)

Prepare the variable IDs for nested clique Gibbs mini-batch calculations, by assembing these clique data fields:
- `directPriorMsgIDs`
- `directvarIDs`
- `itervarIDs`
- `msgskipIDs`
- `directFrtlMsgIDs`

"""
function setCliqMCIDs!(cliq::TreeClique)
  getCliqueData(cliq).directPriorMsgIDs = directPriorMsgIDs(cliq)

  # NOTE -- directvarIDs are combined into itervarIDs
  getCliqueData(cliq).directvarIDs = directAssignmentIDs(cliq)
  # TODO find itervarIDs that have upward child singleton messages and update them last in iter list
  getCliqueData(cliq).itervarIDs = mcmcIterationIdsOrdered(cliq)

  getCliqueData(cliq).msgskipIDs = skipThroughMsgsIDs(cliq)
  getCliqueData(cliq).directFrtlMsgIDs = directFrtlMsgIDs(cliq)

  # TODO add initialization sequence var id list too

  nothing
end

# post order tree traversal and build potential functions
function buildCliquePotentials(dfg::G, bt::AbstractBayesTree, cliq::TreeClique; solvable::Int=1) where G <: AbstractDFG
    for child in childCliqs(bt, cliq)#tree
        buildCliquePotentials(dfg, bt, child)
    end
    @debug "Get potentials $(getLabel(cliq))"
    setCliqPotentials!(dfg, bt, cliq, solvable=solvable)

    compCliqAssocMatrices!(dfg, bt, cliq)
    setCliqMCIDs!(cliq)

    nothing
end

"""
    $(SIGNATURES)

Return a vector of child cliques to `cliq`.
"""
function childCliqs(treel::MetaBayesTree, cliq::TreeClique)
  cliqKey = treel.bt[:cliqId][cliq.id]
  childcliqs = TreeClique[]
  for cIdx in MetaGraphs.outneighbors(treel.bt, cliqKey)
    push!(childcliqs, get_prop(treel.bt, cIdx, :clique))
  end
  return childcliqs
end

"""
    $(SIGNATURES)

Return a vector of child cliques to `cliq`.
"""
getChildren(treel::AbstractBayesTree, frtsym::Symbol) = childCliqs(treel, frtsym)
getChildren(treel::AbstractBayesTree, cliq::TreeClique) = childCliqs(treel, cliq)

"""
    $SIGNATURES
Get edges to children cliques
"""
function getEdgesChildren(tree::MetaBayesTree, cliqkey::Int)
  [MetaGraphs.Edge(cliqkey, chkey) for chkey in MetaGraphs.outneighbors(tree.bt, cliqkey)]
end

getEdgesChildren(tree::MetaBayesTree, cliq::TreeClique) = getEdgesChildren(tree, tree.bt[:cliqId][cliq.id])

"""
    $SIGNATURES
Get edges to parent clique
"""
function getEdgesParent(tree::MetaBayesTree, cliqkey::Int)
  [MetaGraphs.Edge(pkey, cliqkey) for pkey in MetaGraphs.inneighbors(tree.bt, cliqkey)]
end

getEdgesParent(tree::MetaBayesTree, cliq::TreeClique) = getEdgesParent(tree, tree.bt[:cliqId][cliq.id])

"""
    $SIGNATURES

Return a vector of all siblings to a clique, which defaults to not `inclusive` the calling `cliq`.
"""
function getCliqSiblings(treel::AbstractBayesTree, cliq::TreeClique, inclusive::Bool=false)::Vector{TreeClique}
  prnt = getParent(treel, cliq)
  if length(prnt) > 0
    allch = getChildren(treel, prnt[1])
  end
  if inclusive
    return allch
  end
  sibs = TreeClique[]
  for ch in allch
    if ch.id != cliq.id
      push!(sibs, ch)
    end
  end
  return sibs
end

"""
    $(SIGNATURES)

Return `cliq`'s parent clique.
"""
function parentCliq(treel::MetaBayesTree, cliq::TreeClique)
  cliqKey = treel.bt[:cliqId][cliq.id]
  parentcliqs = TreeClique[]
  for pIdx in  MetaGraphs.inneighbors(treel.bt, cliqKey)
    push!(parentcliqs, get_prop(treel.bt, pIdx, :clique))
  end
  return parentcliqs
end

"""
    $(SIGNATURES)

Return number of cliques in a tree.
"""
getNumCliqs(tree::MetaBayesTree) = MetaGraphs.nv(tree.bt)

"""
    $(SIGNATURES)

Return `cliq`'s parent clique.
"""
getParent(treel::AbstractBayesTree, afrontal::Union{Symbol, TreeClique}) = parentCliq(treel, afrontal)

"""
    $SIGNATURES

Return one symbol (a frontal variable) from each clique in the `::BayesTree`.

Notes
- Frontal variables only occur once in a clique per tree, therefore is a unique identifier.

Related:

whichCliq, printCliqHistorySummary
"""
function getTreeAllFrontalSyms(::AbstractDFG, tree::AbstractBayesTree)
  cliqs = getCliques(tree)
  syms = Vector{Symbol}(undef, length(cliqs))
  for (id,cliq) in cliqs
    syms[id] = getCliqFrontalVarIds(cliq)[1]
  end
  return syms
end

"""
    $SIGNATURES

Return the variable elimination order stored in a tree object.
"""
getEliminationOrder(treel::AbstractBayesTree) = treel.eliminationOrder

"""
    $SIGNATURES

EXPERIMENTAL, Save a Bayes (Junction) tree object to file.

Notes
- Converts and saves to JLD2 format a set of `PackedBayesTreeNodeData` objects.
- IIF issue #481

Related

IIF.loadTree, DFG.saveDFG, DFG.loadDFG, BSON.@save, BSON.@load
"""
function saveTree(treel::AbstractBayesTree,
                  filepath=joinpath("/tmp","caesar","savetree.bson") )
  #
  savetree = deepcopy(treel)
  for i in 1:length(getCliques(savetree))
    if  getCliqueData(savetree, i) isa BayesTreeNodeData
      setCliqueData!(getClique(savetree, i), convert(PackedBayesTreeNodeData, getCliqueData(savetree, i)))
    end
  end

  BSON.@save(filepath, savetree)
  return filepath
end

function saveTree(treeArr::Vector{T},
                  filepath=joinpath("/tmp","caesar","savetrees.bson") ) where T <: AbstractBayesTree
  #
  savetree = deepcopy(treeArr)
  for savtre in savetree, i in 1:length(getCliques(savtre))
    if getCliqueData(savtre, i) isa BayesTreeNodeData
      setCliqueData!(getClique(savtre,i), convert(PackedBayesTreeNodeData, getCliqueData(savtre, i)))
    end
  end

  BSON.@save(filepath, savetree)
  return filepath
end

"""
    $SIGNATURES

EXPERIMENTAL, Save a Bayes (Junction) tree object to file.

Notes
- Converts and saves to JLD2 format a set of `PackedBayesTreeNodeData` objects.
- IIF issue #481

Related

[`saveTree`](@ref), [`saveDFG`](@ref), [`loadDFG`](@ref), `BSON.@save`, `BSON.@load`
"""
function loadTree(filepath=joinpath("/tmp","caesar","savetree.bson"))
  data = BSON.@load(filepath, savetree)

  # convert back to a type that which could not be serialized by JLD2
  if savetree isa Vector
      for savtre in savetree, i in 1:length(getCliques(savtre))
        if getCliqueData(savtre, i) isa PackedBayesTreeNodeData
          setCliqueData!(getClique(savtre, i), convert(BayesTreeNodeData, getCliqueData(savtre, i)))
        end
      end
  else
    for i in 1:length(getCliques(savetree))
      if getCliqueData(savetree, i) isa PackedBayesTreeNodeData
        setCliqueData!(getClique(savetree, i), convert(BayesTreeNodeData, getCliqueData(savetree, i)))
      end
    end
  end

  # return loaded and converted tree
  return savetree
end

"""
    $SIGNATURES

Return Tuple of number cliques (Marginalized, Reused).
"""
function calcCliquesRecycled(tree::AbstractBayesTree)
  numMarg = 0
  numReused = 0
  numBoth = 0

  for (key, cliq) in tree.cliques
    numReused += getCliqueData(cliq).isCliqReused ? 1 : 0
    numMarg += getCliqueData(cliq).allmarginalized ? 1 : 0
    numBoth += getCliqueData(cliq).allmarginalized && getCliqueData(cliq).isCliqReused ? 1 : 0
  end

  return length(tree.cliques), numMarg, numReused, numBoth
end

## Tree Reuse

"""
    $SIGNATURES

Special internal function to try return the clique data if succesfully identified in `othertree::AbstractBayesTree`,
based on contents of `seeksSimilar::BayesTreeNodeData`.

Notes
- Used to identify and skip similar cliques (i.e. recycle computations)
"""
function attemptTreeSimilarClique(othertree::AbstractBayesTree,
                                  seeksSimilar::BayesTreeNodeData  )
  #
  # inner convenience function for returning empty clique
  function EMPTYCLIQ()
    clq = TreeClique(-1)
    setLabel!(clq, "")
    setCliqueData!(clq, BayesTreeNodeData())
    return clq
  end

  # does the other clique even exist?
  seekFrontals = getCliqFrontalVarIds(seeksSimilar)
  if !hasClique(othertree, seekFrontals[1])
    return EMPTYCLIQ()
  end

  # do the cliques share the same frontals?
  otherCliq = getClique(othertree, seekFrontals[1])
  otherFrontals = getCliqFrontalVarIds(otherCliq)
  commonFrontals = intersect(seekFrontals, otherFrontals)
  if length(commonFrontals) != length(seekFrontals) || length(commonFrontals) != length(otherFrontals)
    return EMPTYCLIQ()
  end

  # do the cliques share the same separator variables?
  seekSeparator = getCliqSeparatorVarIds(seeksSimilar)
  otherSeparator = getCliqSeparatorVarIds(otherCliq)
  commonSep = intersect(seekSeparator, otherSeparator)
  if length(commonSep) != length(seekSeparator) || length(commonSep) != length(otherSeparator)
    return EMPTYCLIQ()
  end

  # do the cliques use the same factors (potentials)
  seekPotentials = getCliqFactorIds(seeksSimilar)
  otherFactors = getCliqFactorIds(otherCliq)
  commonFactors = intersect(seekPotentials, otherFactors)
  if length(commonFactors) != length(seekPotentials) || length(commonFactors) != length(otherFactors)
    return EMPTYCLIQ()
  end

  # lets assume they are the same
  return otherCliq::TreeClique
end


text:
module BlockDates

export blockdates,
       dateformat2regex,
       replacemonth,
       removedays,
       removestopwords,
       removecommonprefix,
       removejoiner,
       singleGenerator,
       singleYearMissGen,
       cleanDate,
       DateMod,
       formatRange,
       andSplit,
       textToBlock,
       spreadOverNonDates,
       joinDuplicateDates,
       fillMissing,
       scoreDatesOutside!,
       dropBadMatches,
       fixBlock,
       scoreOutofOrder!,
       scoreBlock

##############################################################################
##
## Dependencies
##
##############################################################################

using Dates,
    Distributions,
    StatsBase,
    IterTools,
    DataFramesMeta,
    Pipe,
    Test,
    NBInclude

##############################################################################
##
## Exported methods and types (in addition to everything reexported above)
##
##############################################################################

### Code Structure
# - text with dates are passed one by one to the softdate function
# - softdate passes txt to dt2block function which first passes txt to rangeformatter
# - if rangeformatter finds a range match then dates are spread over that range and returned to softdate
# - if rangeformatter does not find a match then singleformatter is attempted
# - if that finds a match then a single value is returned

# Previous dateformat2regex & rmwhitespace

"""dateformat2regex converts input dateformats into potential regex matches.
"""
function dateformat2regex(m::String)
    m = replace(m,   r"\bm\b" => "(?:[1][0-9]|[0]?[1-9])")
    m = replace(m,   r"\by\b" => "(?:[12][0-9]|20[12][0-9])")
    m = replace(m,   r"\bd\b" => "(?:[1-3][0-9]|[0]?[1-9])")
end

"Remove extra white space from x"
function rmwhitespace(x)
    strip(replace(x, r"\s+" => " "))
end

monthlist = [r"\bJanu?a?r?y?\b"i
            ,r"\bFebr?u?a?r?y?\b"i
            ,r"\bMarc?h?\b"i
            ,r"\bApri?l?\b"i
            ,r"\bMay\b"i
            ,r"\bJune?\b"i
            ,r"\bJuly?\b"i
            ,r"\bAugu?s?t?\b"i
            ,r"\bSept?e?m?b?e?r?\b"i
            ,r"\bOcto?b?e?r?\b"i
            ,r"\bNove?m?b?e?r?\b"i
            ,r"\bDece?m?b?e?r?\b"i]

"""
Replace month matches with numbers when found in text.
"""
function replacemonth(x, I...)
    for i in 1:12;
        x = replace(x, monthlist[i]=> string(i))
    end
    x
end

"""
Remove potential day name matches from text
"""
function removedays(x, I...)
    dayabrev = Regex("(?i)(^|\\b)(" * join(unique([[dn[1:min(i,end)]
      for dn in dayname.(now() + Day.(0:6)), i in 9:-1:1]...]),"|") * ")(\\b|\$)")
    replace(x, dayabrev => "") |> strip
end

"""
Remove potential stop words  and common words from x:
    "NOTE|Between|On|At|Continued|Date" and
    "On|At|from"
"""
removestopwords(x) = replace(x,
  r"^(?i)(-|:| )*(NOTE|Between|On|At|Continued|Date)\b(:| )*(On|At|from)*"=>"")

"""
Remove common prefixes in the text.
"""
removecommonprefix(x) = replace(x, "No Wireline Logs Run"=>"")

"""
Remove joiner text: "thru|through|to"
"""
removejoiner(x) = replace(x, r"\b(thru|through|to)\b" => " ")

monthN = "[0-9]{1,2}" # 02, 11, 10, 12, 4 , 3
dayN   = "[0-9]{1,2}" # 1 , 04, 15, 29, 30, 31
yearN  = "[0-9]{1,4}" # 2021, 2019, 2021

"""
    rangeGenerator(dateformat = "mm dd yyyy", scoremod = 5, mod = Day(0); dateformat2 = "", mod2 = Day(0))

Generates a function that will interpret the date range from text passed to it.

# Arguments
- `dateformat`:  date that the rangeInterpretor will attempt to match
  dateformat to look for are of the type used with the DateFormat() function
- `score`:       base score to assign to this range interpretation
- `mod`:         number of Days, Years to add to the date after interpretation
- `dateformat2`: date that the rangeInterpretor will attempt to match for the second date
  if left empty it will use dateformat
- `mod2`:        number of Days, Years to add to the date2 after interpretation
"""
function rangeGenerator(dateformat = "mm dd yyyy", score = 5, mod = Day(0); dateformat2 = "", mod2 = Day(0))
    (dateformat2 == "") && (dateformat2 = dateformat; mod2 = mod)

    regmatcher = "^($dateformat) ($dateformat2)"

    for v in [r"\byyyy\b"=>yearN, r"\bdd\b"=>dayN , r"\bmm\b"=>monthN]
        regmatcher = replace(regmatcher, v)
    end

    function rangeInterpretor(textin; scoremod=0, defaultyear = year(now()))

        rangematch = match(Regex(regmatcher), textin)
        (rangematch === nothing) && return nothing

        try
          starter  = Date(rangematch.captures[1], DateFormat(dateformat))  + mod
          ender    = Date(rangematch.captures[2], DateFormat(dateformat2)) + mod2
          return (
            starter  = starter  ,
            ender    = ender,
            scoremod = scoremod + score)
        catch
        end
    end
end

"""Interprets dates in the form "mm dd yyyy mm dd yyyy" and scores them as 5
"""
range1 = rangeGenerator("mm dd yyyy", 5)

"""Interprets dates in the form "dd mm yyyy dd mm yyyy" and scores them as 5
"""
range2 = rangeGenerator("dd mm yyyy", 5)

"""Allow for missing year in first date in range eg. `mm dd till mm dd yyyy` or
  `dd mm till dd mm yyyy`
"""
function rangeSubGen(infunction::Function, penalty = -3)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{2,4})" =>
            s"\1 \2 \5 \3 \4 \5")
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""Interprets dates in the form "mm dd mm dd yyyy" and scores them as 2
"""
rangeSub1 = rangeSubGen(range1)

"""Interprets dates in the form "dd mm dd mm yyyy" and scores them as 2
"""
rangeSub2 = rangeSubGen(range2)

"""
Missing Year in Date Range
Allow for missing years in date range eg. `mm dd mm dd`
"""
function rangeYearMissGen(infunction::Function, penalty = -4)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2})" =>
            SubstitutionString("\\1 \\2 $defaultyear \\3 \\4 $defaultyear"))
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""Interprets dates in the form "mm dd mm dd" and scores them as 1
"""
rangeYearMiss1 = rangeYearMissGen(range1)

"""Interprets dates in the form "dd mm dd mm" and scores them as 1
"""
rangeYearMiss2 = rangeYearMissGen(range2)

"""
Missing Month in Part of Date Range
Allow for missing years in date range eg. `mm dd` through `dd yy`
"""
function rangeMonthMissGen(infunction::Function, penalty = -5)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,4})" =>
            SubstitutionString("\\1 \\2 \\4 \\1 \\3 \\4"))
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""Interprets dates in the form "mm dd dd yyyy" and scores them as 0
"""
rangeMonthMiss1 = rangeMonthMissGen(range1)

"""
Allow for missing year in first date `12/11 through 12/19/20` or
    `11/12 through 19/12/20`
"""
function rangeMissYearFirstGen(infunction::Function, penalty = -5)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,4})" =>
            SubstitutionString("\\1 \\2 \\5 \\3 \\4 \\5"))
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""Interprets dates in the form "mm dd mm dd yyyy" and scores them as 0
"""
rangeMissYearFirst1 = rangeMissYearFirstGen(range1)

"""Interprets dates in the form "dd mm dd mm yyyy" and scores them as 0
"""
rangeMissYearFirst2 = rangeMissYearFirstGen(range2)

"""
Allow for missing month in first date `14/20 through 07/21/20`.
"""
function rangeMissFirstMonthGen(infunction::Function, penalty = -6)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,4}) ([0-9]{1,2}) ([0-9]{1,2}) ([0-9]{1,4})" =>
            SubstitutionString("\\3 \\1 \\2 \\3 \\4 \\5"))
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""
Allow for missing month in first date `14/20 through 07/21/20`.
"""
rangeMissFirstMonth1 = rangeMissFirstMonthGen(range1)

"""
Allow for date to be found within the text `The event on July 3 2020 to July 10 2020.`
"""
function inTextGen(infunction::Function, penalty = -1)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        m = match(r"([0-9]{1,4} [0-9]{1,4} [0-9]{1,4}\s*)+", txtin)
        (m != nothing) && (txtin = "$(String(m.match)) $txtin")
        infunction(txtin, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""
Allow for date to be found in text of the form
    "On the date range mm dd yyyy mm dd yyyy something..."
"""
rangeInText1 = inTextGen(range1)

"""
Allow for date to be found in text of the form
    "On the date dd mm yyyy dd mm yyyy something..."
"""
rangeInText2 = inTextGen(range2)

rangeset = [range1, range2 , rangeSub1 , rangeSub2 ,
    rangeYearMiss1, rangeYearMiss2, rangeMonthMiss1,
    rangeMissYearFirst1, rangeMissYearFirst2,
    rangeMissFirstMonth1, rangeInText1, rangeInText2]

"""
    singleGenerator(dateformat = "mm dd yyyy", scoremod = 5, mod = Day(0))

Generates a function that will interprets a single date from text passed to it.

# Arguments
- `dateformat`:  date that the rangeInterpretor will attempt to match
  dateformat to look for are of the type used with the DateFormat() function
- `scoremod`:    score to assign to this range interpretation
- `mod`:         number of Days, Years to add to the date after interpretation
"""
function singleGenerator(dateformat = "mm dd yyyy", score = 5, mod = Day(0))

    regmatcher = "^($dateformat)"

    for v in [r"\byyyy\b"=>yearN, r"\bdd\b"=>dayN, r"\bmm\b"=>monthN]
        regmatcher = replace(regmatcher, v)
    end

    function singleInterpretor(textin; scoremod = 0, defaultyear = now())

        singlematch = match(Regex(regmatcher), textin)

        (singlematch === nothing) && return nothing

        try
#             @show regmatcher, textin, dateformat, singlematch.captures[1]
            starter  = Date(singlematch.captures[1], DateFormat(dateformat)) + mod
            return (starter  = starter  ,
                    scoremod = scoremod + score)
        catch
        end
    end
end

"""
Allow for date to be found in text of the form "mm dd yyyy something..."
"""
single0(x; scoremod = 0, defaultyear = now()) = (starter = Date(0), scoremod = 0)

"""
    single1(text, scoremod = 0, mod = Day(0))

Allow for date to be found in text of the form "mm dd yyyy something..."
"""
single1 = singleGenerator("mm dd yyyy", 5)

"""
    single2(text, scoremod = 0, mod = Day(0))

Allow for date to be found in text of the form "dd mm yyyy something..."
"""
single2 = singleGenerator("dd mm yyyy", 5)

"""
    single3(text, scoremod = 0, mod = Day(0))

Allow for date to be found in text of the form "yyyy mm dd something..."
"""
single3 = singleGenerator("yyyy mm dd", 5)

"""
    single4(text, scoremod = 0, mod = Day(0))

Allow for date to be found in text of the form "yyyy mm dd something..."
"""
single4 = singleGenerator("yyyy dd mm", 5)

"""
    singleYearMissGen(infunction::Function, penalty = -3)

Missing Year in Date Single
Allow for missing year in date eg. `mm dd` or `dd mm`
"""
function singleYearMissGen(infunction::Function, penalty = -3)
    function(txtin;  scoremod=0, defaultyear = year(now()))
        txtout = replace(txtin,
            r"([0-9]{1,2}) ([0-9]{1,2})" => SubstitutionString("\\1 \\2 $defaultyear"))
        infunction(txtout, scoremod= scoremod + penalty, defaultyear=defaultyear)
    end
end

"""
    singleYearMiss1(text, scoremod = 0, defaultyear = year(now()))

Allow for date to be found in text of the form "mm dd something..."
"""
singleYearMiss1 = singleYearMissGen(single1)

"""
    singleYearMiss2(text, scoremod = 0, defaultyear = year(now()))

Allow for date to be found in text of the form "dd mm something..."
"""
singleYearMiss2 = singleYearMissGen(single2)

"""
    singleInText1(text, scoremod = 0, defaultyear = year(now()))

Allow for date to be found in text of the form "Something something mm dd yyyy something"
"""
singleInText1 = inTextGen(single1)

"""
    singleInText2(text, scoremod = 0, defaultyear = year(now()))

Allow for date to be found in text of the form "Something something mm dd yyyy something"
"""
singleInText2 = inTextGen(single2)

singleset = [single1, single2, single3, single4,
             singleYearMiss1, singleYearMiss2,
             singleInText1, singleInText2, single0]

"""
    cleanDate(text)

Make text more readable by the date interpreter. (make lowercase, replace month with #,
remove days, remove common prefixes, remove stop words, remove joiners, replace markers used
to divide dates, and remove common numeric ordering tags (st, nd, rd).
"""
function cleanDate(txtin)
    @pipe txtin |>
        lowercase |>
        replacemonth |> removedays |> removecommonprefix |>
        removestopwords |> removejoiner |>
        replace(_, r"[|./\-\\,\\:+]" => " ") |>
        replace(_, r"(st|nd|rd|th)\b"=>"") |>
        replace(_, r"[ ]+" => " ") |>
        strip
end

"""
    DateMod(inDate, scoremod; mods = "", defaultdate = Date(0))

Adjust dates to be within feasible range.
"""
function DateMod(inDate, scoremod; mods = "", defaultdate = Date(0))
    # If less than date(01-01-100) Add 2000 yrs
    (inDate < Date(100)) &&
         (inDate += Year(2000); scoremod -= 1; mods *= " +2000yr")

    # If someone swapped 220x when meant to write 202x
    (Date(2200) < inDate < Date(2220)) &&
       (inDate -= Year(180); scoremod -= 2; mods *= " -180yr")

    # If date 30xx when meant to write 20xx
    (Date(3000) < inDate < Date(3030)) &&
         (inDate -= Year(1000); scoremod -= 2; mods *= " -1000yr")

     # If date 203x when meant to write 202x
    (Date(2030) < inDate < Date(2040)) &&
         (inDate -= Year(10); scoremod -= 2; mods *= " -10yr")

     # If date 102x when meant to write 202x
    (Date(1000) < inDate < Date(1040)) &&
         (inDate -= Year(10); scoremod -= 2; mods *= " +1000yr")

     # If date 21x when meant to write 201x or 22x when meant 202x
    (Date(200) < inDate < Date(250)) &&
         (inDate += Year(1800); scoremod -= 2; mods *= " +1800yr")

     # In date is more than .5 years from the default date but no more than 1.5 years
     (defaultdate != Date(0)) && (Day(180) < defaultdate - inDate < Day(540)) &&
         (inDate += Year(1); scoremod -= 2; mods *= " +1yr")

     # In date is more than .5 years from the default date but no more than 1.5 years
     (defaultdate != Date(0)) && (Day(180) < inDate - defaultdate < Day(540)) &&
         (inDate -= Year(1); scoremod -= 2; mods *= " -1yr")

    (inDate=inDate, scoremod=scoremod, mods=strip(mods))
end

"""
    DateMod(inDate::Date, ender::Date, scoremod; mods = "", defaultdate = Date(0))

    Apply DateMod to inDate and ender separately then do two more modifications.
    If inDate is greater than ender swap.
    If the date range is too wide (greater than 30 days) force the end date to be
    only 6 days after the start date.
"""
function DateMod(inDate::Date, ender::Date, scoremod;
    mods = "", defaultdate = Date(0))
    inDate, scoremod, mods = DateMod(inDate, scoremod, mods = mods, defaultdate=defaultdate)
    ender , scoremod, mods = DateMod(ender , scoremod, mods = mods, defaultdate=defaultdate)

    if inDate > ender; # Penalize ranges when start and end dates are swapped
        inDate , ender, scoremod = ender, inDate, scoremod -3;
        mods *= " startEndSwap"
    end

    if (ender - inDate) > Day(30); # If date range is too wide "fix" ender
        inDate , ender, scoremod = inDate, inDate + Day(6), scoremod -5;
        mods *= " interprettedRangeTooLarge"
    end

    (inDate=inDate, ender=ender, scoremod=scoremod, mods=strip(mods))
end

"""
    formatRange(textin; rangeformat = 1, defaultdate = Date(0))

Convert range from "11/29/2020 - 12/05/2020" to a dataframe with scoremods.
"""
function formatRange(textin; rangeformat = 1, defaultdate = Date(0))
    textCleaned = cleanDate(textin)

    r = rangeset[rangeformat](textCleaned, defaultyear=year(defaultdate))

    (r === nothing) &&
      (return DataFrame(date = Date(0), text = textin, scoremod = 0, mods = ""))

    starter , ender , scoremod, mods = DateMod(r..., defaultdate = defaultdate)
    return DataFrame(
        date     = starter:Day(1):ender,
        text     = textin,
        scoremod = scoremod,
        mods     = String(mods))
end

"""
formatSingle(textin; singleformat = 1, defaultdate = Date(0))

Convert range from "11/29/2020 ..." to a dataframe with scoremods.
"""
function formatSingle(textin; singleformat = 1, defaultdate = Date(0))
    textclean = cleanDate(textin)
    r = singleset[singleformat](textclean, defaultyear = year(defaultdate))

    (r === nothing) &&
        (return DataFrame(date = Date(0), text = textin, scoremod = 0, mods = ""))

    starter , scoremod, mods = DateMod(r..., defaultdate = defaultdate)

    return DataFrame(date     = starter,
                     text     = textin,
                     scoremod = scoremod,
                     mods     = strip(String(mods)))
end

"""
    andSplit(txtin, splitters = "(?:\\/)", ands = "(?:and|,)")

A function to split dates that have groupings like "and" and ",".
"""
function andSplit(txtin, splitters = "(?:\\/)", ands = "(?:and|,)");
    andmatcher = r"^([a-zA-Z ]{0,5}((\d{1,4}(\/)\d{1,4})+\s*(,|and|\s)+\s*)+)"
    if occursin(andmatcher, txtin)
        m = match(andmatcher, txtin)
        txtdup = replace(txtin, m.captures[1] => "")
        x = [string(x[1]) for x in eachmatch(
            Regex("((?:[0-9]+" * splitters * ")+[0-9]+)"), m.captures[1])]
        return join(x .* " " .* txtdup, "\n")
    end
    return txtin
end

"""
    textToBlock(txtsplit::Array{String};
                    singleformat = 1,
                    rangeformat  = 1,
                    defaultdate = Date(0))

Takes a date text string list and tries to match each element first as a range
then as a single.
"""
function textToBlock(txtsplit::Array{String};
    singleformat = 1,
    rangeformat  = 1,
    defaultdate  = Date(0))

    outframe = DataFrame()

    for tx in txtsplit;
        rangeattempt = formatRange(tx,
            rangeformat = rangeformat,
            defaultdate = defaultdate)

        if size(rangeattempt,1) > 1
            rangeattempt.mods .*= " rangefound"
            outframe = vcat(outframe, rangeattempt)
            continue
        end

        singleattempt = formatSingle(tx,
                singleformat  = singleformat,
                defaultdate   = defaultdate)
        outframe = vcat(outframe, singleattempt)

    end

    return outframe[strip.(outframe.text) .!= "", :]
end

"""
    spreadOverNonDates(inframe)

Merges rows with missing dates with previous row (unless first row then merges with next row).
"""
function spreadOverNonDates(inframe)

    # Create column to keep track of input dates == date(0)
    inframe[!, :date0] .= inframe.date

    # Track order that text is in
    inframe[!, :order] = 1:size(inframe,1)

    # Treat input text as a groupby unique ID to use for joining text fields without dates
    inframe[!, :block] .= inframe.text

    # Grab previous date if current date is missing
    for i in 2:size(inframe,1)
      (inframe[i, :date] == Date(0)) &&
          (inframe[i, [:block, :date]]  = inframe[i-1, [:block, :date]])
    end

    # Grab next date if current date is missing
    for i in (size(inframe,1)-1):-1:1
      (inframe[i, :date] == Date(0)) &&
          (inframe[i, [:block, :date]]  = inframe[i+1, [:block, :date]])
    end

    # Combine text across the block
    @pipe DataFramesMeta.groupby(inframe , :block) |>
      @transform(_, :text = join(unique(:text), " \n "))

    # Drop anything that started with date == Date(0) ie missing
    inframe = inframe[(inframe.date0 .!= Date(0)) .| (inframe.date .== Date(0)),
          Not([:date0, :block])]

    inframe
end

"""
    joinDuplicateDates(inframe, penalty = 7)

Join any duplicate dates which have different text together.
Scoremod penalizes these observations by -7 number of duplicate dates.
"""
function joinDuplicateDates(inframe, penalty = 3)
  inframe = @pipe DataFramesMeta.groupby(inframe , :date) |>
    @combine(_,
      :mods     = join(ifelse(length(:mods) > 1,
                 vcat(unique(:mods)..., "duplicateDates"), unique(:mods)), " "),
      :scoremod = sum(:scoremod) - penalty*(length(:mods)-1),
      :order    = [[:order...]],
      :text     = join(unique(:text), " \n ")
  )

  (Date(0) in inframe.date) && (inframe.scoremod = [0]; inframe.mods = ["noDateFound"])

  inframe
end

"""
    fillMissing(inframe, dtstart = Date(0), dtend   = Date(0))

Join any duplicate dates which have different text together.
Scoremod penalizes these observations by -7 number of duplicate dates.
"""
function fillMissing(inframe, dtstart = Date(0), dtend   = Date(0))

    # Make sure all of the dates appear in the data
    (dtstart == Date(0)) && return inframe

    outframe = outerjoin(DataFrame(date = dtstart:Day(1):dtend), inframe, on = :date)

    outframe[ismissing.(outframe.text), :scoremod] .= -5

    outframe[!, :text] =
      [coalesce(outframe.text[i], "<<NO TEXT FOUND FOR $(outframe.date[i])>>")
          for i in 1:size(outframe,1)]

    outframe[ismissing.(outframe.mods), :mods] .= "filled"

    if (Date(0) in outframe.date)
        outframe.text = "Header Text: " .* outframe.text[outframe.date .== Date(0)] .*
                        " " .* outframe.text
        outframe = outframe[outframe.date .!= Date(0), :]
    end

    outframe
end

"""
   scoreDatesOutside!(inblock; dtstart, dtend, penalty = 5)

Scores dates outside of the dtstart - dtend range. Dates which are outside
get penalized by the penalty value. If they are outside by more than 1 2 4 8 16 25 100
weeks then then accumulate another penalty for each step and the "outOfRange" gets a "*"
appended to it for each step.

Parameters:
inblock: DataFrame containing date, mod, and scoremod columns
dtstart: the start of the interval of interest
dtend: the end of the interval of interest
penalty: the penalty to the scoremod for each step out of the dtstart/dtend range
"""
function scoreDatesOutside!(inblock; dtstart, dtend, penalty = 5)
    outOfRange = (inblock.date .> dtend) .| (inblock.date .< dtstart)
    inblock[outOfRange, :mods] = strip.(inblock[outOfRange, :mods] .* " outOfRange")
    inblock[outOfRange, :scoremod] .-= penalty

    step = Day(7)
    for i in [1 2 4 8 16 25 50 100]
        outOfRange = (inblock.date .> dtend + i*step) .| (inblock.date .< dtstart - i*step)
        inblock[outOfRange, :mods] = strip.(inblock[outOfRange, :mods] .* "*")
        inblock[outOfRange, :scoremod] .-= penalty
    end

    inblock
end

"""
   dropBadMatches(inframe::DataFrame, dropscore = -15)

Removes the date and score from any dates for which their interpretation is extremely
low scored. This is most often helpful when a non-date becomes interpetted as a date.
"""
function dropBadMatches(inframe::DataFrame, dropscore = -15)
    dropscores = (inframe.scoremod  .<= dropscore) .& (inframe.date .!= Date(0))
    any(dropscores) && (inframe[dropscores, :date]     .= Date(0);
                        inframe[dropscores, :scoremod] .= 0)
    return inframe
end

function fixBlock(inframe::DataFrame; dtstart = Date(0), dtend   = Date(0))

    outframe = copy(inframe)
  # println(outframe)

    outframe = scoreDatesOutside!(outframe, dtstart = dtstart, dtend = dtend)
  # println("outframe = scoreDatesOutside!(outframe, dtstart = dtstart, dtend = dtend)")
  # println(outframe)

    outframe = dropBadMatches(outframe)
  # println("outframe = dropBadMatches(outframe)")
  # println(outframe)

    outframe = spreadOverNonDates(outframe)
  # println("outframe = spreadOverNonDates(outframe)")
  # println(outframe)

    outframe = joinDuplicateDates(outframe)
  # println("outframe = joinDuplicateDates(outframe)")
  # println(outframe)

    (dtstart != Date(0)) && (outframe = fillMissing(outframe, dtstart, dtend))

    outframe = sort(outframe[:, [:date, :mods, :scoremod, :order ,:text]], :date)

    return outframe

end

"""
    scoreOutofOrder!(inblock, dtstart, dtend)

penalizes dates out of order
"""
function scoreOutofOrder!(inblock; dtstart=now(), dtend=now()-Day(7))
    inview = view(inblock, .!ismissing.(inblock.order), :)
    for i in 2:size(inview, 1)
        ((maximum(inview.order[i-1]) > maximum(inview.order[i])) |
         (minimum(inview.order[i-1]) > minimum(inview.order[i]))) &&
            (inview[i, :mods]     = strip(inview[i, :mods] * " outOfOrder");
             inview[i, :scoremod] = inview[i, :scoremod] - 2)
    end
    inblock
end

"""
    scoreBlock(inframe; dtstart = Date(0), dtend = Date(0))

Score the passed block by first adjusting for out of order entries then
    taking the average of the scoremod values.

"""
function scoreBlock(inframe; dtstart = Date(0), dtend = Date(0))
    (dtstart == Date(0)) && (dtstart = minimum(inframe.date))
    (dtend   == Date(0)) && (dtend   = maximum(inframe.date))

#     inframe = scoreDatesOutside!(inframe, dtstart = dtstart, dtend = dtend)
    inframe = scoreOutofOrder!(inframe, dtstart = dtstart, dtend = dtend)

    inframe[!, :score] .= sum(inframe.scoremod)/size(inframe, 1)
    inframe = inframe[!, [:date, :mods, :scoremod, :score, :order, :text]]

    return inframe
end

"""
    softdate(txtin; dtstart = Date(0), dtend = Date(0),
        splits          = r"[\n\r]+",
        singleformats   = length(singleset),
        rangeformats    = length(rangeset),
        targetScore     = 28,
        verbose         = false)

Split the text
"""
function blockdate(txtin; dtstart = Date(0), dtend = Date(0),
    splits          = r"[\n\r]+",
    singleformats   = length(singleset),
    rangeformats    = length(rangeset),
    targetScore     = 28,
    verbose         = false)

    scoreMax, outframeMax = -999, DataFrame()

    txtSplit = String.([strip(tx) for tx in split(txtin, splits) if strip(tx) != ""])

    for s in 1:singleformats, r in 1:rangeformats

    outframe = @pipe textToBlock(txtSplit, singleformat = s, rangeformat  = r, defaultdate = dtstart) |>
    fixBlock(_, dtstart = dtstart, dtend = dtend)  |>
    scoreBlock(_, dtstart = dtstart, dtend = dtend)

    if (outframe.score[1] > scoreMax)
    outframe[!, :singleformat] .= s
    outframe[!, :rangeformat]  .= r
    scoreMax, outframeMax = outframe.score[1], outframe
    verbose && (@show outframeMax)

    (scoreMax >= targetScore) && verbose &&
    (println("targetScore ($targetScore) achieved."); break)
    end

    end

    (scoreMax < targetScore) && verbose &&
    println("targetScore ($targetScore) not achieved.")

    outframeMax
end

#############################################################
end # module

ext: jl


text:
struct CtbSentence
        tree::String
        poswords::Vector{Tuple{Ti, Ti}}
end

struct CtbDocument
        type::String
        sents::Vector{CtbSentence}
end

struct ChTreebank
        postags::Vector{String}
        inntags::Vector{String}
        words::Vector{String}
        docs::Vector{CtbDocument}
end

postags(ctb::ChTreebank) = ctb.postags
inntags(ctb::ChTreebank) = ctb.inntags
words(ctb::ChTreebank) = ctb.words

import Base.show
function show(io::IO, ctb::ChTreebank)
        nd = length(ctb)
        ns = mapreduce(length, +, ctb)
        nw = mapreduce(length, +, Iterators.flatten(ctb))
        print(io, "Chinese Treebank of $(nd) docs, $(ns) sents, $(length(ctb.words))/$(nw) words")
end

function ChTreebank(home::String; nf=0)
        home_data = joinpath(home, "data", "bracketed")
        if (nf <= 0) nf = length(readdir(home_data)) end
        docs = Vector{CtbDocument}(undef, nf)
        postagMap = Dict{String, Ti}()
        inntagMap = Dict{String, Ti}()
        wordMap = Dict{String, Ti}()
        @showprogress 1 "Parsing ChTreebank..." for (i, file_name) in enumerate(readdir(home_data))
                if i > nf break end
                type = ""
                id = parse(Int, file_name[6:9])
                if 0001<=id<=0325 || 0400<=id<=0454 || 0500<=id<=0540 || 0600<=id<=0885 || 0900<=id<=0931 || 4000<=id<=4050 type = "Newwire"
                elseif 0590<=id<=0596 || 1001<=id<=1151 type = "Magazine articles"
                elseif 2000<=id<=3145 || 4051<=id<=4111 type = "Broadcast news"
                elseif 4112<=id<=4197 type = "Broadcast conversations"
                elseif 4198<=id<=4411 type = "Weblogs"
                elseif 5000<=id<=5558 type = "Discussion forums"
                else type = "N/A"
                end
                treestrs = parsectbfile(joinpath(home_data, file_name))
                sents = CtbSentence[]
                for treestr in treestrs
                        tree = CtbTree(treestr)
                        pws = poswords(tree)
                        pws = map(p -> (getid!(postagMap, p[1]), getid!(wordMap, p[2])), pws)
                        push!(sents, CtbSentence(String(treestr), pws))

                        for inntag in inntags(tree) getid!(inntagMap, inntag) end
                end
                docs[i] = CtbDocument(type, sents)
        end
        postags, _inntags, words = vec.((postagMap, inntagMap, wordMap))
        r = ChTreebank(postags, _inntags, words, docs)
        return r
end

function ChTreebank()
        file = KongYiji.dir("ctb.jld2")
        zfile = KongYiji.dir("ctb.jld2.7z")
        if isfile(file)
                r = load(file)["ctb"]
        elseif isfile(zfile)
                r = load(KongYiji.decompress(zfile))["ctb"]
        else
                error("CTB not downloaded.")
        end
        return r
end

Base.length(sent::CtbSentence) = length(sent.poswords)
Base.length(doc::CtbDocument) = length(doc.sents)
Base.length(ctb::ChTreebank) = length(ctb.docs)
Base.getindex(sent::CtbSentence, inds...) = getindex(sent.poswords, inds...)
Base.getindex(doc::CtbDocument, inds...) = getindex(doc.sents, inds...)
Base.getindex(ctb::ChTreebank, inds...) = getindex(ctb.docs, inds...)
Base.iterate(sent::CtbSentence, state=1) = state > length(sent) ? nothing : (sent.poswords[state], state + 1)
Base.iterate(doc::CtbDocument, state=1) = state > length(doc) ? nothing : (doc.sents[state], state + 1)
Base.iterate(ctb::ChTreebank, state=1) = state > length(ctb) ? nothing : (ctb.docs[state], state + 1)

mutable struct Block
    chrs::Vector{Char}
    nlb::Int
end

function Block()
        chrs = Char[]
        sizehint!(chrs, 100)
        #resize!(chrs, 100)
        return Block(chrs, 0)
end

function push!(b::Block, chrs::Vector{Char})
    for c in chrs
        if c == '('
            b.nlb += 1
        elseif c == ')'
            b.nlb -= 1
        end
        push!(b.chrs, c)
    end
end

function text(b::Block)
        resize!(b.chrs, length(b.chrs));
        return b.chrs
end

function ok(block::Block)
    return block.nlb == 0
end

function parsectbfile(file_path)
        ret = Vector{Char}[]
        b = Block()
        open(file_path, "r") do io
            for line in eachline(io)
                if startswith(line, "(")
                    push!(b, collect(line))
                    if !ok(b)
                        for line in eachline(io)
                            push!(b, collect(line))
                            if ok(b) break end
                        end
                    end
                    push!(ret, text(b))
                    b = Block()
                end
            end
        end
        return ret
end

text(tree::CtbTree) = tree.label
ispostag(tree::CtbTree) = length(tree.adj) == 1 && isleaf(tree.adj[1])

function poswords(sent::CtbTree)
        ret = Tuple{String, String}[]
        visitor(tree::CtbTree) = if ispostag(tree) && text(tree) != "-NONE-" push!(ret, (text(tree), text(tree.adj[1]))) end
        dfstraverse(sent, visitor)
        return ret
end

sents(ctb::ChTreebank) = Iterators.flatten(ctb)

function inntags(sent::CtbTree)
        ret = String[]
        visitor(tree::CtbTree) = if !ispostag(tree) && !isleaf(tree); push!(ret, text(tree)); end
        dfstraverse(sent, visitor)
        return ret
end

function rawsents(ctb::ChTreebank)
        r = [join([ctb.words[p[2]] for p in sent]) for sent in sents(ctb)]
        r
end

function wordsents(ctb::ChTreebank)
        r = [[ctb.words[p[2]] for p in sent] for sent in sents(ctb)]
        r
end

function posidsents(ctb::ChTreebank)
        r = [[p[1] for p in sent] for sent in sents(ctb)]
        r
end

function ==(a::ChTreebank, b::ChTreebank)
        return all(fname -> getfield(a, fname) == getfield(b, fname), fieldnames(ChTreebank))
end

function ==(a::CtbDocument, b::CtbDocument)
        return all(fname -> getfield(a, fname) == getfield(b, fname), fieldnames(CtbDocument))
end

function ==(a::CtbSentence, b::CtbSentence)
        return all(fname -> getfield(a, fname) == getfield(b, fname), fieldnames(CtbSentence))
end

function split(ctb::ChTreebank; percents::Vector{Float64}=[0.7, 0.2, 0.1])
        percents ./= sum(percents)
        n = length(ctb)
        caps = map(p -> floor(p * n), percents)
        caps[3] += n - sum(caps)
        idx = randperm(n)
        train = ChTreebank(ctb.tags, ctb.docs[1:caps[1]])
        dev = ChTreebank(ctb.tags, ctb.docs[caps[1]+1:caps[1]+caps[2]])
        test = ChTreebank(ctb.tags, ctb.docs[end-caps[3]+1:end])
        return (train, dev, test)
end

import Base.+
function foldbatch(ctb::ChTreebank, nbatch::Int)
        groups = DefaultDict{String, Vector{CtbDocument}}(()->CtbDocument[])
        for doc in ctb; push!(groups[doc.type], doc); end
        for (_, docs) in groups; shuffle!(docs); end
        nb = min(nbatch, mapreduce(length, max, values(groups)))
        @assert 2 <= nb
        (begin
                tedocs = CtbDocument[]
                trdocs = CtbDocument[]
                for (type, docs) in groups
                        nd = length(docs)
                        npick = div(nd + nb - 1, nb)
                        from = min(nd + 1, (ib - 1) * npick + 1)
                        to = min(nd, from + npick - 1)
                        append!(tedocs, docs[from:to])

                        append!(trdocs, docs[1:from - 1])
                        append!(trdocs, docs[to + 1:nd])
                end
                te = ChTreebank(ctb.postags, ctb.inntags, ctb.words, tedocs)
                tr = ChTreebank(ctb.postags, ctb.inntags, ctb.words, trdocs)
                (tr, te)
        end for ib in 1:nb)
end

function kwictable(;word="", pos="")
        ctb_home = KongYiji.unzip7(joinpath(pathof(KongYiji), "..", "..", "data", "ctb.jld2.7z"))
        ctb = load(ctb_home)["ctb"]
        df = DataFrame(cw=String[], cpos=String[], lw=String[], rw=String[], lpos=String[], rpos=String[])
        for doc in ctb, sent in doc
                nw = length(sent)
                for i in 1:nw
                        cpos, cw = sent[i]
                        if word == cw || pos == cpos
                                #l = i == 1  ? "^" : sent[i - 1][2][end:end]
                                #r = i == nw ? "\$" : sent[i + 1][2][end:end]
                                lpos, lw = i == 1  ? ("^", "") : sent[i - 1]
                                rpos, rw = i == nw ? ("\$", "") : sent[i + 1]
                                push!(df, (cw, cpos, lw, rw, lpos, rpos))
                        end
                end
        end
        return df
end

function mask(ctb::ChTreebank)
        words = mask.(ctb.words) |> sort! |> unique!
        wmp = dict(words)
        nd = length(ctb)
        r = ChTreebank(ctb.postags, ctb.inntags, words, Vector{CtbDocument}(undef, nd))
        for id = 1:nd
                doc1 = ctb.docs[id]
                ns = length(doc1)
                doc2 = CtbDocument(doc1.type, Vector{CtbSentence}(undef, ns))
                for is = 1:ns
                        s1 = doc1[is]
                        s2 = map(p->(p[1], wmp[ctb.words[p[2]] |> mask]), s1)
                        s2 = CtbSentence(s1.tree, s2)
                        doc2.sents[is] = s2
                end
                r.docs[id] = doc2
        end
        r
end


text:
@testset "examples" begin

function find_sources(dir)
    sources = String[]
    for entry in readdir(dir)
        path = joinpath(dir, entry)
        if isdir(path)
            append!(sources, find_sources(path))
        elseif endswith(path, ".jl")
            push!(sources, path)
        end
    end
    return sources
end

examples_dir = joinpath(@__DIR__, "..", "examples")
examples = find_sources(examples_dir)
filter!(file -> readline(file) != "# EXCLUDE FROM TESTING", examples)

for example in examples
    id = relpath(example, examples_dir)
    @eval begin
        @testset $id begin
            file = $example
            # FIXME: doesn't forward arguments (color=yes, compilecache=no)
            @test success(pipeline(`$(Base.julia_cmd()) $file`; stderr=STDERR))
        end
    end
end

end


text:
import SGP4
using Test, Dates

function test_init()
    line1 = "1 00005U 58002B   00179.78495062  .00000023  00000-0  28098-4 0  4753"
    line2 = "2 00005  34.2682 348.7242 1859667 331.7664  19.3264 10.82419157413667"

    wgs72 = SGP4.GravityModel("wgs72")
    satellite = SGP4.twoline2rv(line1, line2, wgs72)

    @test satellite[:satnum] == 5
    return satellite
end

function test_single_time()
    satellite = test_init()

    t = Dates.DateTime(2000, 6, 29, 12, 50, 19)
    (position, velocity) = SGP4.propagate( satellite, 2000, 6, 29, 12, 50, 19)
    (position2, velocity2) = SGP4.propagate( satellite, t )

    @test satellite[:error] == 0

    @test isapprox(position[1], position2[1], atol=eps())
    @test isapprox(position[2], position2[2], atol=eps())
    @test isapprox(position[3], position2[3], atol=eps())

    @test isapprox(velocity[1], velocity2[1], atol=eps())
    @test isapprox(velocity[2], velocity2[2], atol=eps())
    @test isapprox(velocity[3], velocity2[3], atol=eps())

    @test isapprox(position[1], 5576.056952, atol=1e-6)
    @test isapprox(position[2], -3999.371134,atol= 1e-6)
    @test isapprox(position[3], -1521.957159,atol= 1e-6)

    @test isapprox(velocity[1], 4.772627, atol=1e-6)
    @test isapprox(velocity[2], 5.119817, atol=1e-6)
    @test isapprox(velocity[3], 4.275553, atol=1e-6)
end

function test_multiple_sats()
    satellite = test_init()
    t = Dates.DateTime(2000, 6, 29, 12, 50, 19)
    sats = [satellite; satellite; satellite]

    rvs = SGP4.propagate( sats, t )

    @test isapprox(rvs[1][1][1], rvs[3][1][1], atol=eps())
    @test isapprox(rvs[1][1][2], rvs[3][1][2], atol=eps())
    @test isapprox(rvs[1][1][3], rvs[3][1][3], atol=eps())

    @test isapprox(rvs[1][2][1], rvs[3][2][1], atol=eps())
    @test isapprox(rvs[1][2][2], rvs[3][2][2], atol=eps())
    @test isapprox(rvs[1][2][3], rvs[3][2][3], atol=eps())

    dtmin = 10.0

    rvs2 = SGP4.propagate( sats, dtmin )

    @test isapprox(rvs2[1][1][1], rvs2[3][1][1], atol=eps())
    @test isapprox(rvs2[1][1][2], rvs2[3][1][2], atol=eps())
    @test isapprox(rvs2[1][1][3], rvs2[3][1][3], atol=eps())

    @test isapprox(rvs2[1][2][1], rvs2[3][2][1], atol=eps())
    @test isapprox(rvs2[1][2][2], rvs2[3][2][2], atol=eps())
    @test isapprox(rvs2[1][2][3], rvs2[3][2][3], atol=eps())
end

function test_datetime_ephem()
    sat = test_init()
    tstart = Dates.DateTime(2000, 6, 29, 12, 50, 19)
    tstop = Dates.DateTime(2000, 6, 29, 13, 50, 19)
    (pos, vel) = SGP4.propagate(sat, tstart, tstop, 60)

    @test isapprox(pos[1,1], 5576.056952,  atol=1e-6)
    @test isapprox(pos[2,1], -3999.371134, atol=1e-6)
    @test isapprox(pos[3,1], -1521.957159, atol=1e-6)

    @test isapprox(vel[1,1], 4.772627, atol=1e-6)
    @test isapprox(vel[2,1], 5.119817, atol=1e-6)
    @test isapprox(vel[3,1], 4.275553, atol=1e-6)

    @test size(pos,2) == 61
    @test size(vel,2) == 61

    (pos, vel) = SGP4.propagate(sat, tstart:Dates.Second(30):tstop)

    @test isapprox(pos[1,1], 5576.056952, atol=1e-6)
    @test isapprox(pos[2,1], -3999.371134,atol= 1e-6)
    @test isapprox(pos[3,1], -1521.957159,atol= 1e-6)

    @test isapprox(vel[1,1], 4.772627, atol=1e-6)
    @test isapprox(vel[2,1], 5.119817, atol=1e-6)
    @test isapprox(vel[3,1], 4.275553, atol=1e-6)

    @test size(pos,2) == 121
    @test size(vel,2) == 121
end

test_multiple_sats()
test_single_time()
test_datetime_ephem()


text:
using Match
using Test

input = joinpath(@__DIR__, "input")
raw = readlines(input)

struct Operation
    name::String
    value::Int
end

function operations(lines)
    acc = []
    for line in lines
        if startswith(line, "cut")
            _, n = split(line)
            push!(acc, Operation("cut", parse(Int, n)))
        elseif startswith(line, "deal into")
            push!(acc, Operation("reverse", 0))
        elseif startswith(line, "deal with")
            _, _, _, n = split(line)
            push!(acc, Operation("deal", parse(Int, n)))
        end
    end
    return acc
end

deal = (c, n) -> return mod(-1 * c - 1, n)
inc = (c, n, i) -> return mod(c * i, n)
cut = (c, n, i) -> return mod(c - i, n)

function shuffle(idx, deck, ops)
    c = idx
    n = deck
    for op in ops
        @match op.name begin
            "cut" => begin
                c = cut(c, n, op.value)
            end
            "reverse" => begin
                c = deal(c, n)
            end
            "deal" => begin
                c = inc(c, n, op.value)
            end
        end
    end
    return c
end

function fshuffle(idx, deck, ops, iterations)
    offset, increment = BigInt(0), BigInt(1)
    for op in ops
        @match op.name begin
            "cut" => begin
                n = mod(op.value, deck)
                offset = mod(offset + n, deck)
            end
            "reverse" => begin
                increment = mod(-1 * increment, deck)
                offset = mod(-1 * offset - 1, deck)
            end
            "deal" => begin
                n = invmod(mod(op.value, deck), deck)
                increment = mod(increment * n, deck)
                offset = mod(offset * n, deck)
            end
        end
    end
    i = powermod(increment, iterations, deck)
    o = if increment != 1
        (i - 1) * invmod(increment - 1, deck)
    else
        iterations
    end
    return mod(i * idx + o * offset, deck)
end

ops = operations(raw)
p1 = shuffle(2019, 10007, ops)
@assert p1 == 3589

reverse!(ops)
p2 = fshuffle(2020, 119315717514047, ops, 101741582076661)
@assert p2 == 4893716342290

print("-----------------------------------------------------------------------\n")
print("slam shuffle -- part one\n    card: $p1\n")
print("slam shuffle -- part two\n    card: $p2\n")
print("-----------------------------------------------------------------------\n")


text:
function variable_degrees{T<:Real}(num_variables::Integer, poly_degrees::AbstractVector{T})
    max_poly_degree = maximum(poly_degrees)
    poly_degrees = Set(poly_degrees)
    all_variable_degrees = Vector{Int}[]
    var_degrees = zeros(Int, num_variables)
    total = 0
    while var_degrees[1] <= max_poly_degree
        if in(total, poly_degrees)
            push!(all_variable_degrees, copy(var_degrees))
        end
        var_degrees[end] += 1
        total += 1
        for j = length(var_degrees):-1:2
            if var_degrees[j] > max_poly_degree || total > max_poly_degree
                total -= var_degrees[j] - 1
                var_degrees[j] = 0
                var_degrees[j-1] += 1
            else
                break
            end
        end
    end
    return all_variable_degrees
end

function monomial(variable::Symbol, degree::Integer)
    MPoly{Float64}(OrderedDict([degree]=>1.0), [variable])
end

function chebyshev_polynomial_first_kind(variable::Symbol, degree::Integer)
    x = generator(variable)
    @assert degree >= 0
    if degree == 0
        return 1 + 0 * x
    elseif degree == 1
        return x
    else
        return (2x * chebyshev_polynomial_first_kind(variable, degree - 1)
                - chebyshev_polynomial_first_kind(variable, degree - 2))
    end
end

function polynomial_basis(variables, degrees=0:2, generator::Function=monomial)
    map(d -> prod(map(i -> generator(variables[i], d[i]),
                  1:length(variables))),
        variable_degrees(length(variables), degrees))
end

monomials(variables, degrees=0:2) = polynomial_basis(variables, degrees, monomial)
chebyshev_basis_first_kind(variables, degrees=0:2) = polynomial_basis(variables, degrees, chebyshev_polynomial_first_kind)