text:
#=
Copyright (c) 2015, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice,
  this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
=#

module OptFramework

import ..DebugMsg
DebugMsg.init()

using CompilerTools
using CompilerTools.Helper

export @acc, @noacc, addOptPass
export PASS_MACRO, PASS_LOWERED, PASS_UNOPTTYPED, PASS_TYPED

const PASS_MACRO = 0 :: Int
const PASS_LOWERED = 1 :: Int
const PASS_UNOPTTYPED = 2 :: Int
const PASS_TYPED = 3 :: Int

"""
pretty print pass level number as string.
"""
function dumpLevel(level)
  if level == PASS_MACRO "macro AST"
  elseif level == PASS_LOWERED "lowered AST"
  elseif level == PASS_UNOPTTYPED "unoptimized typed AST"
  elseif level == PASS_TYPED "typed AST"
  else "unknown AST"
  end
end

"""
A data structure that holds information about one high-level optimization pass to run.
"func" is the callback function that does the optimization pass and should have the signature (GlobalRef, Expr, Tuple) where the GlobalRef provides the locate of the function to be optimized, Expr is the AST input to this pass, and Tuple is a tuple of all parameter types of the functions. It must return either an optimized Expr, or a Function.
"level" indicates at which level this pass is to be run.
"""
type OptPass
    func  :: Function
    level :: Int # at which level this OptPass is run
end

# Stores the default set of optimization passes if they are not specified on a line-by-line basis.
optPasses = OptPass[]

"""
Set the default set of optimization passes to apply with the @acc macro.
"""
function setOptPasses(passes :: Array{OptPass,1})
  for pass in passes
    addOptPass(pass)
  end
end

"""
Sometimes the original function shouldn't be evaluated since it is not valid Julia syntax.
This happens when DSL constructs are used that cannot have function implementations.
    motivation: DataTables in HPAT
"""
saveOriginalFunction = true
function setSaveOriginalFunction(val::Bool)
    global saveOriginalFunction = val
end

"""
Add an optimization pass. If this is going to be called multiple times then you need some external way of corrdinating the code/modules that are calling this function so that optimization passes are added in some sane order.
"""
function addOptPass(pass :: OptPass)
  level = -1
  if length(optPasses) > 0
    level = optPasses[end].level
  end
  if pass.level < level
      throw(string("Optimization passes cannot handle ", dumpLevel(pass.level), " pass after ", dumpLevel(level)))
  end
  @dprintln(2, "Add optimization pass at level ", dumpLevel(pass.level))
  push!(optPasses, pass)
end

"""
Same as the other addOptPass but with a pass call back function and pass level as input.
"""
function addOptPass(func, level)
  addOptPass(OptPass(func, level))
end

#type FullFuncInfo
#  func
#  sig
#  code
#end

"""
Retrieve the AST of the given function "func" and signature "sig" for at the given pass "level".
"""
function getCodeAtLevel(func, sig::Tuple, level)
  @dprintln(3, "getCodeAtLevel ", func, " ", sig, " ", level)
  if level == PASS_MACRO
    error("AST at macro level must be passed in, and cannot be obtained from inspecting the function itself")
  elseif level == PASS_LOWERED
    ast = code_lowered(func, sig)
  elseif level == PASS_UNOPTTYPED
    ast = code_typed(func, sig, optimize=false)
  elseif level == PASS_TYPED
    ast = code_typed(func, sig, optimize=true)
  else
    error("Unknown AST level: ", level)
  end

  @dprintln(3, "getCodeAtLevel ", dumpLevel(level), " ast = ", ast)
  @assert (length(ast) > 0) ("Failed to retrieve AST for function " * dump(func) * " with signature " * dump(sig))
  @assert (length(ast) == 1) ("Expect one AST but got many for function " * dump(func) * " with signature " * dump(sig) * ": " * dump(ast))
  @dprintln(3, "typeof(ast[1]) ", typeof(ast[1]))
  ast = ast[1]
  if VERSION >= v"0.6.0-pre"
    @dprintln(3, "Return type = ", ast.second)
    t1 = ast
    ast = LambdaInfo(func, sig, t1)
  end

  @assert (isfunctionhead(ast)) #("Expected function head but got " * typeof(ast[1]))
  @dprintln(3, "getCodeAtLevel returns ", ast)
  return ast
end

"""
convert AST from "old_level" to "new_level". The input "ast" can be either Expr or Function type. In the latter case, the result AST will be obtained from this function using an matching signature "sig". The last "func" is a skeleton function that is used internally to facility such conversion.
"""
function convertCodeToLevel(ast :: ANY, sig :: ANY, old_level, new_level, func)
  @dprintln(3,"convertCodeToLevel sig = ", sig, " ", old_level, "=>", new_level, " func = ", func, " typeof(sig) = ", typeof(sig), " typeof(ast) = ", typeof(ast), " typeof(func) = ", typeof(func))
  if isa(ast, Function)
    return getCodeAtLevel(ast, sig, new_level)
  end
  if old_level == new_level
    return ast
  end
  if old_level == PASS_MACRO
    error("Code in macro must be evaled first before converted")
  else
    func_methods = methods(func, sig)
    @assert (length(func_methods) == 1) ("Expect only one matching method for " * func * " of signature " * sig * " but got " * length(func_methods))
    lambda = func_methods[1].func.code
    @assert (isa(lambda, LambdaInfo)) ("LambdaInfo not found for " * func * " of signature " * sig)
    lambda.ast = ccall(:jl_compress_ast, Any, (Any,Any), lambda, ast)
    if new_level == PASS_UNOPTTYPED
      @dprintln(3,"Calling lambdaTypeinf with optimize=false")
      (new_ast, rty) = CompilerTools.LambdaHandling.lambdaTypeinf(lambda, sig, optimize=false)
    else
      assert(new_level == PASS_TYPED)
      @dprintln(3,"Calling lambdaTypeinf with optimize=true")
      @dprintln(4,"lambda = ", lambda)
      new_ast = ast
      #(new_ast, rty) = CompilerTools.LambdaHandling.lambdaTypeinf(lambda, sig, optimize=true)
    end
    @assert (isa(new_ast, Expr) && new_ast.head == :lambda) ("Expect Expr with :lambda head, but got " * new_ast)
    if VERSION >= v"0.6.0-pre"
        ast.code = new_ast
        return ast
    else
        return new_ast
    end
  end
end

"""
A global memo-table that maps both: the triple (function, signature, optPasses) to the trampoline function, and the trampoline function to the real function.
"""
gOptFrameworkDict = Dict{Any,Any}()

"""
This function removes duplicate labels.
"""
function removeDupLabels(stmts)
  if length(stmts) == 0
    return Any[]
  end

  ret = Any[]

  push!(ret, stmts[1])
  last_is_label = false
  last_label = -1
  labelMap = Dict{Int,Int}()

  for i = 2:length(stmts)
    if isa(stmts[i], LabelNode)
      if last_is_label
         labelMap[stmts[i].label] = last_label
         continue
      else
         last_is_label = true
         last_label = stmts[i].label
         labelMap[last_label] = last_label
      end
    elseif isa(stmts[i], LineNumberNode)
    else
      last_is_label = false
    end
    push!(ret, stmts[i])
    #push!(ret, LineNumberNode(i))
  end

  updateBodyWithLabelMap!(ret, labelMap)

  ret
end

function updateBodyWithLabelMap!(body::Array{Any,1}, labelMap :: Dict{Int,Int})
  for i=1:length(body)
    if isa(body[i], LabelNode)
      body[i] = LabelNode(labelMap[body[i].label])
    elseif isa(body[i], GotoNode)
      body[i] = GotoNode(labelMap[body[i].label])
    elseif isa(body[i], Expr) && (body[i].head === :gotoifnot)
      body[i].args[2] = labelMap[body[i].args[2]]
    end
  end
end

"""
Update labels with a new base, return the max label.
"""
function updateLabels!(body::Array{Any,1}, base=0)
  labelMap = Dict{Int,Int}()
  for i=1:length(body)
    if isa(body[i], LabelNode)
      labelMap[body[i].label] = base + i
    end
  end
  updateBodyWithLabelMap!(body, labelMap)
  return base + length(body)
end

"""
Clean up the labels in AST by renaming them, and removing duplicates.
"""
function cleanupBodyLabels(expr::Expr)
  @assert (expr.head == :body)
  @dprintln(3, "cleanupBodyLabels ", expr)
  body = removeDupLabels(expr.args)
  expr.args = body
  @dprintln(3, "cleanupBodyLabels after remove dup labels ", expr)
  max_label = updateLabels!(body)
  expr.args = body
  @dprintln(3, "cleanupBodyLabels after updateLabels! ", expr)
  expr.args = body
  return expr
end

function cleanupBodyLabels(body::Array)
  expr = Expr(:body)
  expr.args = body
  return cleanupBodyLabels(expr).args
end

function eliminateParamAssigns(linfo, body)
  @dprintln(3,"eliminateParamAssigns: body = ", body)
  # Create a set of the input parameters in LHSVar form.
  input_lhsvars = Set{LHSVar}(CompilerTools.LambdaHandling.getInputParametersAsLHSVar(linfo))
  @dprintln(3,"eliminateParamAssigns: input_lhsvars = ", input_lhsvars)
  # Do liveness analysis on the body.
  lives = CompilerTools.LivenessAnalysis.from_lambda(linfo, body)
  assigned_params = Set{LHSVar}()
  # For each basic block in the AST.
  for bbentry in lives.basic_blocks
    bb = bbentry[2]
    # Find which input parameters are assigned in the block by the intersection of the input lhsvars and the definitions in the basic block.
    # Add those modified parameters to the set of all modified parameters in assigned_params.
    assigned_params = union(assigned_params, intersect(input_lhsvars, bb.def))
    @dprintln(3,"eliminateParamAssigns: assigned_params = ", assigned_params, " bb.def = ", bb.def)
  end

  new_body = Any[]

  rdict = Dict{LHSVar, Any}()
  for ap in assigned_params
    vd = CompilerTools.LambdaHandling.getVarDef(ap, linfo)
    @dprintln(3,"eliminateParamAssigns: vd = ", vd)
    rhsvar_copy = CompilerTools.LambdaHandling.addLocalVariable(Symbol(string(vd.name, "_duplicate_to_eliminate_param_assign")), vd.typ, CompilerTools.LambdaHandling.ISASSIGNED, linfo)
    rdict[ap] = rhsvar_copy
    push!(new_body, TypedExpr(vd.typ, :(=), toLHSVar(rhsvar_copy), ap))
  end
  @dprintln(3,"new_body = ", new_body)

  if !isempty(rdict)
      body = CompilerTools.LambdaHandling.replaceExprWithDict!(body, rdict, linfo)
      @dprintln(3,"body after replaceExprWithDict! = ", body, " type = ", typeof(body))
#      for i = 1:3
#          push!(new_body, body.args[i])
#      end
#      push!(new_body, Expr(:return, 1.0))
      append!(new_body, body.args)
      body.args = new_body
      @dprintln(3,"eliminateParamAssigns: final body = ", body)
      @dprintln(3,"eliminateParamAssigns: linfo = ", linfo)
  end

  return linfo, body
end

function cleanupFunction(linfo, body)
  linfo, new_body = eliminateParamAssigns(linfo, body)
  @dprintln(3, "new_body = ", new_body, " type = ", typeof(new_body))
  new_body = CompilerTools.AstWalker.AstWalk(new_body, invoke_to_call, nothing)
  @dprintln(3, "new_body after transforming invoke back to call = ", new_body)
  new_body = cleanupBodyLabels(new_body)
  @dprintln(3, "new_body after label cleanup = ", new_body)
  new_body = CompilerTools.AstWalker.AstWalk(new_body, eliminate_typing, nothing)
  @dprintln(3, "new_body after eliminating typing = ", new_body)
  #new_body_args = CompilerTools.LambdaHandling.getBody(ast).args
  return linfo, new_body
end

if VERSION >= v"0.6.0-pre"

serialize_ast = false

"""
Makes sure that a newly created function is correctly present in the internal Julia method table.
"""
function setCode(func, arg_tuple, ast)
  assert(isfunctionhead(ast))
  #tfuncPresent(func, arg_tuple)
  @dprintln(2, "setCode fields of ast")
#  CompilerTools.Helper.print_by_field(ast)
  @dprintln(2, "setCode fields of ast.def")
#  CompilerTools.Helper.print_by_field(ast.def)

#  info = code_typed(func, arg_tuple)[1]
  def = ast.meth
  meth = methods(func, arg_tuple)
  @dprintln(2, "meth = ", meth, " type = ", typeof(meth))
  def = meth.ms[1]
  assert(isa(def,Method))
  @dprintln(2, "def = ", def, " type = ", typeof(def), " ", fieldnames(def))
  #CompilerTools.Helper.print_by_field(def)
  defspec = def.specializations
  @dprintln(2, "defspec = ", defspec, " type = ", typeof(defspec), " ", fieldnames(defspec))
  #CompilerTools.Helper.print_by_field(defspec)
  defspecfunc = defspec.func
  @dprintln(2, "defspecfunc = ", defspecfunc, " type = ", typeof(defspecfunc), " ", fieldnames(defspecfunc))
  #CompilerTools.Helper.print_by_field(defspecfunc)
  mt = meth.mt
  @dprintln(2, "mt = ", mt, " type = ", typeof(mt), " ", fieldnames(mt))
  #CompilerTools.Helper.print_by_field(mt)
  mtdefs = mt.defs
  @dprintln(2, "mtdefs = ", mtdefs, " type = ", typeof(mtdefs), " ", fieldnames(mtdefs))
  #CompilerTools.Helper.print_by_field(mtdefs)
  mdf = mtdefs.func
  @dprintln(2, "mdf = ", mdf, " type = ", typeof(mdf), " ", fieldnames(mdf))
  #CompilerTools.Helper.print_by_field(mdf)

  @dprintln(2, "ast slotnames: ", ast.code.first.slotnames)
#  @dprintln(2, "setCode fields of def")
#  CompilerTools.Helper.print_by_field(def)
  @dprintln(2, "typeof(def.specializations) = ", typeof(def.specializations))

  @dprintln(3, "ast.code.first = ", ast.code.first)
  #CompilerTools.Helper.print_by_field(ast.code.first)

  if serialize_ast
    (tn, sfile) = mktemp()
    serialize(sfile, ast)
    close(sfile)
    println("setCode for ", func, " is ", tn)
  end

  for i = 1:length(ast.code.first.slottypes)
    ast.code.first.slottypes[i] = Any
  end
  @dprintln(2, "setCode ast before precompile = ", ast)

  def.specializations = nothing
  def.source = ast.code.first
  #def.source = ast.code.first.code
  #mt.defs.func = ast.code.first.code

  #throw(string("stop here"))
  defspecfunc.inferred = false
  precompile(func, arg_tuple)

#  info = code_typed(func, arg_tuple)[1]
#  CompilerTools.Helper.print_by_field(info)

  @dprintln(2, "setCode fields of def after")
#  CompilerTools.Helper.print_by_field(def)
  @dprintln(2, "setCode fields of def.specializations.func after")
#  CompilerTools.Helper.print_by_field(def.tfunc.func)

  #@dprintln(2, def.specializations)
  @dprintln(2, "code typed")
  @dprintln(2, code_typed(func, arg_tuple))
  @dprintln(2, "Done precompiling.")
end

elseif VERSION > v"0.5.0-dev+3260"

serialize_ast = false

"""
Makes sure that a newly created function is correctly present in the internal Julia method table.
"""
function setCode(func, arg_tuple, ast)
  assert(isfunctionhead(ast))
  #tfuncPresent(func, arg_tuple)
  @dprintln(2, "setCode fields of ast")
#  CompilerTools.Helper.print_by_field(ast)
  @dprintln(2, "setCode fields of ast.def")
#  CompilerTools.Helper.print_by_field(ast.def)

  info = code_typed(func, arg_tuple)[1]
  meth_list = methods(func, arg_tuple).ms
  @dprintln(2, "meth_list = ", meth_list, " type = ", typeof(meth_list), " len = ", length(meth_list))
  def = meth_list[1]
  assert(isa(def,Method))

  @dprintln(2, "ast slotnames: ", ast.slotnames)
#  @dprintln(2, "setCode fields of def")
#  CompilerTools.Helper.print_by_field(def)
  @dprintln(2, "typeof(def.specializations) = ", typeof(def.specializations))
#  @dprintln(2, "setCode fields of def.specializations")
#  CompilerTools.Helper.print_by_field(def.specializations)
  #assert(isa(def.specializations.func, LambdaInfo))
#  @dprintln(2, "setCode fields of def.specializations.func")
#  CompilerTools.Helper.print_by_field(def.tfunc.func)

  #ast.code = deepcopy(CompilerTools.LambdaHandling.getBody(ast).args)
  def.specializations = nothing
  #def.specializations.func = ast
  #def.specializations.func.def = def
#  ast.code = new_body.args # ccall(:jl_compress_ast, Any, (Any,Any), ast, new_body_args)
  def.lambda_template = ast # def.specializations.func
#  ast.slotnames[1] = info.slotnames[1]
#  ast.slottypes[1] = info.slottypes[1]
#  ast.slotflags[1] = info.slotflags[1]

  if serialize_ast
    (tn, sfile) = mktemp()
    serialize(sfile, ast)
    close(sfile)
    println("setCode for ", func, " is ", tn)
  end

  for i = 1:length(ast.slottypes)
    ast.slottypes[i] = Any
  end
  @dprintln(2, "setCode ast before precompile = ", ast)

  #throw(string("stop here"))
  ast.inferred = false
  #ast.inferred = true
  #@dprintln(2, printExprAst(ast))
  precompile(func, arg_tuple)

#  info = code_typed(func, arg_tuple)[1]
#  CompilerTools.Helper.print_by_field(info)

  @dprintln(2, "setCode fields of def after")
#  CompilerTools.Helper.print_by_field(def)
  @dprintln(2, "setCode fields of def.specializations.func after")
#  CompilerTools.Helper.print_by_field(def.tfunc.func)

  #@dprintln(2, def.specializations)
  @dprintln(2, "code typed")
  @dprintln(2, code_typed(func, arg_tuple))
  @dprintln(2, "Done precompiling.")
end

else
"""
Makes sure that a newly created function is correctly present in the internal Julia method table.
"""
function tfuncPresent(func, tt)
  meth_list = methods(func, tt)
  @dprintln(2, "meth_list = ", meth_list, " type = ", typeof(meth_list))
  m = meth_list[1]
  def = m.func.code
  if def.tfunc === nothing
    @dprintln(2, "tfunc NOT present before code_typed")
    code_typed(func, tt)
    if def.tfunc === nothing
      error("tfunc still NOT present after code_typed")
    else
      @dprintln(2, "tfunc present after code_typed")
    end
  else
    @dprintln(2, "tfunc present on call")
  end
end

function setCode(func, arg_tuple, ast)
  assert(isfunctionhead(ast))
  tfuncPresent(func, arg_tuple)
  method = methods(new_func, call_sig_arg_tuple)
  assert(length(method) == 1)
  method = method[1]
  @dprintln(2, "tfunc type = ", typeof(method.func.code.tfunc))
  method.func.code.tfunc[2] = ccall(:jl_compress_ast, Any, (Any,Any), method.func.code, ast)
end
end

"""
Takes a function, a signature, and a set of optimizations and applies that set of optimizations to the function,
returns a new optimized function without modifying the input.  Argument explanation follows:
1) func - the function being optimized
2) call_sig_arg_tuple - the signature of the function, i.e., the types of each of its arguments
3) per_site_opt_set - the set of optimization passes to apply to this function.
"""
function processFuncCall(func :: ANY, call_sig_arg_tuple :: ANY, per_site_opt_set :: ANY)
  @dprintln(3,"processFuncCall starting = ", func, " func type = ", typeof(func), " call_sig_arg_tuple = ", call_sig_arg_tuple)
  @assert (isa(func, Function)) ("processFuncCall can only optimize functions, but got " * typeof(func))
  if per_site_opt_set == nothing
    per_site_opt_set = optPasses
  end
  @assert (length(per_site_opt_set) > 0) "There are no registered optimization passes."
  func_module = Base.function_module(func, call_sig_arg_tuple)
  func_ref = GlobalRef(func_module, Base.function_name(func))
  @dprintln(3,"processFuncCall ", func_ref, " ", call_sig_arg_tuple, " opt_set = ", per_site_opt_set)

  # Create a skeleton of the incoming function, which is used to facilitate AST conversion, and will eventually be the result function to be returned.
  new_func_name = gensym(string(func_ref))
  fake_args = Any[ gensym() for a in call_sig_arg_tuple ]
  new_func = Core.eval(func_module, :(function $(new_func_name)($(fake_args...)) end))
  @dprintln(2,"temp_func is ", new_func)
  code_typed(new_func, call_sig_arg_tuple)

  # Remember what level the AST was in.
  cur_level = per_site_opt_set[1].level
  cur_ast = func

  @dprintln(3,"Initial code to optimize = ", cur_ast, " ", typeof(cur_ast))

  # For each optimization pass in the optimization set.
  for i = 1:length(per_site_opt_set)
      @dprintln(3,"optimization pass ", i)
      # See if the current optimization pass uses optimized AST form and the previous optimization pass used unoptimized AST form.
      cur_level = per_site_opt_set[i].level
      if cur_level > PASS_MACRO
        cur_ast = convertCodeToLevel(cur_ast, call_sig_arg_tuple, cur_level, cur_level, new_func)
        #assert(isfunctionhead(cur_ast))
        # Call the current optimization on the current AST.
        try
          cur_ast = per_site_opt_set[i].func(func_ref, cur_ast, call_sig_arg_tuple)
          #assert(isfunctionhead(cur_ast) || isa(cur_ast, Function))
        catch texp
          if CompilerTools.DebugMsg.PROSPECT_DEV_MODE
            rethrow(texp)
          end
          println("OptFramework failed to optimize function ", func, " in optimization pass ", per_site_opt_set[i].func, " with error ", texp)
          return nothing
        end
        @dprintln(3,"AST after optimization pass ", i, " = ", cur_ast)
      end
  end

  @dprintln(3,"After optimization passes")

  #if isa(cur_ast, Expr)
  if isa(cur_ast,Tuple) && length(cur_ast)==2 && isa(cur_ast[1], CompilerTools.LambdaHandling.LambdaVarInfo) && isa(cur_ast[2], Expr)
    lvi = cur_ast[1]
    body = cur_ast[2]
    @dprintln(3,"After last optimization pass, got Tuple of LambdaVarInfo and Expr with head = ", body.head)
    linfo, body = convertCodeToLevel(cur_ast, call_sig_arg_tuple, cur_level, PASS_TYPED, new_func)
    linfo, body = cleanupFunction(linfo, body)
    ast = CompilerTools.LambdaHandling.LambdaVarInfoToLambda(linfo, body)
    assert(isfunctionhead(ast))
    @dprintln(3,"Last opt pass after converting to typed AST.\n", CompilerTools.LambdaHandling.getBody(ast))
    setCode(new_func, call_sig_arg_tuple, ast)

    @dprintln(3,"Final processFuncCall = ", code_typed(new_func, call_sig_arg_tuple)[1])
    return new_func
  elseif isfunctionhead(cur_ast)
    ast = convertCodeToLevel(cur_ast, call_sig_arg_tuple, cur_level, PASS_TYPED, new_func)
    assert(isfunctionhead(ast))
    @dprintln(3,"Last opt pass after converting to typed AST.\n", CompilerTools.LambdaHandling.getBody(ast))

    # Write the modifed code back to the function.
    @dprintln(2,"Before methods at end of processFuncCall.")
    setCode(new_func, call_sig_arg_tuple, ast)
#    tfuncPresent(new_func, call_sig_arg_tuple)
#    method = methods(new_func, call_sig_arg_tuple)
#    assert(length(method) == 1)
#    method = method[1]
#    method.func.code.tfunc[2] = ccall(:jl_compress_ast, Any, (Any,Any), method.func.code, cur_ast)

    @dprintln(3,"Final processFuncCall = ", code_typed(new_func, call_sig_arg_tuple)[1])
    return new_func
  elseif isa(cur_ast, Function)
  #elseif isa(cur_ast, FullFuncInfo)
  #  @dprintln(3,"AST after optimization passes is of type ", typeof(cur_ast.code), ".")
    return cur_ast
  else
    @dprintln(1,"typeof(cur_ast) = ", typeof(cur_ast))
    error("Expect either AST or Function after processFuncCall, but got ", cur_ast)
  end
end

"""
A hack to get around Julia's type inference. This is essentially an identity conversion,
but forces inferred return type to be the given type.
Characters coming from C should have Cchar type (which is single byte); Julia can convert them to Char (multi byte).
"""
function identical(::Type{Char}, x::Cchar)
    y::Char = x
    return y
end

identical{T}(::Type{T}, x::T)=x

function createWrapperFuncArgWithType(i, arg::Symbol)
    return Symbol(string("makeWrapperFuncArgument",i))
end

function createWrapperFuncArgWithType(i, arg::Expr)
    assert(arg.head == :(::))
    typ = arg.args[2]
    return Expr(:(::), Symbol(string("makeWrapperFuncArgument",i)), typ)
end

if VERSION >= v"0.5"
  gOptFrameworkDictLock = Base.Threads.Mutex()

  function lock_check(dict, fs, n)
     lock(gOptFrameworkDictLock)
     func_to_call = get(dict, fs, nothing)
     return func_to_call
  end

else

  function lock_check(dict, fs, n)
     func_to_call = get(dict, fs, nothing)
     return func_to_call
  end

end

"""
Define a wrapper function with the name given by "new_func" that when called will try to optimize the "real_func" function, and run it with given parameters in "call_sig_args". The input "per_site_opt_set" can be either nothing, or a quoted Expr that refers to an array of OptPass.
"""
function makeWrapperFunc(new_fname::Symbol, real_fname::Symbol, call_sig_args::Array{Any, 1}, per_site_opt_set)
  mod = current_module()
  new_func = GlobalRef(mod, new_fname)
  real_func = GlobalRef(mod, real_fname)
  @dprintln(3, "Create wrapper function ", new_func, " for actual function ", real_func)

  #bt = backtrace() ;
  #s = sprint(io->Base.show_backtrace(io, bt))
  #@dprintln(3, "makeWrapperFunc backtrace ")
  #@dprintln(3, s)

  @dprintln(3, "call_sig_args = ", call_sig_args)
  new_call_sig_args = Symbol[ Symbol(string("makeWrapperFuncArgument",i)) for i = 1:length(call_sig_args)]
  new_call_sig_args_types = [ Expr(:call, GlobalRef(Base, :typeof), arg) for arg in new_call_sig_args]
  new_call_sig_args_with_types = Any[ createWrapperFuncArgWithType(i, call_sig_args[i]) for i = 1:length(call_sig_args)]
  @dprintln(3, "new_call_sig_args = ", new_call_sig_args)
  @dprintln(3, "new_call_sig_args_with_types = ", new_call_sig_args_with_types)
  lc = GlobalRef(CompilerTools.OptFramework, :lock_check)
  gofdl = GlobalRef(CompilerTools.OptFramework, :gOptFrameworkDictLock)
  gofd = GlobalRef(CompilerTools.OptFramework, :gOptFrameworkDict)
  proc = GlobalRef(CompilerTools.OptFramework, :processFuncCall)
  dpln = GlobalRef(CompilerTools.OptFramework, :dprintln)
  idtc = GlobalRef(CompilerTools.OptFramework, :identical)

  if VERSION >= v"0.5.0-dev+5381"
    rettype = Expr(:call, GlobalRef(Core.Inference, :return_type), real_func,
                Expr(:call, GlobalRef(Core, :apply_type), GlobalRef(Base, :Tuple), new_call_sig_args_types...))
    retexpr = Expr(:call, idtc, rettype, :res)
  else
    retexpr = Expr(:call, idtc, Expr(:static_typeof, :ret), :res)
  end
  wrapper_ast = :(function $new_fname($(new_call_sig_args_with_types...))
         #CompilerTools.OptFramework.@dprintln(3,"new_func running ", $(new_call_sig_args...))
         call_sig_arg_tuple = tuple($(new_call_sig_args_types...))
         opt_set = $per_site_opt_set
         fs = ($real_func, call_sig_arg_tuple, opt_set)
         func_to_call = $lc($gofd, fs, nothing)
         if func_to_call == nothing
           tic()
           $dpln(1,"wrapper real_func = ", $real_func, " type = ", typeof($real_func))
           process_res = $proc($real_func, call_sig_arg_tuple, opt_set)
           t = toq()
           $dpln(1,$real_func," optimization time = ", t)
           if process_res != nothing
             # We did optimize it in some way we will call the optimized version.
             $dpln(3,"processFuncCall DID optimize ", $real_func)
             func_to_call = process_res
           else
             # We did not optimize it so we will call the original function.
             $dpln(3,"processFuncCall didn't optimize ", $real_func)
             func_to_call = $real_func
           end # Remember this optimization result for this function/type combination.
           $gofd[fs] = func_to_call
         end

         if VERSION >= v"0.5"
           unlock($gofdl)
         end

         $dpln(3,"calling ", func_to_call)
         if 1 < 0
           ret = $real_func($(new_call_sig_args...))
         end
         # This eval is necessary to get around the world-age issue with Julia 0.6.
         res = eval(Expr(:call, func_to_call, $(new_call_sig_args...)))
         $retexpr
        end)
  @dprintln(4,"wrapper_ast = ", wrapper_ast, " type = ", typeof(wrapper_ast))
  gOptFrameworkDict[new_func] = real_func
  return wrapper_ast
end

type opt_calls_insert_trampoline_state
  per_site_opt_set
  found_callsite

  function opt_calls_insert_trampoline_state(opt_set)
    new(opt_set, false)
  end
end

function invoke_to_call(x :: Expr, state, top_level_number, is_top_level, read)
  if x.head == :invoke || x.head == :call
    local fun  = getCallFunction(x)
    local args = getCallArguments(x)

    if isa(fun, GlobalRef)
      if fun.mod === Core.Intrinsics
        return TypedExpr(x.typ, :call, GlobalRef(Base, fun.name), args...)
      end
    end
  end

  if x.head == :invoke
    return TypedExpr(x.typ, :call, x.args[2:end]...)
  end
  return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function invoke_to_call(x, state, top_level_number, is_top_level, read)
  return CompilerTools.AstWalker.ASTWALK_RECURSE
end

function eliminate_typing(x :: TypedVar, state, top_level_number, is_top_level, read)
  return toLHSVar(x)
end

function eliminate_typing(x :: Expr, state, top_level_number, is_top_level, read)
  for i = 1:length(x.args)
    x.args[i] = CompilerTools.AstWalker.AstWalk(x.args[i], eliminate_typing, nothing)
  end

  x.typ = Any
  return x
end

function eliminate_typing(x, state, top_level_number, is_top_level, read)
  return CompilerTools.AstWalker.ASTWALK_RECURSE
end

"""
An AstWalk callback function.
Finds call sites in the AST and replaces them with calls to newly generated trampoline functions.
These trampolines functions allow us to capture runtime types which in turn enables optimization passes to run on fully typed AST.
If a function/signature combination has not previously been optimized then call processFuncCall to optimize it.
"""
function opt_calls_insert_trampoline(x, state:: opt_calls_insert_trampoline_state, top_level_number, is_top_level, read)
  if typeof(x) == Expr
    if x.head == :call
      # We found a call expression within the larger expression.
      call_expr = x.args[1]          # Get the target of the call.
      call_sig_args = x.args[2:end]  # Get the arguments to the call.
      @dprintln(2, "Start opt_calls = ", call_expr, " signature = ", call_sig_args, " typeof(call_expr) = ", typeof(call_expr))

      # The name of the new trampoline function.
      real_func = Symbol(string(call_expr))

      # Recursively process the arguments to this function possibly finding other calls to replace.
      for i = 2:length(x.args)
        x.args[i] = CompilerTools.AstWalker.AstWalk(x.args[i], opt_calls_insert_trampoline, state)
      end

      trampoline_func = Symbol(string("opt_calls_trampoline_", real_func))
      wrapper_ast = makeWrapperFunc(trampoline_func, real_func, call_sig_args, state.per_site_opt_set)
      Core.eval(current_module(), wrapper_ast)
      # Update the call expression to call our trampoline and pass the original function so that we can
      # call it if nothing can be optimized.
      x.args = [ trampoline_func; x.args[2:end] ]

      @dprintln(2, "Replaced call_expr = ", call_expr, " type = ", typeof(call_expr), " new = ", x.args[1])

      state.found_callsite = true

      return x
    end
  end
  return CompilerTools.AstWalker.ASTWALK_RECURSE
end

"""
When @acc is used at a function's callsite, we use AstWalk to search for callsites via the opt_calls_insert_trampoline callback and to then insert trampolines.
That updated expression containing trampoline calls is then returned as the generated code from the @acc macro.
"""
function convert_expr(per_site_opt_set, ast)
  if per_site_opt_set === nothing && length(optPasses) == 0
    # skip translation since opt_set is empty
    return ast
  else
    @dprintln(2, "convert_expr ", ast, " ", typeof(ast), " per_site_opt_set = ", per_site_opt_set)
    state = opt_calls_insert_trampoline_state(per_site_opt_set)
    res = CompilerTools.AstWalker.AstWalk(ast, opt_calls_insert_trampoline, state)
    @dprintln(2,"converted expression = ", res)

    if state.found_callsite == false
        println("WARNING: @acc macro used on an expression not containing a call-site.")
    end

    return res
  end
end

type WorkItem
  per_site_opt_set
  opt_set
  macros
  ast
end

type MoreWork
  returned_macro_ast
  more_work :: Array{WorkItem,1}
end

"""
When @acc is used at a function definition, it creates a trampoline function, when called with a specific set of signature types, will try to optimize the original function, and call it with the real arguments.  The input "ast" should be an AST of the original function at macro level, which will be   replaced by the trampoline.
"""
function convert_function(per_site_opt_set, opt_set, macros, ast)
    @dprintln(3,"convert_function ast = ", ast, " typeof(ast) = ", typeof(ast))
    if isa(ast, Expr)
       @dprintln(3,"ast.head = ", ast.head)
    end
    fname = ast.args[1].args[1]
    assert(isa(fname, Symbol))
    mod = current_module()
    call_sig_args = ast.args[1].args[2:end]
    macro_ast     = deepcopy(ast)
    macro_only    = all(Bool[p.level <= PASS_MACRO for p in opt_set]) || (length(macros) > 0 && macros[1] == Symbol("@inline"))
    real_fname    = gensym(string(fname))
    real_func     = GlobalRef(mod, real_fname)
    macro_fname   = macro_only ? fname : gensym(string(fname))
    macro_func    = GlobalRef(mod, macro_fname)
    ast.args[1].args[1] = real_fname
    macro_ast.args[1].args[1] = macro_fname
    work_array = WorkItem[]
    orig_macro_ast_type = typeof(macro_ast)
    @dprintln(3, "@acc convert_function fname = ", fname, " real_fname = ", real_fname, " macro_fname = ", macro_fname)

    @dprintln(3, "@acc converting function = ", macro_ast)
    for i in 1:length(opt_set)
      x = opt_set[i]
      if x.level == PASS_MACRO
        macro_ast = x.func(macro_func, macro_ast, nothing) # macro level transformation only takes input ast as its argument
        if typeof(macro_ast) != orig_macro_ast_type
            @dprintln(3, "Macro pass returned more work = ", macro_ast.more_work)
            assert(typeof(macro_ast) == MoreWork)
            append!(work_array, macro_ast.more_work)
            macro_ast = macro_ast.returned_macro_ast
        end
        @dprintln(3, "After pass[", i, "], AST = ", macro_ast)
      end
    end
    if length(macros) > 0
       @dprintln(3, "Adding macrocall macros = ", macros)
       ast = Expr(:macrocall, macros..., ast)
       macro_ast = Expr(:macrocall, macros..., macro_ast)
    end
    if !macro_only
      @dprintln(3, "Calling makeWrapperFunc")
      if saveOriginalFunction
          ast = Expr(:block, ast, macro_ast, makeWrapperFunc(fname, macro_fname, call_sig_args, per_site_opt_set))
      else
          ast = Expr(:block, macro_ast, makeWrapperFunc(fname, macro_fname, call_sig_args, per_site_opt_set))
      end
    else
      @dprintln(3, "macro_only conversion")
      if saveOriginalFunction
          ast = Expr(:block, ast, macro_ast)
      else
          ast = Expr(:block, macro_ast)
      end
    end
    @dprintln(3, "macro_func = ", macro_func, " real_func = ", real_func)
    gOptFrameworkDict[macro_func] = real_func

    if !isempty(work_array)
        @dprintln(3, "@acc converting function recursively processing work queue")
    end

    for item in work_array
        @dprintln(3, "@acc work item = ", item)

        item_res = convert_function(item.per_site_opt_set, item.opt_set, item.macros, item.ast)
        assert(is_block(item_res))
        append!(ast.args, item_res.args)
    end

    @dprintln(3, "@acc converting function done = ", ast)

    ast
end

function is_function(expr)
  isa(expr, Expr) && expr.head === :function || expr.head === :(=) && isa(expr.args[1], Expr) && expr.args[1].head === :call
end

function is_block(expr)
  isa(expr, Expr) && expr.head === :block
end

function is_macro(expr)
  isa(expr, Expr) && expr.head === :macrocall
end

function convert_block(per_site_opt_set, opt_set, macros, ast)
  for i = 1 : length(ast.args)
    @dprintln(3, "@acc processing block element ", i)
    expr = ast.args[i]
    if is_function(expr)
      @dprintln(3, "@acc processing function within block = ", expr)
      ast.args[i] = convert_function(per_site_opt_set, opt_set, macros, expr)
      @dprintln(3, "@acc done processing function within block = ", ast.args[i])
    elseif is_block(expr)
      @dprintln(3, "@acc processing block within block = ", expr)
      ast.args[i] = convert_block(per_site_opt_set, opt_set, macros, expr)
      @dprintln(3, "@acc done processing block within block = ", ast.args[i])
    elseif is_macro(expr)
      @dprintln(3, "@acc processing macro within block = ", expr)
      ast.args[i] = convert_macro(per_site_opt_set, opt_set, macros, expr)
      @dprintln(3, "@acc done processing macro within block = ", ast.args[i])
    else
      @dprintln(3, "Don't know how to process block element[", i, "] of type ", typeof(expr))
      @dprintln(3, "Value = ", expr)
    end
  end
  @dprintln(3, "@acc processing block done = ", ast)
  return ast
end

function convert_macro(per_site_opt_set, opt_set, macros, ast)
  if is_function(ast.args[end])
    macros = vcat(macros, ast.args[1:end-1])
    @dprintln(3, "@acc converting macro function = ", ast, " macros = ", macros)
    convert_function(per_site_opt_set, opt_set, macros, ast.args[end])
  else
    return ast
  end
end

"""
Statically evaluate per-site optimization passes setting, and return the result.
"""
function evalPerSiteOptSet(per_site_opt_set)
  opt_set = Core.eval(current_module(), per_site_opt_set)
  for s in opt_set
    @assert isa(s, OptPass) ("Expect OptPass but got " * dump(s))
  end
  return opt_set
end

"""
The @acc macro comes in two forms:
1) @acc expression
3) @acc function ... end
In the first form, the set of optimization passes to apply come from the default set of optimization passes as specified with the funciton setOptPasses.  The @acc macro replaces each call in the expression with a call to a trampolines that determines the types of the call and if that combination of function and signature has not previously been optimized then it calls the set of optimization passes to optimize it.  Then, the trampoline calls the optimized function.
The second form is similar, and instead annotating callsite, the @acc macro can be used in front of a function's declaration. Used this way, it will replace the body of the function with the trampoline itself. The programmer can use @acc either at function callsite, or at function delcaration, but not both.
This macro may optionally take an OptPass array, right after @acc and followed by an expression or function.  In this case, the specified set of optPasses are used just for optimizing the following expression. When used with the second form (in front of a function), the value of this OptPass array will be statically evaluated at the macro expansion stage.
"""
macro acc(ast1, ast2...)
  if isempty(ast2)
    # default to assume global optPasses setting
    per_site_opt_set = nothing
    ast = ast1
  else
    per_site_opt_set = ast1
    ast = ast2[1]
  end
  # unlike convert_expr, we must eval per_site_opt_set statically
  opt_set = per_site_opt_set == nothing ? optPasses : evalPerSiteOptSet(per_site_opt_set)
  if !isa(opt_set, Array) || length(opt_set) == 0
    # skip translation since opt_set is empty
  elseif is_function(ast)
    @dprintln(3, "@acc at function level = ", ast)
    # used at function definition
    ast = convert_function(per_site_opt_set, opt_set, Any[], ast)
  elseif is_block(ast)
    @dprintln(3, "@acc at block level = ", ast)
    ast = convert_block(per_site_opt_set, opt_set, Any[], ast)
  elseif is_macro(ast)
    @dprintln(3, "@acc at macro level = ", ast)
    ast = convert_macro(per_site_opt_set, opt_set, Any[], ast)
  else
    # used at call site
    @dprintln(3, "@acc at call-site = ", ast)
    ast = convert_expr(per_site_opt_set, ast)
  end
  @dprintln(4, "@acc done = ", ast)
  esc(ast)
end

"""
Find the optimizing target function (after @acc macro) for a wrapper function in the given module.
Return the input function if not found. Always return as a GlobalRef.
"""
function findTargetFunc(mod::Module, name::Symbol)
   func = GlobalRef(mod, name)
   get(gOptFrameworkDict, func, func) :: GlobalRef
end

"""
Find the original (before @acc macro) function for a wrapper function in the given module.
Return the input function if not found. Always return as a GlobalRef.
"""
function findOriginalFunc(mod::Module, name::Symbol)
   func = GlobalRef(mod, name)
   orig = get(gOptFrameworkDict, func, func)
   # we lookup twice because the redirection goes like this: wrapper -> macro -> ast
   get(gOptFrameworkDict, orig, orig) :: GlobalRef
end

function recursive_noacc(ast::Symbol)
   if haskey(gOptFrameworkDict, GlobalRef(current_module(), ast))
     findOriginalFunc(current_module(), ast)
   else
     ast
   end
end

function recursive_noacc(ast::Expr)
    start = 1
    if ast.head === :(=)
      start = 2
    elseif ast.head === :call && isa(ast.args[1], Symbol)
      ast.args[1] = recursive_noacc(ast.args[1])
      start = 2
    end
    for i = start:length(ast.args)
      ast.args[i] = recursive_noacc(ast.args[i])
    end
    return ast
end

function recursive_noacc(ast::ANY)
    ast
end

"""
The macro @noacc can be used at call site to specifically run the non-accelerated copy of an accelerated function. It has no effect and gives a warning when the given function is not found to have been accelerated. We do not support nested @acc or @noacc.
"""
macro noacc(ast)
  ast = recursive_noacc(ast)
  esc(ast)
end

end   # end of module


text:
module MITgcmutils

include("read/reads.jl")
include("offlineAnalysis/methods.jl")

# package code goes here

end # module

ext: jl


text:
@generator GECSimpleGen begin
    start() = [a() for i in 1:5]
	a() = choose(Int, 0, 3)
end

@generator GECTypeBoundGen begin
	start() = (a(), b(), c(), d(), e(), f(), g())
	a() = choose(Int, 4, 9)
	b() = choose(Int, 1)
	c() = choose(Bool)
	d() = choose(Float64, 3.7)
	e() = choose(Float64, 4.2, 9.7)
	f() = choose(Int, typemin(Int), 10.0)
	g() = choose(Int, typemin(Int), typemax(Int))
end

@testset "GE choice model" begin

	@testset "constructors" begin

		cm1 = GEChoiceModel(4)
	    @test typeof(cm1) == GEChoiceModel

		cm2 = GEChoiceModel([1.0, 0.0, 7.0])
	    @test typeof(cm2) == GEChoiceModel

		@test_throws ErrorException cm3 = GEChoiceModel(0)

	end

	@testset "set/get parameters and ranges" begin

		cm1 = GEChoiceModel(4)
		params = getparams(cm1)
	    @test typeof(params) <: Vector{Float64}
		@test params == [0.0, 0.0, 0.0, 0.0]

		ranges = paramranges(cm1)
	    @test typeof(ranges) <: Vector{Tuple{Float64,Float64}}
		@test ranges == fill((Float64(0),Float64(typemax(UInt32))),4)

		setparams!(cm1, [8.0, 9.0, 5.0, 5.0])
		@test getparams(cm1) == [8.0, 9.0, 5.0, 5.0]

		@test_throws ErrorException setparams!(cm1, [8.0, 9.0, 5.0])
		@test_throws ErrorException setparams!(cm1, [8.0, 9.0, 5.0, 5.0, 2.0])
		@test_throws ErrorException setparams!(cm1, [8.0, -1.0, 5.0])
		setparams!(cm1, [8.0, 0.0, 5.0, 5.0])

		setparams!(cm1, [1.5, 7.6, 2.7, 5.2])
		@test getparams(cm1) == [2.0, 8.0, 3.0, 5.0]

		cm2 = GEChoiceModel([1.0, 0.0, 7.0])
		@test getparams(cm2) == [1.0, 0.0, 7.0]
		@test paramranges(cm2) == fill((Float64(0),Float64(typemax(UInt32))),3)
		setparams!(cm2, [2.0, 19.0, 27.0])
		@test getparams(cm2) == [2.0, 19.0, 27.0]

	end

	@testset "sampling" begin

		@testset "samples from genome" begin
			gn = GECSimpleGen()
			setchoicemodel!(gn, GEChoiceModel([1.0, 3.0, 2.0, 0.0, 2.0, 3.0]))
			@test choose(gn) == [1, 3, 2, 0, 2]
		end

		@testset "samples modulo from genome" begin
			gn = GECSimpleGen()
			setchoicemodel!(gn, GEChoiceModel([11.0, 14.0, 20.0, 21.0, 89.0, 39.0]))
			@test choose(gn) == [3, 2, 0, 1, 1]
		end

		@testset "wraps when sampling genome" begin
			gn = GECSimpleGen()
			setchoicemodel!(gn, GEChoiceModel([11.0, 14.0, 20.0]))
			@test choose(gn) == [3, 2, 0, 3, 2]
		end

		@testset "samples according to types and bounded intervals" begin
			gn = GECTypeBoundGen()
			setchoicemodel!(gn, GEChoiceModel([11.0, 14.0, 20.0, 3.0, 9.0, 2.0, 39.0]))
			x = choose(gn)
			@test typeof(x[1]) <: Int
			@test x[1] == 9
			@test typeof(x[2]) <: Int
			@test x[2] == 15
			@test typeof(x[3]) <: Bool
			@test x[3] == false
			@test typeof(x[4]) <: Float64
			@test isapprox(x[4], 6.7)
			@test typeof(x[5]) <: Float64
			@test isapprox(x[5], 7.7)
			@test typeof(x[6]) <: Int
			@test x[6] == 8
			@test typeof(x[7]) <: Int
			@test x[7] == 39
		end

	end

end


text:
include("../src/JOH.jl")
include("../MDMKP/MDMKP.jl")
include("SSIT.jl")

using CPLEX
const Optimizer = CPLEX

function extract_info(d)
    d1 = split(d, "(Number[")[2]
    d2 = split(d1, "], Number[")
    first = d2[1]
    d3 = split(d2[2], "], \"")
    second = d3[1]
    d4 = split(d3[2], "\", ")
    third = d4[1]
    d5 = split(d4[2], ", ")
    fourth = d5[1]
    fifth = split(d5[2], ")")[1]
    [first, second, third, fourth, fifth]
end

function parse_info(info)
    numbers(data, type) = [parse(type, n) for n in split(data, ", ")]

    methods = SE.make_SSIT_methods()
    get_method(name) = filter(m->m.name == name, methods)
    get_method(info[3])[1]
end

function load_ssit_method(d)
    parse_info(extract_info(d))
end

struct Error
    problem::JOH.Problem
    method
    model
    solution::AbstractArray{Bool}
    phase::Int
end

function load_folder_info(filename)
    s = split(filename, "(")[2]
    s = split(s, ",")[1]
    id = parse(Int, s)
    problem = MDMKP.load_folder()[id]

    phase = parse(Int, split(filename, "_")[3])

    method = open(joinpath("./error_log/", filename, "method.dat")) do io
        load_ssit_method(read(io, String))
    end

    model = MDMKP.create_MIPS_model(problem, Optimizer.Optimizer)

    solution = open(joinpath("./error_log/", filename, "solution.dat")) do io
        read(io, String)
    end
    sol = solution[2:end-1]
    parser(x) = parse(Float64, x) == 1.0
    solution = parser.(split(sol, ","))

    Error(problem, method, model, solution, phase)
end

all_problems = MDMKP.load_folder()
errors = load_folder_info.(readdir("./error_log/"))

function test_error(e::Error; mod=1)
    model = e.model
	model[:x] = e.solution
	JOH.Matheur.set_threads!(model, e.method.num_threads)

	JOH.Matheur.set_tolerance!(model, e.method.tolerances[e.phase])
	time_limit = e.method.times[e.phase]/mod
	JOH.Matheur.set_time!(model, time_limit)
	elapsed_time = JOH.Matheur.silent_optimize!(model)
	ratio = elapsed_time/time_limit
	return (elapsed_time, ratio)
end

test_error(errors[2], mod=60)

# println(test_error.(errors))

max_stars_repo_name: Dibillilia/JuliaOptHeuristics

max_issues_repo_name: Dibillilia/JuliaOptHeuristics

max_forks_repo_name: Dibillilia/JuliaOptHeuristics


text:
############################################################################################
"""
$(TYPEDEF)

MCMC Diagnostics container.

# Fields
$(TYPEDFIELDS)
"""
struct MCMCDiagnostics{
    P,
    K<:MCMCKernelDiagnostics,
    G
} <: AbstractDiagnostics
    "Diagnostics used for all Baytes kernels"
    base::BaytesCore.BaseDiagnostics{P}
    "Kernel specific diagnostics."
    kernel::K
    "Boolean if diverged."
    divergence::Bool
    "Acceptance Rate of current step."
    accept::AcceptStatistic
    "Generated quantities specified for objective"
    generated::G
    function MCMCDiagnostics(
        base::BaytesCore.BaseDiagnostics{P},
        kerneldiagnostics::K,
        divergence::Bool,
        accept::AcceptStatistic,
        generated::G,
    ) where {P,K<:MCMCKernelDiagnostics,G}
        return new{P,K,G}(
            base, kerneldiagnostics, divergence, accept, generated
        )
    end
end

############################################################################################
function generate_showvalues(diagnostics::D) where {D<:MCMCDiagnostics}
    kernel = generate_showvalues(diagnostics.kernel)
    return function showvalues()
        return (:mcmc, "diagnostics"),
        (:iter, diagnostics.base.iter),
        (:logobjective, diagnostics.base.ℓobjective),
        (:Temperature, diagnostics.base.temperature),
        (:accepted, diagnostics.accept.accepted),
        (:acceptancerate, diagnostics.accept.rate),
        kernel()...
    end
end

############################################################################################
"""
$(SIGNATURES)
Print all divergences from a vector of MCMC Diagnostics.

# Examples
```julia
```

"""
function print_divergences(
    diagnosticsᵛ::AbstractVector{M}, phase::PhaseTune
) where {M<:MCMCDiagnostics}
    ## Assign phase names and ending indices
    phase_names = unique(phase.name)
    phase_sequence = cumsum(phase.window)
    phase_iterations = [phase.slices[iter] for iter in phase_sequence]
    push!(phase_iterations, max(size(diagnosticsᵛ, 1), phase_iterations[end] + 1))
    counter = zeros(Int64, length(phase_names))
    ## Loop through all chains and iterations to check for divergences
    for iter in eachindex(diagnosticsᵛ)
        if diagnosticsᵛ[iter].divergence
            _, phaseindex = findmax(iter .<= phase_iterations)
            counter[phaseindex] += 1
        end
    end
    ## Print results
    print("Divergences in: ")
    for iter in 1:(length(counter) - 1)
        print(Base.nameof(typeof(phase_names[iter])), ": ", counter[iter], ", ")
    end
    for iter in length(counter)
        println(Base.nameof(typeof(phase_names[iter])), ": ", counter[iter], ".")
    end
    ## Return divergences
    return phase_names, counter
end

############################################################################################
# Export
export MCMCDiagnostics, generate_showvalues


text:
using Test
using Logging
using OpenTelemetryAPI
using OpenTelemetrySDK
using OpenTelemetryProto
using OpenTelemetryExporterOtlpProtoGrpc

# exporter = OtlpProtoGrpcTraceExporter()

@testset "OtlpProtoGrpc Trace" begin
    # NonRecordingSpan
    with_span("foo") do
        req = convert(
            OpenTelemetryProto.OpentelemetryClients.ExportTraceServiceRequest,
            current_span(),
        )
    end

    # Span
    with_span("foo", Tracer(provider = TracerProvider())) do
        s = current_span()
        s["a"] = "a"
        s["b"] = true
        s["c"] = 1.0
        s["d"] = UInt8[0, 1, 2]
        s["e"] = -1
        push!(s, Event(; name = "example", attributes = StaticAttrs("e" => "eee")))
        push!(
            s,
            Link(
                SpanContext(;
                    trace_id = rand(TraceIdType),
                    span_id = rand(SpanIdType),
                    is_remote = false,
                ),
                StaticAttrs("f" => "fff"),
            ),
        )

        req = convert(
            OpenTelemetryProto.OpentelemetryClients.ExportTraceServiceRequest,
            current_span(),
        )
    end
end

@testset "OtlpProtoGrpc Metrics" begin
    p = MeterProvider()
    m = Meter("abc"; provider = p)
    c = Counter{Int}("fruit_counter", m)
    c(1; name = "lemon", color = "yellow")

    h = Histogram{Float64}("fake_hist", m)
    h(1; name = "lemon", color = "yellow")

    convert(OpenTelemetryProto.OpentelemetryClients.ExportMetricsServiceRequest, metrics(p))
end

@testset "OtlpProtoGrpc Logs" begin
    r = LogRecord(;
        timestamp = UInt(time() * 10^9),
        trace_id = INVALID_TRACE_ID,
        span_id = INVALID_SPAN_ID,
        trace_flags = TraceFlag(),
        severity_text = "INFO",
        severity_number = 1,
        name = "",
        body = "Hi",
        resource = Resource(),
        attributes = StaticAttrs(),
        instrumentation_info = InstrumentationInfo(),
    )
    convert(OpenTelemetryProto.OpentelemetryClients.ExportLogsServiceRequest, r)
end


text:
using JuliaSyntax.Tokenize
using JuliaSyntax.Tokenize.Lexers
using Test

const T = Tokenize.Tokens

tok(str, i = 1) = collect(tokenize(str))[i]

@testset "tokens" begin
    for s in ["a", IOBuffer("a")]
        l = tokenize(s)
        @test Lexers.readchar(l) == 'a'

        # @test l.current_pos == 0
        l_old = l
        @test l == l_old
        @test Lexers.eof(l)
        @test Lexers.readchar(l) == Lexers.EOF_CHAR

        # @test l.current_pos == 0
    end
end # testset

@testset "tokenize unicode" begin
    str = "𝘋 =2β"
    for s in [str, IOBuffer(str)]
        l = tokenize(s)
        kinds = [T.IDENTIFIER, T.WHITESPACE, T.OP,
                T.INTEGER, T.IDENTIFIER, T.ENDMARKER]
        token_strs = ["𝘋", " ", "=", "2", "β", ""]
        for (i, n) in enumerate(l)
            @test T.kind(n) == kinds[i]
            @test untokenize(n, str)  == token_strs[i]
        end
    end
end # testset

@testset "tokenize complex piece of code" begin

    str = """
    function foo!{T<:Bar}(x::{T}=12)
        @time (x+x, x+x);
    end
    try
        foo
    catch
        bar
    end
    @time x+x
    y[[1 2 3]]
    [1*2,2;3,4]
    "string"; 'c'
    (a&&b)||(a||b)
    # comment
    #= comment
    is done here =#
    2%5
    a'/b'
    a.'\\b.'
    `command`
    12_sin(12)
    {}
    '
    """

    # Generate the following with
    # ```
    # for t in Tokens.kind.(collect(tokenize(str)))
    #    print("T.", t, ",")
    # end
    # ```
    # and *check* it afterwards.

    kinds = [T.KEYWORD,T.WHITESPACE,T.IDENTIFIER,T.LBRACE,T.IDENTIFIER,
            T.OP,T.IDENTIFIER,T.RBRACE,T.LPAREN,T.IDENTIFIER,T.OP,
            T.LBRACE,T.IDENTIFIER,T.RBRACE,T.OP,T.INTEGER,T.RPAREN,

            T.NEWLINE_WS,T.AT_SIGN,T.IDENTIFIER,T.WHITESPACE,T.LPAREN,
            T.IDENTIFIER,T.OP,T.IDENTIFIER,T.COMMA,T.WHITESPACE,
            T.IDENTIFIER,T.OP,T.IDENTIFIER,T.RPAREN,T.SEMICOLON,

            T.NEWLINE_WS,T.KEYWORD,

            T.NEWLINE_WS,T.KEYWORD,
            T.NEWLINE_WS,T.IDENTIFIER,
            T.NEWLINE_WS,T.KEYWORD,
            T.NEWLINE_WS,T.IDENTIFIER,
            T.NEWLINE_WS,T.KEYWORD,

            T.NEWLINE_WS,T.AT_SIGN,T.IDENTIFIER,T.WHITESPACE,T.IDENTIFIER,
            T.OP,T.IDENTIFIER,

            T.NEWLINE_WS,T.IDENTIFIER,T.LSQUARE,T.LSQUARE,T.INTEGER,T.WHITESPACE,
            T.INTEGER,T.WHITESPACE,T.INTEGER,T.RSQUARE,T.RSQUARE,

            T.NEWLINE_WS,T.LSQUARE,T.INTEGER,T.OP,T.INTEGER,T.COMMA,T.INTEGER,
            T.SEMICOLON,T.INTEGER,T.COMMA,T.INTEGER,T.RSQUARE,

            T.NEWLINE_WS,T.DQUOTE,T.STRING,T.DQUOTE,T.SEMICOLON,T.WHITESPACE,T.CHAR,

            T.NEWLINE_WS,T.LPAREN,T.IDENTIFIER,T.OP,T.IDENTIFIER,T.RPAREN,T.OP,
            T.LPAREN,T.IDENTIFIER,T.OP,T.IDENTIFIER,T.RPAREN,

            T.NEWLINE_WS,T.COMMENT,

            T.NEWLINE_WS,T.COMMENT,

            T.NEWLINE_WS,T.INTEGER,T.OP,T.INTEGER,

            T.NEWLINE_WS,T.IDENTIFIER,T.OP,T.OP,T.IDENTIFIER,T.OP,

            T.NEWLINE_WS,T.IDENTIFIER,T.OP,T.OP,T.OP,T.IDENTIFIER,T.OP,T.OP,

            T.NEWLINE_WS,T.BACKTICK,T.CMD,T.BACKTICK,

            T.NEWLINE_WS,T.INTEGER,T.IDENTIFIER,T.LPAREN,T.INTEGER,T.RPAREN,

            T.NEWLINE_WS,T.LBRACE,T.RBRACE,

            T.NEWLINE_WS,T.ERROR,T.ENDMARKER]

    for (i, n) in enumerate(tokenize(str))
        @test Tokens.kind(n) == kinds[i]
    end

    @testset "roundtrippability" begin
        @test join(untokenize.(collect(tokenize(str)), str)) == str
    end

    @test all((t.endbyte - t.startbyte + 1)==sizeof(untokenize(t, str)) for t in tokenize(str))
end # testset

@testset "issue 5, '..'" begin
    @test Tokens.kind.(collect(tokenize("1.23..3.21"))) == [T.FLOAT,T.OP,T.FLOAT,T.ENDMARKER]
end

@testset "issue 17, >>" begin
    str = ">> "
    @test untokenize(tok(str), str)==">>"
end

@testset "test added operators" begin
    @test tok("1+=2",  2).kind == T.PLUS_EQ
    @test tok("1-=2",  2).kind == T.MINUS_EQ
    @test tok("1:=2",  2).kind == T.COLON_EQ
    @test tok("1*=2",  2).kind == T.STAR_EQ
    @test tok("1^=2",  2).kind == T.CIRCUMFLEX_EQ
    @test tok("1÷=2",  2).kind == T.DIVISION_EQ
    @test tok("1\\=2", 2).kind == T.BACKSLASH_EQ
    @test tok("1\$=2", 2).kind == T.EX_OR_EQ
    @test tok("1-->2", 2).kind == T.RIGHT_ARROW
    @test tok("1<--2", 2).kind == T.LEFT_ARROW
    @test tok("1<-->2", 2).kind == T.DOUBLE_ARROW
    @test tok("1>:2",  2).kind == T.ISSUPERTYPE
end

@testset "infix" begin
    @test tok("1 in 2",  3).kind == T.IN
    @test tok("1 in[1]", 3).kind == T.IN

    @test tok("1 isa 2",  3).kind == T.ISA
    @test tok("1 isa[2]", 3).kind == T.ISA
end

@testset "tokenizing true/false literals" begin
    @test tok("somtext true", 3).kind == T.TRUE
    @test tok("somtext false", 3).kind == T.FALSE
    @test tok("somtext tr", 3).kind == T.IDENTIFIER
    @test tok("somtext falsething", 3).kind == T.IDENTIFIER
end

function test_roundtrip(str, kind, val)
    T = tok(str)
    @test T.kind == kind
    @test untokenize(T, str) == val
end

@testset "tokenizing juxtaposed numbers and dotted operators/identifiers" begin
    test_roundtrip("1234 .+1",     Tokens.INTEGER, "1234")
    test_roundtrip("1234.0+1",     Tokens.FLOAT,   "1234.0")
    test_roundtrip("1234.0 .+1",   Tokens.FLOAT,   "1234.0")
    test_roundtrip("1234.f(a)",    Tokens.FLOAT,   "1234.")
    test_roundtrip("1234 .f(a)",   Tokens.INTEGER, "1234")
    test_roundtrip("1234.0.f(a)",  Tokens.INVALID_NUMERIC_CONSTANT,   "1234.0.")
    test_roundtrip("1234.0 .f(a)", Tokens.FLOAT,   "1234.0")
end

@testset "lexing anon functions '->' " begin
    @test tok("a->b", 2).kind==Tokens.ANON_FUNC
end

@testset "comments" begin
    toks = collect(tokenize("""
       #
       \"\"\"
       f
       \"\"\"
       1
       """))

    kinds = [T.COMMENT, T.NEWLINE_WS,
             T.TRIPLE_DQUOTE, T.STRING, T.STRING, T.TRIPLE_DQUOTE, T.NEWLINE_WS,
             T.INTEGER, T.NEWLINE_WS,
             T.ENDMARKER]
    @test T.kind.(toks) == kinds
end

@testset "primes" begin
    str = """
    ImageMagick.save(fn, reinterpret(ARGB32, [0xf0884422]''))
    D = ImageMagick.load(fn)
    """
    tokens = collect(tokenize(str))
    @test string(untokenize(tokens[16], str))==string(untokenize(tokens[17], str))=="'"

    test_roundtrip("'a'",  Tokens.CHAR, "'a'")
    test_roundtrip("''",   Tokens.CHAR, "''")
    test_roundtrip("'''",  Tokens.CHAR, "'''")
    test_roundtrip("''''", Tokens.CHAR, "'''")

    @test tok("''''", 1).kind == Tokens.CHAR
    @test tok("''''", 2).kind == Tokens.PRIME
    @test tok("()'", 3).kind == Tokens.PRIME
    @test tok("{}'", 3).kind == Tokens.PRIME
    @test tok("[]'", 3).kind == Tokens.PRIME
    @test tok("outer'", 2).kind == Tokens.PRIME
    @test tok("mutable'", 2).kind == Tokens.PRIME
    @test tok("as'", 2).kind == Tokens.PRIME
    @test tok("isa'", 2).kind == Tokens.PRIME
end

@testset "keywords" begin
      for kw in    ["baremodule",
                    "begin",
                    "break",
                    "catch",
                    "const",
                    "continue",
                    "do",
                    "else",
                    "elseif",
                    "end",
                    "export",
                    "finally",
                    "for",
                    "function",
                    "global",
                    "if",
                    "import",
                    "let",
                    "local",
                    "macro",
                    "module",
                    "quote",
                    "return",
                    "struct",
                    "try",
                    "using",
                    "while",

                    "abstract",
                    "as",
                    "doc",
                    "mutable",
                    "outer",
                    "primitive",
                    "type",
                    "var"]

        @test T.kind(tok(kw)) == T.KEYWORD
    end
end

@testset "issue in PR #45" begin
    @test length(collect(tokenize("x)"))) == 3
end

@testset "errors" begin
    @test tok("#=   #=   =#",           1).kind == T.EOF_MULTICOMMENT
    @test tok("'dsadsa",                1).kind == T.EOF_CHAR
    @test tok("aa **",                  3).kind == T.INVALID_OPERATOR
end

@testset "xor_eq" begin
    @test tok("1 ⊻= 2", 3).kind==T.XOR_EQ
end

@testset "lex binary" begin
    @test tok("0b0101").kind==T.BIN_INT
end

@testset "show" begin
    io = IOBuffer()
    show(io, collect(tokenize("\"abc\nd\"ef"))[2])
    @test String(take!(io)) == "1-5        STRING         "
end

~(tok::T.Token, t::Tuple) = tok.kind == t[1] && untokenize(tok, t[3]) == t[2]

@testset "raw strings" begin
    str = raw""" str"x $ \ y" """
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.WHITESPACE , " "        , str)
    @test ts[2] ~ (T.IDENTIFIER , "str"      , str)
    @test ts[3] ~ (T.DQUOTE     , "\""       , str)
    @test ts[4] ~ (T.STRING     , "x \$ \\ y", str)
    @test ts[5] ~ (T.DQUOTE     , "\""       , str)
    @test ts[6] ~ (T.WHITESPACE , " "        , str)
    @test ts[7] ~ (T.ENDMARKER  , ""         , str)

    str = raw"""`x $ \ y`"""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.BACKTICK  , "`"         , str)
    @test ts[2] ~ (T.CMD       , "x \$ \\ y" , str)
    @test ts[3] ~ (T.BACKTICK  , "`"         , str)
    @test ts[4] ~ (T.ENDMARKER , ""          , str)

    # str"\\"
    str = "str\"\\\\\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.IDENTIFIER , "str"  , str)
    @test ts[2] ~ (T.DQUOTE     , "\""   , str)
    @test ts[3] ~ (T.STRING     , "\\\\" , str)
    @test ts[4] ~ (T.DQUOTE     , "\""   , str)
    @test ts[5] ~ (T.ENDMARKER  , ""     , str)

    # str"\\\""
    str = "str\"\\\\\\\"\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.IDENTIFIER , "str"      , str)
    @test ts[2] ~ (T.DQUOTE     , "\""       , str)
    @test ts[3] ~ (T.STRING     , "\\\\\\\"" , str)
    @test ts[4] ~ (T.DQUOTE     , "\""       , str)
    @test ts[5] ~ (T.ENDMARKER  , ""         , str)

    # Contextual keywords and operators allowed as raw string prefixes
    str = raw""" var"x $ \ y" """
    ts = collect(tokenize(str))
    @test ts[2] ~ (T.VAR        , "var", str)
    @test ts[4] ~ (T.STRING     , "x \$ \\ y", str)

    str = raw""" outer"x $ \ y" """
    ts = collect(tokenize(str))
    @test ts[2] ~ (T.OUTER      , "outer", str)
    @test ts[4] ~ (T.STRING     , "x \$ \\ y", str)

    str = raw""" isa"x $ \ y" """
    ts = collect(tokenize(str))
    @test ts[2] ~ (T.ISA        , "isa", str)
    @test ts[4] ~ (T.STRING     , "x \$ \\ y", str)
end

@testset "string escaped newline whitespace" begin
    str = "\"x\\\n \ty\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.DQUOTE, "\"", str)
    @test ts[2] ~ (T.STRING, "x", str)
    @test ts[3] ~ (T.WHITESPACE, "\\\n \t", str)
    @test ts[4] ~ (T.STRING, "y", str)
    @test ts[5] ~ (T.DQUOTE, "\"", str)

    # No newline escape for raw strings
    str = "r\"x\\\ny\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.IDENTIFIER , "r", str)
    @test ts[2] ~ (T.DQUOTE, "\"", str)
    @test ts[3] ~ (T.STRING, "x\\\ny", str)
    @test ts[4] ~ (T.DQUOTE , "\"", str)
end

@testset "triple quoted string line splitting" begin
    str = "\"\"\"\nx\r\ny\rz\n\r\"\"\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.TRIPLE_DQUOTE , "\"\"\"", str)
    @test ts[2] ~ (T.STRING        , "\n", str)
    @test ts[3] ~ (T.STRING        , "x\r\n", str)
    @test ts[4] ~ (T.STRING        , "y\r", str)
    @test ts[5] ~ (T.STRING        , "z\n", str)
    @test ts[6] ~ (T.STRING        , "\r", str)
    @test ts[7] ~ (T.TRIPLE_DQUOTE , "\"\"\"", str)

    # Also for raw strings
    str = "r\"\"\"\nx\ny\"\"\""
    ts = collect(tokenize(str))
    @test ts[1] ~ (T.IDENTIFIER    , "r", str)
    @test ts[2] ~ (T.TRIPLE_DQUOTE , "\"\"\"", str)
    @test ts[3] ~ (T.STRING        , "\n", str)
    @test ts[4] ~ (T.STRING        , "x\n", str)
    @test ts[5] ~ (T.STRING        , "y", str)
    @test ts[6] ~ (T.TRIPLE_DQUOTE , "\"\"\"", str)
end

@testset "interpolation" begin
    @testset "basic" begin
        str = "\"\$x \$y\""
    ts = collect(tokenize(str))
        @test ts[1]  ~ (T.DQUOTE     , "\"", str)
        @test ts[2]  ~ (T.EX_OR      , "\$", str)
        @test ts[3]  ~ (T.IDENTIFIER , "x" , str)
        @test ts[4]  ~ (T.STRING     , " " , str)
        @test ts[5]  ~ (T.EX_OR      , "\$", str)
        @test ts[6]  ~ (T.IDENTIFIER , "y" , str)
        @test ts[7]  ~ (T.DQUOTE     , "\"", str)
        @test ts[8]  ~ (T.ENDMARKER  , ""  , str)
    end

    @testset "nested" begin
        str = """"str: \$(g("str: \$(h("str"))"))" """
        ts = collect(tokenize(str))
        @test length(ts) == 23
        @test ts[1]  ~ (T.DQUOTE    , "\""   , str)
        @test ts[2]  ~ (T.STRING    , "str: ", str)
        @test ts[3]  ~ (T.EX_OR     , "\$"   , str)
        @test ts[4]  ~ (T.LPAREN    , "("    , str)
        @test ts[5]  ~ (T.IDENTIFIER, "g"    , str)
        @test ts[6]  ~ (T.LPAREN    , "("    , str)
        @test ts[7]  ~ (T.DQUOTE    , "\""   , str)
        @test ts[8]  ~ (T.STRING    , "str: ", str)
        @test ts[9]  ~ (T.EX_OR     , "\$"   , str)
        @test ts[10] ~ (T.LPAREN    , "("    , str)
        @test ts[11] ~ (T.IDENTIFIER, "h"    , str)
        @test ts[12] ~ (T.LPAREN    , "("    , str)
        @test ts[13] ~ (T.DQUOTE    , "\""   , str)
        @test ts[14] ~ (T.STRING    , "str"  , str)
        @test ts[15] ~ (T.DQUOTE    , "\""   , str)
        @test ts[16] ~ (T.RPAREN    , ")"    , str)
        @test ts[17] ~ (T.RPAREN    , ")"    , str)
        @test ts[18] ~ (T.DQUOTE    , "\""   , str)
        @test ts[19] ~ (T.RPAREN    , ")"    , str)
        @test ts[20] ~ (T.RPAREN    , ")"    , str)
        @test ts[21] ~ (T.DQUOTE    , "\""   , str)
        @test ts[22] ~ (T.WHITESPACE, " "    , str)
        @test ts[23] ~ (T.ENDMARKER , ""     , str)
    end

    @testset "duplicate \$" begin
        str = "\"\$\$\""
    ts = collect(tokenize(str))
        @test ts[1]  ~ (T.DQUOTE     , "\"", str)
        @test ts[2]  ~ (T.EX_OR      , "\$", str)
        @test ts[3]  ~ (T.EX_OR      , "\$", str)
        @test ts[4]  ~ (T.DQUOTE     , "\"", str)
        @test ts[5]  ~ (T.ENDMARKER  , ""  , str)
    end

    @testset "Unmatched parens" begin
        # issue 73: https://github.com/JuliaLang/Tokenize.jl/issues/73
        str = "\"\$(fdsf\""
    ts = collect(tokenize(str))
        @test ts[1] ~ (T.DQUOTE     , "\""   , str)
        @test ts[2] ~ (T.EX_OR      , "\$"   , str)
        @test ts[3] ~ (T.LPAREN     , "("    , str)
        @test ts[4] ~ (T.IDENTIFIER , "fdsf" , str)
        @test ts[5] ~ (T.DQUOTE     , "\""   , str)
        @test ts[6] ~ (T.ENDMARKER  , ""     , str)
    end

    @testset "Unicode" begin
        # issue 178: https://github.com/JuliaLang/Tokenize.jl/issues/178
        str = """ "\$uₕx \$(uₕx - ux)" """
    ts = collect(tokenize(str))
        @test ts[ 1] ~ (T.WHITESPACE , " "   , str)
        @test ts[ 2] ~ (T.DQUOTE     , "\""  , str)
        @test ts[ 3] ~ (T.EX_OR      , "\$"  , str)
        @test ts[ 4] ~ (T.IDENTIFIER , "uₕx" , str)
        @test ts[ 5] ~ (T.STRING     , " "   , str)
        @test ts[ 6] ~ (T.EX_OR      , "\$"  , str)
        @test ts[ 7] ~ (T.LPAREN     , "("   , str)
        @test ts[ 8] ~ (T.IDENTIFIER , "uₕx" , str)
        @test ts[ 9] ~ (T.WHITESPACE , " "   , str)
        @test ts[10] ~ (T.MINUS      , "-"   , str)
        @test ts[11] ~ (T.WHITESPACE , " "   , str)
        @test ts[12] ~ (T.IDENTIFIER , "ux"  , str)
        @test ts[13] ~ (T.RPAREN     , ")"   , str)
        @test ts[14] ~ (T.DQUOTE     , "\""  , str)
        @test ts[15] ~ (T.WHITESPACE , " "   , str)
        @test ts[16] ~ (T.ENDMARKER  , ""    , str)
    end

    @testset "var\"...\" disabled in interpolations" begin
        str = """ "\$var"x" " """
    ts = collect(tokenize(str))
        @test ts[ 1] ~ (T.WHITESPACE , " "   , str)
        @test ts[ 2] ~ (T.DQUOTE     , "\""  , str)
        @test ts[ 3] ~ (T.EX_OR      , "\$"  , str)
        @test ts[ 4] ~ (T.VAR        , "var" , str)
        @test ts[ 5] ~ (T.DQUOTE     , "\""  , str)
        @test ts[ 6] ~ (T.IDENTIFIER , "x"   , str)
        @test ts[ 7] ~ (T.DQUOTE     , "\""  , str)
        @test ts[ 8] ~ (T.STRING     , " "   , str)
        @test ts[ 9] ~ (T.DQUOTE     , "\""  , str)
        @test ts[10] ~ (T.WHITESPACE , " "   , str)
        @test ts[11] ~ (T.ENDMARKER  , ""    , str)
    end

    @testset "invalid chars after identifier" begin
        str = """ "\$x෴" """
    ts = collect(tokenize(str))
        @test ts[4] ~ (T.IDENTIFIER , "x" , str)
        @test ts[5] ~ (T.INVALID_INTERPOLATION_TERMINATOR      , ""  , str)
        @test ts[6] ~ (T.STRING     , "෴" , str)
        @test Tokens.iserror(ts[5].kind)
        @test ts[5].kind == Tokens.INVALID_INTERPOLATION_TERMINATOR
    end
end

@testset "inferred" begin
    l = tokenize("abc")
    @inferred Tokenize.Lexers.next_token(l)
end

@testset "modifying function names (!) followed by operator" begin
    @test tok("a!=b",  2).kind == Tokens.NOT_EQ
    @test tok("a!!=b", 2).kind == Tokens.NOT_EQ
    @test tok("!=b",   1).kind == Tokens.NOT_EQ
end

@testset "lex integers" begin
    @test tok("1234").kind            == T.INTEGER
    @test tok("12_34").kind           == T.INTEGER
    @test tok("_1234").kind           == T.IDENTIFIER
    @test tok("1234_").kind           == T.INTEGER
    @test tok("1234_", 2).kind        == T.IDENTIFIER
    @test tok("1234x").kind           == T.INTEGER
    @test tok("1234x", 2).kind        == T.IDENTIFIER
end

@testset "floats with trailing `.` " begin
    @test tok("1.0").kind == Tokens.FLOAT
    @test tok("1.a").kind == Tokens.FLOAT
    @test tok("1.(").kind == Tokens.FLOAT
    @test tok("1.[").kind == Tokens.FLOAT
    @test tok("1.{").kind == Tokens.FLOAT
    @test tok("1.)").kind == Tokens.FLOAT
    @test tok("1.]").kind == Tokens.FLOAT
    @test tok("1.{").kind == Tokens.FLOAT
    @test tok("1.,").kind == Tokens.FLOAT
    @test tok("1.;").kind == Tokens.FLOAT
    @test tok("1.@").kind == Tokens.FLOAT
    @test tok("1.").kind == Tokens.FLOAT
    @test tok("1.\"text\" ").kind == Tokens.FLOAT

    @test tok("1..").kind  == Tokens.INTEGER
    @test T.kind.(collect(tokenize("1f0./1"))) == [T.FLOAT, T.OP, T.INTEGER, T.ENDMARKER]
end

@testset "lex octal" begin
    @test tok("0o0167").kind == T.OCT_INT
end

@testset "lex float/bin/hex/oct w underscores" begin
    @test tok("1_1.11").kind           == T.FLOAT
    @test tok("11.1_1").kind           == T.FLOAT
    @test tok("1_1.1_1").kind           == T.FLOAT
    @test tok("_1.1_1", 1).kind           == T.IDENTIFIER
    @test tok("_1.1_1", 2).kind           == T.FLOAT
    @test tok("0x0167_032").kind           == T.HEX_INT
    @test tok("0b0101001_0100_0101").kind  == T.BIN_INT
    @test tok("0o01054001_0100_0101").kind == T.OCT_INT
    @test T.kind.(collect(tokenize("1.2."))) == [T.ERROR, T.ENDMARKER]
    @test tok("1__2").kind == T.INTEGER
    @test tok("1.2_3").kind == T.FLOAT
    @test tok("1.2_3", 2).kind == T.ENDMARKER
    @test T.kind.(collect(tokenize("3e2_2"))) == [T.FLOAT, T.IDENTIFIER, T.ENDMARKER]
    @test T.kind.(collect(tokenize("1__2"))) == [T.INTEGER, T.IDENTIFIER, T.ENDMARKER]
    @test T.kind.(collect(tokenize("0x2_0_2"))) == [T.HEX_INT, T.ENDMARKER]
    @test T.kind.(collect(tokenize("0x2__2"))) == [T.HEX_INT, T.IDENTIFIER, T.ENDMARKER]
    @test T.kind.(collect(tokenize("3_2.5_2"))) == [T.FLOAT, T.ENDMARKER]
    @test T.kind.(collect(tokenize("3.2e2.2"))) == [T.ERROR, T.INTEGER, T.ENDMARKER]
    @test T.kind.(collect(tokenize("3e2.2"))) == [T.ERROR, T.INTEGER, T.ENDMARKER]
    @test T.kind.(collect(tokenize("0b101__101"))) == [T.BIN_INT, T.IDENTIFIER, T.ENDMARKER]
end

@testset "floating points" begin
    @test tok("1.0e0").kind  == Tokens.FLOAT
    @test tok("1.0e-0").kind == Tokens.FLOAT
    @test tok("1.0E0").kind  == Tokens.FLOAT
    @test tok("1.0E-0").kind == Tokens.FLOAT
    @test tok("1.0f0").kind  == Tokens.FLOAT
    @test tok("1.0f-0").kind == Tokens.FLOAT

    @test tok("0e0").kind    == Tokens.FLOAT
    @test tok("0e+0").kind   == Tokens.FLOAT
    @test tok("0E0").kind    == Tokens.FLOAT
    @test tok("201E+0").kind == Tokens.FLOAT
    @test tok("2f+0").kind   == Tokens.FLOAT
    @test tok("2048f0").kind == Tokens.FLOAT
    @test tok("1.:0").kind == Tokens.FLOAT
    @test tok("0x00p2").kind == Tokens.FLOAT
    @test tok("0x00P2").kind == Tokens.FLOAT
    @test tok("0x0.00p23").kind == Tokens.FLOAT
    @test tok("0x0.0ap23").kind == Tokens.FLOAT
    @test tok("0x0.0_0p2").kind == Tokens.FLOAT
    @test tok("0x0_0_0.0_0p2").kind == Tokens.FLOAT
    @test tok("0x0p+2").kind == Tokens.FLOAT
    @test tok("0x0p-2").kind == Tokens.FLOAT

    # Floating point with \minus rather than -
    @test tok("1.0e−0").kind == Tokens.FLOAT
    @test tok("1.0f−0").kind == Tokens.FLOAT
    @test tok("0x0p−2").kind == Tokens.FLOAT
end

@testset "1e1" begin
    @test tok("1e", 1).kind == Tokens.INTEGER
    @test tok("1e", 2).kind == Tokens.IDENTIFIER
end

@testset "jl06types" begin
    @test tok("mutable").kind   == Tokens.MUTABLE
    @test tok("primitive").kind == Tokens.PRIMITIVE
    @test tok("struct").kind    == Tokens.STRUCT
    @test tok("where").kind     == Tokens.WHERE
    @test tok("mutable struct s{T} where T",  1).kind == Tokens.MUTABLE
    @test tok("mutable struct s{T} where T",  3).kind == Tokens.STRUCT
    @test tok("mutable struct s{T} where T", 10).kind == Tokens.WHERE
end

@testset "CMDs" begin
    @test tok("`cmd`",1).kind == T.BACKTICK
    @test tok("`cmd`",2).kind == T.CMD
    @test tok("`cmd`",3).kind == T.BACKTICK
    @test tok("`cmd`",4).kind == T.ENDMARKER
    @test tok("```cmd```", 1).kind == T.TRIPLE_BACKTICK
    @test tok("```cmd```", 2).kind == T.CMD
    @test tok("```cmd```", 3).kind == T.TRIPLE_BACKTICK
    @test tok("```cmd```", 4).kind == T.ENDMARKER
    @test tok("```cmd````cmd`", 1).kind == T.TRIPLE_BACKTICK
    @test tok("```cmd````cmd`", 2).kind == T.CMD
    @test tok("```cmd````cmd`", 3).kind == T.TRIPLE_BACKTICK
    @test tok("```cmd````cmd`", 4).kind == T.BACKTICK
    @test tok("```cmd````cmd`", 5).kind == T.CMD
    @test tok("```cmd````cmd`", 6).kind == T.BACKTICK
    @test tok("```cmd````cmd`", 7).kind == T.ENDMARKER
end

@testset "where" begin
    @test tok("a where b", 3).kind == T.WHERE
end

@testset "IO position" begin
    io = IOBuffer("#1+1")
    skip(io, 1)
    @test length(collect(tokenize(io))) == 4
end

@testset "hex/bin/octal errors" begin
    @test tok("0x").kind == T.INVALID_NUMERIC_CONSTANT
    @test tok("0b").kind == T.INVALID_NUMERIC_CONSTANT
    @test tok("0o").kind == T.INVALID_NUMERIC_CONSTANT
    @test tok("0x 2", 1).kind == T.INVALID_NUMERIC_CONSTANT
    @test tok("0x.1p1").kind == T.FLOAT
end

@testset "dotted and suffixed operators" begin
ops = collect(values(Tokenize.Tokens.UNICODE_OPS_REVERSE))

for op in ops
    op in (:isa, :in, :where, Symbol('\''), :?, :(:)) && continue
    strs = [
        1 => [ # unary
            "$(op)b",
            ".$(op)b",
        ],
        2 => [ # binary
            "a $op b",
            "a .$op b",
            "a $(op)₁ b",
            "a $(op)\U0304 b",
            "a .$(op)₁ b"
        ]
    ]

    for (arity, container) in strs
        for str in container
            expr = Meta.parse(str, raise = false)
            if VERSION < v"1.7" && str == "a .&& b"
                expr = Expr(Symbol(".&&"), :a, :b)
            end
            if expr isa Expr && (expr.head != :error && expr.head != :incomplete)
                tokens = collect(tokenize(str))
                exop = expr.head == :call ? expr.args[1] : expr.head
                @test Symbol(Tokenize.Tokens.untokenize(tokens[arity == 1 ? 1 : 3], str)) == exop
            else
                break
            end
        end
    end
end
end

@testset "Normalization of Unicode symbols" begin
    # https://github.com/JuliaLang/julia/pull/25157
    @test tok("\u00b7").kind == T.UNICODE_DOT
    @test tok("\u0387").kind == T.UNICODE_DOT
    @test tok(".\u00b7").dotop
    @test tok(".\u0387").dotop

    # https://github.com/JuliaLang/julia/pull/40948
    @test tok("−").kind == T.MINUS
    @test tok("−=").kind == T.MINUS_EQ
    @test tok(".−").dotop
end

@testset "perp" begin
    @test tok("1 ⟂ 2", 3).kind==T.PERP
end

@testset "outer" begin
    @test tok("outer", 1).kind==T.OUTER
end

function test_error(tok, kind)
    @test Tokens.iserror(tok.kind)
    @test tok.kind == kind
end

@testset "token errors" begin
    test_error(tok("1.2e2.3",1), Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("1.2.",1),    Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("1.2.f",1),   Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("0xv",1),     Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("0b3",1),     Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("0op",1),     Tokens.INVALID_NUMERIC_CONSTANT)
    test_error(tok("--",1),      Tokens.INVALID_OPERATOR)
    test_error(tok("1**2",2),    Tokens.INVALID_OPERATOR)
end

@testset "hat suffix" begin
    @test tok("ŝ", 1).kind==Tokens.IDENTIFIER
    @test untokenize(collect(tokenize("ŝ"))[1], "ŝ") == "ŝ"
end

@testset "suffixed op" begin
    s = "+¹"
    @test Tokens.isoperator(tok(s, 1).kind)
    @test untokenize(collect(tokenize(s))[1], s) == s
end

@testset "invalid float juxt" begin
    s = "1.+2"
    @test tok(s, 1).kind == Tokens.ERROR
    @test Tokens.isoperator(tok(s, 2).kind)
    test_roundtrip("1234.+1", Tokens.ERROR, "1234.")
    @test tok("1.+ ").kind == Tokens.ERROR
    @test tok("1.⤋").kind  == Tokens.ERROR
    @test tok("1.?").kind == Tokens.ERROR
end

@testset "comments" begin
    s = "#=# text=#"
    @test length(collect(tokenize(s))) == 2
end

@testset "invalid hexadecimal" begin
    s = "0x."
    tok(s, 1).kind === Tokens.ERROR
end

@testset "circ arrow right op" begin
    s = "↻"
    @test collect(tokenize(s))[1].kind == Tokens.CIRCLE_ARROW_RIGHT
end

@testset "invalid float" begin
    s = ".0."
    @test collect(tokenize(s))[1].kind == Tokens.INVALID_NUMERIC_CONSTANT
end

@testset "allow prime after end" begin
    @test tok("begin end'", 4).kind === Tokens.PRIME
end

@testset "new ops" begin
    ops = [
        raw"= += -= *= /= //= \= ^= ÷= %= <<= >>= >>>= |= &= ⊻= ≔ ⩴ ≕ ~ := $="
        raw"=>"
        raw"?"
        raw"← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫ ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ -->"
        raw"||"
        raw"&&"
        raw"> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≂ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣ ⟂ <: >:"
        raw"<|"
        raw"|>"
        raw": .. … ⁝ ⋮ ⋱ ⋰ ⋯"
        raw"$ + - ¦ | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣"
        raw"* / ⌿ ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗"
        raw"//"
        raw"<< >> >>>"
        raw"^ ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ￪ ￬"
        raw"::"
        raw"."
    ]
    if VERSION >= v"1.6.0"
        push!(ops, raw"<-- <-->")
    end
    allops = split(join(ops, " "), " ")
    @test all(s->Base.isoperator(Symbol(s)) == Tokens.isoperator(first(collect(tokenize(s))).kind), allops)
end

const all_kws = Set([
    # Keywords
    "baremodule",
    "begin",
    "break",
    "catch",
    "const",
    "continue",
    "do",
    "else",
    "elseif",
    "end",
    "export",
    "finally",
    "for",
    "function",
    "global",
    "if",
    "import",
    "let",
    "local",
    "macro",
    "module",
    "quote",
    "return",
    "struct",
    "try",
    "using",
    "while",
    # Contextual keywords
    "abstract",
    "as",
    "doc",
    "mutable",
    "outer",
    "primitive",
    "type",
    "var",
    # Literals
    "true",
    "false",
    # Word-like operators
    "in",
    "isa",
    "where",
])

function check_kw_hashes(iter)
    for cs in iter
        str = String([cs...])
        if Lexers.simple_hash(str) in keys(Tokenize.Lexers.kw_hash)
            @test str in all_kws
        end
    end
end

@testset "simple_hash" begin
    @test length(all_kws) == length(Tokenize.Lexers.kw_hash)

    @testset "Length $len keywords" for len in 1:5
        check_kw_hashes(String([cs...]) for cs in Iterators.product(['a':'z' for _ in 1:len]...))
    end
end

@testset "UTF-8 BOM" begin
    @test Tokenize.Tokens.kind.(collect(tokenize("\ufeff[1\ufeff2]"))) == [
        Tokens.WHITESPACE,
        Tokens.LSQUARE,
        Tokens.INTEGER,
        Tokens.WHITESPACE,
        Tokens.INTEGER,
        Tokens.RSQUARE,
        Tokens.ENDMARKER
    ]
end


text:
struct Footer <: FlatBuffers.Table
    bytes::Vector{UInt8}
    pos::Base.Int
end

Base.propertynames(x::Footer) = (:version, :schema, :dictionaries, :recordBatches, :custom_metadata)

function Base.getproperty(x::Footer, field::Symbol)
    if field === :version
        o = FlatBuffers.offset(x, 4)
        o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion)
        return MetadataVersion.V1
    elseif field === :schema
        o = FlatBuffers.offset(x, 6)
        if o != 0
            y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x))
            return FlatBuffers.init(Schema, FlatBuffers.bytes(x), y)
        end
    elseif field === :dictionaries
        o = FlatBuffers.offset(x, 8)
        if o != 0
            return FlatBuffers.Array{Block}(x, o)
        end
    elseif field === :recordBatches
        o = FlatBuffers.offset(x, 10)
        if o != 0
            return FlatBuffers.Array{Block}(x, o)
        end
    elseif field === :custom_metadata
        o = FlatBuffers.offset(x, 12)
        if o != 0
            return FlatBuffers.Array{KeyValue}(x, o)
        end
    end
    return nothing
end

footerStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 4)
footerAddVersion(b::FlatBuffers.Builder, version::MetadataVersion) = FlatBuffers.prependslot!(b, 0, version, 0)
footerAddSchema(b::FlatBuffers.Builder, schema::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, schema, 0)
footerAddDictionaries(b::FlatBuffers.Builder, dictionaries::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, dictionaries, 0)
footerStartDictionariesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8)
footerAddRecordBatches(b::FlatBuffers.Builder, recordbatches::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, recordbatches, 0)
footerStartRecordBatchesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8)
footerEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)

struct Block <: FlatBuffers.Struct
    bytes::Vector{UInt8}
    pos::Base.Int
end

FlatBuffers.structsizeof(::Base.Type{Block}) = 24

Base.propertynames(x::Block) = (:offset, :metaDataLength, :bodyLength)

function Base.getproperty(x::Block, field::Symbol)
    if field === :offset
        return FlatBuffers.get(x, FlatBuffers.pos(x), Int64)
    elseif field === :metaDataLength
        return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int32)
    elseif field === :bodyLength
        return FlatBuffers.get(x, FlatBuffers.pos(x) + 16, Int64)
    end
    return nothing
end

function createBlock(b::FlatBuffers.Builder, offset::Int64, metadatalength::Int32, bodylength::Int64)
    FlatBuffers.prep!(b, 8, 24)
    prepend!(b, bodylength)
    FlatBuffers.pad!(b, 4)
    prepend!(b, metadatalength)
    prepend!(b, offset)
    return FlatBuffers.offset(b)
end


text:
module TikzPictures

export TikzPicture, PDF, TEX, TIKZ, SVG, save, tikzDeleteIntermediate, tikzCommand, TikzDocument, push!
import Base: push!
import LaTeXStrings: LaTeXString, @L_str, @L_mstr
export LaTeXString, @L_str, @L_mstr

_tikzDeleteIntermediate = true
_tikzCommand = "lualatex"

mutable struct TikzPicture
    data::AbstractString
    options::AbstractString
    preamble::AbstractString
    environment::AbstractString
    width::AbstractString
    height::AbstractString
    keepAspectRatio::Bool
    enableWrite18::Bool
    TikzPicture(data::AbstractString; options="", preamble="", environment="tikzpicture", width="", height="", keepAspectRatio=true, enableWrite18=true) = new(data, options, preamble, environment, width, height, keepAspectRatio, enableWrite18)
end

mutable struct TikzDocument
    pictures::Vector{TikzPicture}
    captions::Vector{AbstractString}
end
TikzDocument() = TikzDocument(TikzPicture[], String[])

# svg handling
include("svg.jl")
__init__() = __init__svg()

tikzUsePoppler() = svgBackend() == PopplerBackend()
tikzUsePoppler(value::Bool) = svgBackend(value ? PopplerBackend() : DVIBackend())

# standalone workaround:
# see http://tex.stackexchange.com/questions/315025/lualatex-texlive-2016-standalone-undefined-control-sequence
_standaloneWorkaround = false

function standaloneWorkaround()
    global _standaloneWorkaround
    _standaloneWorkaround
end

function standaloneWorkaround(value::Bool)
    global _standaloneWorkaround
    _standaloneWorkaround = value
    nothing
end

function tikzDeleteIntermediate(value::Bool)
    global _tikzDeleteIntermediate
    _tikzDeleteIntermediate = value
    nothing
end

function tikzDeleteIntermediate()
    global _tikzDeleteIntermediate
    _tikzDeleteIntermediate
end

function tikzCommand(value::AbstractString)
    global _tikzCommand
    _tikzCommand = value
    nothing
end

function tikzCommand()
    global _tikzCommand
    _tikzCommand
end

function push!(td::TikzDocument, tp::TikzPicture; caption="")
    push!(td.pictures, tp)
    push!(td.captions, caption)
end

function removeExtension(filename::AbstractString, extension::AbstractString)
    if endswith(filename, extension) || endswith(filename, uppercase(extension))
        return filename[1:(end - length(extension))]
    else
        return filename
    end
end

abstract type SaveType end

mutable struct PDF <: SaveType
    filename::AbstractString
    PDF(filename::AbstractString) = new(removeExtension(filename, ".pdf"))
end

mutable struct TEX <: SaveType
    filename::AbstractString
    limit_to::Symbol
    TEX(filename::AbstractString; include_preamble::Bool=true, limit_to::Symbol=(include_preamble ? :all : :picture)) =
        new(removeExtension(filename, ".tex"), limit_to)
end

mutable struct TIKZ <: SaveType
    filename::AbstractString
    limit_to::Symbol
    TIKZ(filename::AbstractString; include_preamble::Bool=true, limit_to::Symbol=(include_preamble ? :all : :picture)) = new(removeExtension(filename, ".tikz"), limit_to)
end

mutable struct SVG <: SaveType
    filename::AbstractString
    SVG(filename::AbstractString) = new(removeExtension(filename, ".svg"))
end

extension(f::SaveType) = lowercase(split("$(typeof(f))",".")[end])

resize(tp::TikzPicture) = !isempty(tp.width) || !isempty(tp.height)

function write_adjustbox_options(tex::IO, tp::TikzPicture)
    adjustbox_options = []
    if !isempty(tp.width)
        push!(adjustbox_options, "width=$(tp.width)")
    end
    if !isempty(tp.height)
        push!(adjustbox_options, "height=$(tp.height)")
    end
    if tp.keepAspectRatio
        push!(adjustbox_options, "keepaspectratio")
    end
    adjustbox_option_string = join(adjustbox_options, ',')
    println(tex, "\\begin{adjustbox}{$adjustbox_option_string}")
end

showable(::MIME"image/svg+xml", tp::TikzPicture) = true

function save(f::Union{TEX,TIKZ}, tp::TikzPicture)
    if !in(f.limit_to, [:all, :picture, :data])
        throw(ArgumentError("limit_to must be one of :all, :picture, and :data"))
    end
    filename = f.filename
    ext = extension(f)
    tex = open("$(filename).$(ext)", "w")
    if f.limit_to in [:all, :picture]
        if f.limit_to == :all
            if standaloneWorkaround()
                println(tex, "\\RequirePackage{luatex85}")
            end
            if resize(tp)
                # [tikz] class option has to be moved to a \usepackage
                # https://tex.stackexchange.com/questions/455546/can-i-resize-a-tikz-picture-to-have-certain-dimensions-width-height
                println(tex, "\\documentclass{standalone}")
                println(tex, "\\usepackage{tikz}")
                println(tex, "\\usepackage{adjustbox}")
            else
                println(tex, "\\documentclass[tikz]{standalone}")
            end
            println(tex, tp.preamble)
            println(tex, "\\begin{document}")
        end

        if resize(tp)
            write_adjustbox_options(tex, tp)
        end
        print(tex, "\\begin{$(tp.environment)}[")
        print(tex, tp.options)
        println(tex, "]")
    end
    println(tex, tp.data)
    if f.limit_to in [:all, :picture]
        println(tex, "\\end{$(tp.environment)}")
        if resize(tp)
            println(tex, "\\end{adjustbox}")
        else
            print(tex, "\n")
        end
        if f.limit_to == :all
            println(tex, "\\end{document}")
        end
    end
    close(tex)
end

function save(f::TEX, td::TikzDocument)
    if isempty(td.pictures)
        throw(ArgumentError("TikzDocument does not contain pictures"))
    elseif !in(f.limit_to, [:all, :picture])
        throw(ArgumentError("limit_to must be either :all or :picture"))
    end
    filename = f.filename
    tex = open("$(filename).tex", "w")
    if f.limit_to == :all
        println(tex, "\\documentclass{article}")
        println(tex, "\\usepackage{caption}")
        println(tex, "\\usepackage{tikz}")
        println(tex, td.pictures[1].preamble)
        println(tex, "\\begin{document}")
    end
    println(tex, "\\centering")
    @assert length(td.pictures) == length(td.captions)
    i = 1
    for tp in td.pictures
        println(tex, "\\centering")
        if resize(tp)
            write_adjustbox_options(tex, tp)
        end
        print(tex, "\\begin{$(tp.environment)}[")
        print(tex, tp.options)
        println(tex, "]")
        println(tex, tp.data)
        println(tex, "\\end{$(tp.environment)}")
        if resize(tp)
            println(tex, "\\end{adjustbox}")
        end
        print(tex, "\\captionof{figure}{")
        print(tex, td.captions[i])
        println(tex, "}")
        println(tex, "\\vspace{5ex}")
        println(tex)
        i += 1
    end
    if f.limit_to == :all
        println(tex, "\\end{document}")
    end
    close(tex)
end

function latexerrormsg(s)
    beginError = false
    for l in split(s, '\n')
        if beginError
            if !isempty(l) && l[1] == '?'
                return
            else
                println(l)
            end
        else
            if !isempty(l) && l[1] == '!'
                println(l)
                beginError = true
            end
        end
    end
end

_joinpath(a, b) = "$a/$b"

function save(f::PDF, tp::TikzPicture)

    # Isolate basename and foldername of file
    basefilename = basename(f.filename)
    working_dir = dirname(abspath(f.filename))

    # Call anonymous function to do task and automatically return
    cd(working_dir) do
        temp_dir = mktempdir("./")
        temp_filename = _joinpath(temp_dir,basefilename)

        # Save the TEX file in tmp dir
        save(TEX(temp_filename * ".tex"), tp)

        # From the .tex file, generate a pdf within the tmp folder
        latexCommand = ``
        if tp.enableWrite18
            latexCommand = `$(tikzCommand()) --enable-write18 --output-directory=$(temp_dir) $(temp_filename*".tex")`
        else
            latexCommand = `$(tikzCommand()) --output-directory=$(temp_dir) $(temp_filename*".tex")`
        end

        latexSuccess = success(latexCommand)

        tex_log = ""
        try
            tex_log = read(temp_filename * ".log", String)
        catch
            tex_log = read(_joinpath(temp_dir,"texput.log"), String)
        end

        if occursin("LaTeX Warning: Label(s)", tex_log)
            latexSuccess = success(latexCommand)
        end

        # Move PDF out of tmpdir regardless
        # Give warning if PDF file already exists
        if isfile("$(basefilename).pdf")
            @warn "$(basefilename).pdf already exists, overwriting!"
        end
        if latexSuccess
            mv("$(temp_filename).pdf", "$(basefilename).pdf",force=true)
        end

        try
            # Shouldn't need to be try-catched anymore, but best to be safe
            # This failing is NOT critical either, so just make it a warning
            if tikzDeleteIntermediate()
                # Delete tmp dir
                rm(temp_dir, recursive=true)
            end
        catch
            @warn "TikzPictures: Your intermediate files are not being deleted."
        end

        if !latexSuccess
            # Remove failed attempt.
            if !standaloneWorkaround() && occursin("\\sa@placebox ->\\newpage \\global \\pdfpagewidth", tex_log)
                standaloneWorkaround(true)
                save(f, tp)
                return
            end
            latexerrormsg(tex_log)
            error("LaTeX error")
        end
    end
end

function save(f::PDF, td::TikzDocument)
    # Isolate basename and foldername of file
    basefilename = basename(f.filename)
    working_dir = dirname(abspath(f.filename))

    # Call anonymous function to do task and automatically return
    cd(working_dir) do
        # Create tmp dir in working directory
        temp_dir = mktempdir("./")
        temp_filename = _joinpath(temp_dir,basefilename)

        try
            save(TEX(temp_filename * ".tex"), td)
            if td.pictures[1].enableWrite18
                success(`$(tikzCommand()) --enable-write18 --output-directory=$(temp_dir) $(temp_filename)`)
            else
                success(`$(tikzCommand()) --output-directory=$(temp_dir) $(temp_filename)`)
            end

            # Move PDF out of tmpdir regardless
            if isfile("$(basefilename).pdf")
                @warn "$(basefilename).pdf already exists, overwriting!"
            end
            mv("$(temp_filename).pdf", "$(basefilename).pdf",force=true)

            try
                # Shouldn't need to be try-catched anymore, but best to be safe
                # This failing is NOT critical either, so just make it a warning
                if tikzDeleteIntermediate()
                    # Delete tmp dir
                    rm(temp_dir, recursive=true)
                end
            catch
                @warn "TikzPictures: Your intermediate files are not being deleted."
            end
        catch
            @warn "Error saving as PDF."
            rethrow()
        end
    end
end

function save(f::SVG, tp::TikzPicture)
    basefilename = basename(f.filename)
    working_dir = dirname(abspath(f.filename))

    # Call anonymous function to do task and automatically return
    cd(working_dir) do
        # Create tmp dir in working directory
        temp_dir = mktempdir("./")
        temp_filename = _joinpath(temp_dir,basefilename)

        # Save the TEX file in tmp dir, then convert to SVG
        save(TEX(temp_filename * ".tex"), tp)
        _mkTempSvg(tp, temp_dir, temp_filename)

        # Move SVG out of tmpdir into working dir and give warning if overwriting
        if isfile("$(basefilename).svg")
            @warn "$(basefilename).svg already exists, overwriting!"
        end
        mv("$(temp_filename).svg", "$(basefilename).svg",force=true)

        try
            # Shouldn't need to be try-catched anymore, but best to be safe
            # This failing is NOT critical either, so just make it a warning
            if tikzDeleteIntermediate()
                # Delete tmp dir
                rm(temp_dir, recursive=true)
            end
        catch
            @warn "TikzPictures: Your intermediate files are not being deleted."
        end
    end
end

# this is needed to work with multiple images in ijulia (kind of a hack)
global _tikzid = round(UInt64, time() * 1e6)

function Base.show(f::IO, ::MIME"image/svg+xml", tp::TikzPicture)
    global _tikzid
    filename = "tikzpicture"
    save(SVG(filename), tp)
    s = read("$filename.svg", String)
    s = replace(s, "glyph" => "glyph-$(_tikzid)-")
    s = replace(s, "\"clip" => "\"clip-$(_tikzid)-")
    s = replace(s, "#clip" => "#clip-$(_tikzid)-")
    s = replace(s, "\"image" => "\"image-$(_tikzid)-")
    s = replace(s, "#image" => "#image-$(_tikzid)-")
    s = replace(s, "linearGradient id=\"linear" => "linearGradient id=\"linear-$(_tikzid)-")
    s = replace(s, "#linear" => "#linear-$(_tikzid)-")
    s = replace(s, "image id=\"" => "image style=\"image-rendering: pixelated;\" id=\"")
    _tikzid += 1
    println(f, s)
    if tikzDeleteIntermediate()
        rm("$filename.svg")
    end
end

end # module