text:
#=
include("display_gfx.jl")
include("move.jl")
include("move_user_drop.jl")
include("move_user_move.jl")
include("move_user_resign.jl")
include("networking.jl")
include("start.jl")
include("board.jl")

=#
include("display_gfx.jl")

using Tk

w = Toplevel("SHOGI", 400, 300)
tcl("pack", "propagate", w, false) ## or pack_stop_propagate(w)

mb = Menu(w)            ## makes menu, adds to top-level window
fmenu = menu_add(mb, "File")
omenu = menu_add(mb, "Options")

menu_add(fmenu, "Open file...", (path) -> println("Open file dialog, ..."))
menu_add(fmenu, Separator(w))   ## second argument is Tk_Separator instance
menu_add(fmenu, "Close window", (path) -> destroy(w))

cb = Checkbutton(w, "Something visible")
set_value(cb, false)     ## initialize
menu_add(omenu, cb)     ## second argument is Tk_Checkbutton instance

menu_add(omenu, Separator(w))   ## put in a separator

rb = Radio(w, ["option 1", "option 2"])
set_value(rb, "option 1")   ## initialize
menu_add(omenu, rb)     ## second argument is Tk_Radio instance

global JRMvalue
JRMvalue = false
global VARvalue
VARvalue = "shogi"
global DIFFvalue
DIFFvalue = "normal"
global CHEATvalue
CHEATvalue = false
global FIRSTvalue
FIRSTvalue = false
global TIMEvalue
TIMEvalue = false
global LIMITvalue
LIMITvalue = "infinity"
global TheTimeLimit
TheTimeLimit = 0
global VSAI
VSAI = false
global VSH
VSH = false
global HAI
VSAI = false
global HSH
VSH = false

n = Button(w, "• Start a new game")
o = Button(w, "• Continue an old game")
r = Button(w, "• Replay a finished game")
q = Button(w, "• Quit")

pack(n, expand=true, fill="both")
pack(o, expand=true, fill="both")
pack(r, expand=true, fill="both")
pack(q, expand=true, fill="both")

function callbackLIMIT(path)

  global JRMvalue
  global VARvalue
  global DIFFvalue
  global CHEATvalue
  global FIRSTvalue
  global TIMEvalue
  global LIMITvalue
  global TheTimeLimit
  global VSAI
  global VSH
  global HAI
  global HSH
  z = Toplevel()
  f = Frame(z); pack(f, expand=true, fill="both")

#sepextra3 = Label(f, "                         ")
#pack(sepextra2, expand=true, fill="both")

e = Entry(f)
b = Button(f, "Ok")

formlayout(e, "Time Limit (seconds):")
formlayout(b, nothing)
focus(e)            ## put keyboard focus on widget

#=
map(u -> pack(u, anchor="w"), (e, b))

cbcTIME =Checkbutton(f, "Use time limit")
map(u -> pack(u, anchor="w"), (sepextra3, cbcTIME))

pack(sepextra3, expand=true, fill="both")
pack(cbcTIME, expand=true, fill="both")
=#

#make different functions for each path
function callbackLIMITTIME(path)
  global JRMvalue
  global VARvalue
  global DIFFvalue
  global CHEATvalue
  global FIRSTvalue
  global TIMEvalue
  global TheTimeLimit
  global VSAI
  global VSH
  global HAI
  global HSH

  global LIMITvalue = get_value(e)
  println("$LIMITvalue")
  #msg = "You have a nice name $val"
  #Messagebox(w,  msg)
  global TheTimeLimit
  if  TIMEvalue == true
    if parse(Int64,LIMITvalue) >= 1
      TheTimeLimit = parse(Int64,LIMITvalue)
    #  println("the time limit is :$TheTimeLimit")
    end
  end
#=
  JRM = false
  if  JRMvalue == true
      JRM = JRMvalue
      println("japanese roulette mode? :$JRM")
  end

  shvar = " "
  if  VARvalue != "no shogi"
      shvar = VARvalue
      println("shogi variant :$shvar")
  else
    shvar = "yes shogi"
    println("shogi variant :$shvar")
  end

  dfvar = " "
  if  DIFFvalue != "not normal"
      dfvar = DIFFvalue
      println("difficulty :$dfvar")
  else
    dfvar = "normal"
    println("difficulty :$dfvar")
  end

  CHEAT = false
  if  CHEATvalue == true
      CHEAT = CHEATvalue
      println("cheating? :$CHEAT")
  end

  FIRST = false
  if  FIRSTvalue == true
      FIRST = FIRSTvalue
      println("going first? :$FIRST")
  end
=#
  #return LIMITvalue

#while true
#  dispgfx()
if VARvalue == "shogi"
  dispgfx()
else
  mini()
end

end

bind(b, "command", callbackLIMITTIME)
bind(b, "<Return>", callbackLIMITTIME)
bind(e, "<Return>", callbackLIMITTIME)  ## bind to a certain key press event

#=
if  TIMEvalue == true
  if parse(Int64,LIMITvalue) >= 1
    TheTimeLimit = parse(Int64,LIMITvalue)
    println("the time limit is :$TheTimeLimit")
  end
end

JRM = false
if  JRMvalue == true
    JRM = JRMvalue
    println("japanese roulette mode? :$JRM")
end

shvar = " "
if  VARvalue != "no shogi"
    shvar = VARvalue
    println("shogi variant :$shvar")
else
  shvar = "yes shogi"
  println("shogi variant :$shvar")
end

dfvar = " "
if  DIFFvalue != "not normal"
    dfvar = DIFFvalue
    println("difficulty :$dfvar")
else
  dfvar = "normal"
  println("difficulty :$dfvar")
end

CHEAT = false
if  CHEATvalue == true
    CHEAT = CHEATvalue
    println("cheating? :$CHEAT")
end

FIRST = false
if  FIRSTvalue == true
    FIRST = FIRSTvalue
    println("going first? :$FIRST")
end

=#

end

function callback(path)
  vals = map(get_value, (cb, rb))
  println(vals)
end

function callbackr(path)
  aa = Messagebox(w, message="Are you sure you want to quit")
  msg = aa[:textvariable]
end

function callbacknVSAIOPT(path)
  y = Toplevel("Options")
  f = Frame(y)
  pack(f, expand=true, fill="both")

  global JRMvalue
#  JRMvalue = false
  global VARvalue
#  VARvalue = "shogi"
  global DIFFvalue
#  DIFFvalue = "normal"
  global CHEATvalue
#  CHEATvalue = false
  global FIRSTvalue
#  FIRSTvalue = false
  global TIMEvalue
#  TIMEvalue = false
  global LIMITvalue
#  LIMITvalue = "infinity"
global VSAI
VSAI = true
global VSH
global HAI
global HSH

  cbcJRM =Checkbutton(f, "Japanese roulette mode")
  #pack(cbcJRM)
  pack(cbcJRM, expand=true, fill="both")

  sepextra = Label(f, "                         ")

  cbcCHEAT =Checkbutton(f, "Permit AI to cheat")
  map(u -> pack(u, anchor="w"), (sepextra, cbcCHEAT))

  #=
  pack(sepextra, expand=true, fill="both")
  pack(cbcCHEAT, expand=true, fill="both")
=#

  sepextra1 = Label(f, "                         ")

  cbcFIRST =Checkbutton(f, "Go First")
  map(u -> pack(u, anchor="w"), (sepextra1, cbcFIRST))
  #=
  pack(sepextra1, expand=true, fill="both")
  pack(cbcFIRST, expand=true, fill="both")

  =#
  sepextra2 = Label(f, "                         ")

  cbcTIME =Checkbutton(f, "Use time limit")
  map(u -> pack(u, anchor="w"), (sepextra2, cbcTIME))
  #=
  pack(sepextra2, expand=true, fill="both")
  pack(cbcTIME, expand=true, fill="both")

  =#

  sep1 = Label(f, "                         ")

  varmes  = Label(f, "shogi variant:")
  vartype = Radio(f, ["shogi", "minishogi", "chu shogi", "tenjiku shogi"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep1, varmes, vartype))     ## pack in left to right
#=
  pack(sep1, expand=true, fill="both")
  pack(varmes, expand=true, fill="both")
  pack(vartype, expand=true, fill="both")
=#

  sep2 = Label(f, "                         ")

  diffmes  = Label(f, "difficulty:")
  difftype = Radio(f, ["normal", "hard", "suicidal", "protracted death", "random AI"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep2, diffmes, difftype))     ## pack in left to right
#=
  pack(sep2, expand=true, fill="both")
  pack(diffmes, expand=true, fill="both")
  pack(difftype, expand=true, fill="both")
=#

bOK = Button(y, "Ok")
pack(bOK, expand=true, fill="both")

  function callbackJRM(path)        ## callbacks have at least one argument
    global JRMvalue = get_value(cbcJRM)
    println("$JRMvalue")
    return JRMvalue
  end

  function callbackCHEAT(path)        ## callbacks have at least one argument
    global CHEATvalue = get_value(cbcCHEAT)
    println("$CHEATvalue")
    return CHEATvalue
  end

  function callbackFIRST(path)        ## callbacks have at least one argument
    global FIRSTvalue = get_value(cbcFIRST)
    println("$FIRSTvalue")
    return FIRSTvalue
  end

  function callbackTIME(path)        ## callbacks have at least one argument
    global TIMEvalue = get_value(cbcTIME)
    println("$TIMEvalue")
    return TIMEvalue
  end

  function callbackvar(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
    global VARvalue = get_value(vartype)
    println("$VARvalue")
  #  Messagebox(w, msg)
    return VARvalue
  end

  function callbackdiff(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
  =#
    global DIFFvalue = get_value(difftype)
    println("$DIFFvalue")
  #  Messagebox(w, msg)
    return DIFFvalue
  end

  bind(cbcJRM, "command", callbackJRM)   ## bind to command option
#  println("$JRMvalue")

  bind(cbcCHEAT, "command", callbackCHEAT)   ## bind to command option

  bind(cbcFIRST, "command", callbackFIRST)   ## bind to command option

  bind(cbcTIME, "command", callbackTIME)   ## bind to command option

  bind(vartype, "command", callbackvar)

  bind(difftype, "command", callbackdiff)

#  bind(bOK, "command", callback)
#  bind(bOK, "<Return>", callback)

  #if TIMEvalue == true
    bind(bOK, "command", callbackLIMIT)
    bind(bOK, "<Return>", callbackLIMIT)
    callback_add(bOK, callbackLIMIT)
  #end

end

function callbacknVSHOPT(path)
  y = Toplevel("Options")
  f = Frame(y)
  pack(f, expand=true, fill="both")

  global JRMvalue
#  JRMvalue = false
  global VARvalue
#  VARvalue = "shogi"
  #global DIFFvalue
  #DIFFvalue = "normal"
  #global CHEATvalue
  #CHEATvalue = false
  #global FIRSTvalue
  #FIRSTvalue = false
  global TIMEvalue
#  TIMEvalue = false
  global LIMITvalue
#  LIMITvalue = "infinity"
global VSAI
global VSH
VSH = true
global HAI
global HSH

  cbcJRM =Checkbutton(f, "Japanese roulette mode")
  #pack(cbcJRM)
  pack(cbcJRM, expand=true, fill="both")

#=
  sepextra = Label(f, "                         ")

  cbcCHEAT =Checkbutton(f, "Permit AI to cheat")
  map(u -> pack(u, anchor="w"), (sepextra, cbcCHEAT))
=#
  #=
  pack(sepextra, expand=true, fill="both")
  pack(cbcCHEAT, expand=true, fill="both")
=#

#=
  sepextra1 = Label(f, "                         ")

  cbcFIRST =Checkbutton(f, "Go First")
  map(u -> pack(u, anchor="w"), (sepextra1, cbcFIRST))

  =#
  #=
  pack(sepextra1, expand=true, fill="both")
  pack(cbcFIRST, expand=true, fill="both")

  =#
  sepextra2 = Label(f, "                         ")

  cbcTIME =Checkbutton(f, "Use time limit")
  map(u -> pack(u, anchor="w"), (sepextra2, cbcTIME))
  #=
  pack(sepextra2, expand=true, fill="both")
  pack(cbcTIME, expand=true, fill="both")

  =#

  sep1 = Label(f, "                         ")

  varmes  = Label(f, "shogi variant:")
  vartype = Radio(f, ["shogi", "minishogi", "chu shogi", "tenjiku shogi"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep1, varmes, vartype))     ## pack in left to right
#=
  pack(sep1, expand=true, fill="both")
  pack(varmes, expand=true, fill="both")
  pack(vartype, expand=true, fill="both")
=#
#=
  sep2 = Label(f, "                         ")

  diffmes  = Label(f, "difficulty:")
  difftype = Radio(f, ["normal", "hard", "suicidal", "protracted death", "random AI"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep2, diffmes, difftype))     ## pack in left to right

=#
#=
  pack(sep2, expand=true, fill="both")
  pack(diffmes, expand=true, fill="both")
  pack(difftype, expand=true, fill="both")
=#

bOK = Button(y, "Ok")
pack(bOK, expand=true, fill="both")

  function callbackJRM(path)        ## callbacks have at least one argument
    global JRMvalue = get_value(cbcJRM)
    println("$JRMvalue")
    return JRMvalue
  end

#=
  function callbackCHEAT(path)        ## callbacks have at least one argument
    global CHEATvalue = get_value(cbcCHEAT)
    println("$CHEATvalue")
    return CHEATvalue
  end

  function callbackFIRST(path)        ## callbacks have at least one argument
    global FIRSTvalue = get_value(cbcFIRST)
    println("$FIRSTvalue")
    return FIRSTvalue
  end
=#

  function callbackTIME(path)        ## callbacks have at least one argument
    global TIMEvalue = get_value(cbcTIME)
    println("$TIMEvalue")
    return TIMEvalue
  end

  function callbackvar(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
    global VARvalue = get_value(vartype)
    println("$VARvalue")
  #  Messagebox(w, msg)
    return VARvalue
  end

#  function callbackdiff(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
  #  global DIFFvalue = get_value(difftype)
  #  println("$DIFFvalue")
  #  Messagebox(w, msg)
  #  return DIFFvalue
  #end

  bind(cbcJRM, "command", callbackJRM)   ## bind to command option
#  println("$JRMvalue")

#  bind(cbcCHEAT, "command", callbackCHEAT)   ## bind to command option

#  bind(cbcFIRST, "command", callbackFIRST)   ## bind to command option

  bind(cbcTIME, "command", callbackTIME)   ## bind to command option

  bind(vartype, "command", callbackvar)

#  bind(difftype, "command", callbackdiff)

#  bind(bOK, "command", callback)
#  bind(bOK, "<Return>", callback)

  #if TIMEvalue == true
    bind(bOK, "command", callbackLIMIT)
    bind(bOK, "<Return>", callbackLIMIT)
    callback_add(bOK, callbackLIMIT)
  #end

end
function callbacknHAIOPT(path)
  y = Toplevel("Options")
  f = Frame(y)
  pack(f, expand=true, fill="both")

  global JRMvalue
#  JRMvalue = false
  global VARvalue
#  VARvalue = "shogi"
  #global DIFFvalue
  #DIFFvalue = "normal"
  #global CHEATvalue
  #CHEATvalue = false
  #global FIRSTvalue
  #FIRSTvalue = false
  global TIMEvalue
#  TIMEvalue = false
  global LIMITvalue
#  LIMITvalue = "infinity"
global VSAI
global VSH
global HAI
VSAI = true
global HSH

  cbcJRM =Checkbutton(f, "Japanese roulette mode")
  #pack(cbcJRM)
  pack(cbcJRM, expand=true, fill="both")

#=
  sepextra = Label(f, "                         ")

  cbcCHEAT =Checkbutton(f, "Permit AI to cheat")
  map(u -> pack(u, anchor="w"), (sepextra, cbcCHEAT))
=#
  #=
  pack(sepextra, expand=true, fill="both")
  pack(cbcCHEAT, expand=true, fill="both")
=#

#=
  sepextra1 = Label(f, "                         ")

  cbcFIRST =Checkbutton(f, "Go First")
  map(u -> pack(u, anchor="w"), (sepextra1, cbcFIRST))

  =#
  #=
  pack(sepextra1, expand=true, fill="both")
  pack(cbcFIRST, expand=true, fill="both")

  =#
  sepextra2 = Label(f, "                         ")

  cbcTIME =Checkbutton(f, "Use time limit")
  map(u -> pack(u, anchor="w"), (sepextra2, cbcTIME))
  #=
  pack(sepextra2, expand=true, fill="both")
  pack(cbcTIME, expand=true, fill="both")

  =#

  sep1 = Label(f, "                         ")

  varmes  = Label(f, "shogi variant:")
  vartype = Radio(f, ["shogi", "minishogi", "chu shogi", "tenjiku shogi"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep1, varmes, vartype))     ## pack in left to right
#=
  pack(sep1, expand=true, fill="both")
  pack(varmes, expand=true, fill="both")
  pack(vartype, expand=true, fill="both")
=#
#=
  sep2 = Label(f, "                         ")

  diffmes  = Label(f, "difficulty:")
  difftype = Radio(f, ["normal", "hard", "suicidal", "protracted death", "random AI"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep2, diffmes, difftype))     ## pack in left to right

=#
#=
  pack(sep2, expand=true, fill="both")
  pack(diffmes, expand=true, fill="both")
  pack(difftype, expand=true, fill="both")
=#

bOK = Button(y, "Ok")
pack(bOK, expand=true, fill="both")

  function callbackJRM(path)        ## callbacks have at least one argument
    global JRMvalue = get_value(cbcJRM)
    println("$JRMvalue")
    return JRMvalue
  end

#=
  function callbackCHEAT(path)        ## callbacks have at least one argument
    global CHEATvalue = get_value(cbcCHEAT)
    println("$CHEATvalue")
    return CHEATvalue
  end

  function callbackFIRST(path)        ## callbacks have at least one argument
    global FIRSTvalue = get_value(cbcFIRST)
    println("$FIRSTvalue")
    return FIRSTvalue
  end
=#

  function callbackTIME(path)        ## callbacks have at least one argument
    global TIMEvalue = get_value(cbcTIME)
    println("$TIMEvalue")
    return TIMEvalue
  end

  function callbackvar(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
    global VARvalue = get_value(vartype)
    println("$VARvalue")
  #  Messagebox(w, msg)
    return VARvalue
  end

#  function callbackdiff(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
  #  global DIFFvalue = get_value(difftype)
  #  println("$DIFFvalue")
  #  Messagebox(w, msg)
  #  return DIFFvalue
  #end

  bind(cbcJRM, "command", callbackJRM)   ## bind to command option
#  println("$JRMvalue")

#  bind(cbcCHEAT, "command", callbackCHEAT)   ## bind to command option

#  bind(cbcFIRST, "command", callbackFIRST)   ## bind to command option

  bind(cbcTIME, "command", callbackTIME)   ## bind to command option

  bind(vartype, "command", callbackvar)

#  bind(difftype, "command", callbackdiff)

#  bind(bOK, "command", callback)
#  bind(bOK, "<Return>", callback)

  #if TIMEvalue == true
    bind(bOK, "command", callbackLIMIT)
    bind(bOK, "<Return>", callbackLIMIT)
    callback_add(bOK, callbackLIMIT)
  #end

end

function callbacknHHOPT(path)
  y = Toplevel("Options")
  f = Frame(y)
  pack(f, expand=true, fill="both")

  global JRMvalue
#  JRMvalue = false
  global VARvalue
#  VARvalue = "shogi"
  #global DIFFvalue
  #DIFFvalue = "normal"
  #global CHEATvalue
  #CHEATvalue = false
  #global FIRSTvalue
  #FIRSTvalue = false
  global TIMEvalue
#  TIMEvalue = false
  global LIMITvalue
#  LIMITvalue = "infinity"
global VSAI
global VSH
global HAI
global HSH
VSH = true

  cbcJRM =Checkbutton(f, "Japanese roulette mode")
  #pack(cbcJRM)
  pack(cbcJRM, expand=true, fill="both")

#=
  sepextra = Label(f, "                         ")

  cbcCHEAT =Checkbutton(f, "Permit AI to cheat")
  map(u -> pack(u, anchor="w"), (sepextra, cbcCHEAT))
=#
  #=
  pack(sepextra, expand=true, fill="both")
  pack(cbcCHEAT, expand=true, fill="both")
=#

#=
  sepextra1 = Label(f, "                         ")

  cbcFIRST =Checkbutton(f, "Go First")
  map(u -> pack(u, anchor="w"), (sepextra1, cbcFIRST))

  =#
  #=
  pack(sepextra1, expand=true, fill="both")
  pack(cbcFIRST, expand=true, fill="both")

  =#
  sepextra2 = Label(f, "                         ")

  cbcTIME =Checkbutton(f, "Use time limit")
  map(u -> pack(u, anchor="w"), (sepextra2, cbcTIME))
  #=
  pack(sepextra2, expand=true, fill="both")
  pack(cbcTIME, expand=true, fill="both")

  =#

  sep1 = Label(f, "                         ")

  varmes  = Label(f, "shogi variant:")
  vartype = Radio(f, ["shogi", "minishogi", "chu shogi", "tenjiku shogi"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep1, varmes, vartype))     ## pack in left to right
#=
  pack(sep1, expand=true, fill="both")
  pack(varmes, expand=true, fill="both")
  pack(vartype, expand=true, fill="both")
=#
#=
  sep2 = Label(f, "                         ")

  diffmes  = Label(f, "difficulty:")
  difftype = Radio(f, ["normal", "hard", "suicidal", "protracted death", "random AI"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep2, diffmes, difftype))     ## pack in left to right

=#
#=
  pack(sep2, expand=true, fill="both")
  pack(diffmes, expand=true, fill="both")
  pack(difftype, expand=true, fill="both")
=#

bOK = Button(y, "Ok")
pack(bOK, expand=true, fill="both")

  function callbackJRM(path)        ## callbacks have at least one argument
    global JRMvalue = get_value(cbcJRM)
    println("$JRMvalue")
    return JRMvalue
  end

#=
  function callbackCHEAT(path)        ## callbacks have at least one argument
    global CHEATvalue = get_value(cbcCHEAT)
    println("$CHEATvalue")
    return CHEATvalue
  end

  function callbackFIRST(path)        ## callbacks have at least one argument
    global FIRSTvalue = get_value(cbcFIRST)
    println("$FIRSTvalue")
    return FIRSTvalue
  end
=#

  function callbackTIME(path)        ## callbacks have at least one argument
    global TIMEvalue = get_value(cbcTIME)
    println("$TIMEvalue")
    return TIMEvalue
  end

  function callbackvar(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
    global VARvalue = get_value(vartype)
    println("$VARvalue")
  #  Messagebox(w, msg)
    return VARvalue
  end

#  function callbackdiff(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
  #  global DIFFvalue = get_value(difftype)
  #  println("$DIFFvalue")
  #  Messagebox(w, msg)
  #  return DIFFvalue
  #end

  bind(cbcJRM, "command", callbackJRM)   ## bind to command option
#  println("$JRMvalue")

#  bind(cbcCHEAT, "command", callbackCHEAT)   ## bind to command option

#  bind(cbcFIRST, "command", callbackFIRST)   ## bind to command option

  bind(cbcTIME, "command", callbackTIME)   ## bind to command option

  bind(vartype, "command", callbackvar)

#  bind(difftype, "command", callbackdiff)

#  bind(bOK, "command", callback)
#  bind(bOK, "<Return>", callback)

  #if TIMEvalue == true
    bind(bOK, "command", callbackLIMIT)
    bind(bOK, "<Return>", callbackLIMIT)
    callback_add(bOK, callbackLIMIT)
  #end

end

function callbacknEOPT(path)
  y = Toplevel("Options")
  f = Frame(y)
  pack(f, expand=true, fill="both")

  global JRMvalue
#  JRMvalue = false
  global VARvalue
#  VARvalue = "shogi"
  #global DIFFvalue
  #DIFFvalue = "normal"
  #global CHEATvalue
  #CHEATvalue = false
  #global FIRSTvalue
  #FIRSTvalue = false
  global TIMEvalue
#  TIMEvalue = false
  global LIMITvalue
#  LIMITvalue = "infinity"
global VSAI
global VSH
global HAI
global HSH

  cbcJRM =Checkbutton(f, "Japanese roulette mode")
  #pack(cbcJRM)
  pack(cbcJRM, expand=true, fill="both")

#=
  sepextra = Label(f, "                         ")

  cbcCHEAT =Checkbutton(f, "Permit AI to cheat")
  map(u -> pack(u, anchor="w"), (sepextra, cbcCHEAT))
=#
  #=
  pack(sepextra, expand=true, fill="both")
  pack(cbcCHEAT, expand=true, fill="both")
=#

#=
  sepextra1 = Label(f, "                         ")

  cbcFIRST =Checkbutton(f, "Go First")
  map(u -> pack(u, anchor="w"), (sepextra1, cbcFIRST))

  =#
  #=
  pack(sepextra1, expand=true, fill="both")
  pack(cbcFIRST, expand=true, fill="both")

  =#
  sepextra2 = Label(f, "                         ")

  cbcTIME =Checkbutton(f, "Use time limit")
  map(u -> pack(u, anchor="w"), (sepextra2, cbcTIME))
  #=
  pack(sepextra2, expand=true, fill="both")
  pack(cbcTIME, expand=true, fill="both")

  =#

  sep1 = Label(f, "                         ")

  varmes  = Label(f, "shogi variant:")
  vartype = Radio(f, ["shogi", "minishogi", "chu shogi", "tenjiku shogi"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep1, varmes, vartype))     ## pack in left to right
#=
  pack(sep1, expand=true, fill="both")
  pack(varmes, expand=true, fill="both")
  pack(vartype, expand=true, fill="both")
=#
#=
  sep2 = Label(f, "                         ")

  diffmes  = Label(f, "difficulty:")
  difftype = Radio(f, ["normal", "hard", "suicidal", "protracted death", "random AI"])
  #b  = Button(f, "ok") #at the end
  map(u -> pack(u, anchor="w"), (sep2, diffmes, difftype))     ## pack in left to right

=#
#=
  pack(sep2, expand=true, fill="both")
  pack(diffmes, expand=true, fill="both")
  pack(difftype, expand=true, fill="both")
=#

bOK = Button(y, "Ok")
pack(bOK, expand=true, fill="both")

  function callbackJRM(path)        ## callbacks have at least one argument
    global JRMvalue = get_value(cbcJRM)
    println("$JRMvalue")
    return JRMvalue
  end

#=
  function callbackCHEAT(path)        ## callbacks have at least one argument
    global CHEATvalue = get_value(cbcCHEAT)
    println("$CHEATvalue")
    return CHEATvalue
  end

  function callbackFIRST(path)        ## callbacks have at least one argument
    global FIRSTvalue = get_value(cbcFIRST)
    println("$FIRSTvalue")
    return FIRSTvalue
  end
=#

  function callbackTIME(path)        ## callbacks have at least one argument
    global TIMEvalue = get_value(cbcTIME)
    println("$TIMEvalue")
    return TIMEvalue
  end

  function callbackvar(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
    global VARvalue = get_value(vartype)
    println("$VARvalue")
  #  Messagebox(w, msg)
    return VARvalue
  end

#  function callbackdiff(path)
  #=  msg = (get_value(rb) == "apples") ? "Good choice!  An apple a day keeps the doctor away!" :
                                        "Good choice!  Oranges are full of Vitamin C!"
=#
  #  global DIFFvalue = get_value(difftype)
  #  println("$DIFFvalue")
  #  Messagebox(w, msg)
  #  return DIFFvalue
  #end

  bind(cbcJRM, "command", callbackJRM)   ## bind to command option
#  println("$JRMvalue")

#  bind(cbcCHEAT, "command", callbackCHEAT)   ## bind to command option

#  bind(cbcFIRST, "command", callbackFIRST)   ## bind to command option

  bind(cbcTIME, "command", callbackTIME)   ## bind to command option

  bind(vartype, "command", callbackvar)

#  bind(difftype, "command", callbackdiff)

#  bind(bOK, "command", callback)
#  bind(bOK, "<Return>", callback)

  #if TIMEvalue == true
    bind(bOK, "command", callbackLIMIT)
    bind(bOK, "<Return>", callbackLIMIT)
    callback_add(bOK, callbackLIMIT)
  #end

end

function callbackq(path)
  destroy(w)
end

function callbackn(path)
  x = Toplevel("New Game", 400, 300)
  tcl("pack", "propagate", x, false) ## or pack_stop_propagate(w)

  mb = Menu(x)

  nVSAI = Button(x, "• Start a game against the AI")
  pack(nVSAI, expand=true, fill="both")
  callback_add(nVSAI, callbacknVSAIOPT)

  nVSH = Button(x, "• Start a game against a human on the same computer")
  pack(nVSH, expand=true, fill="both")
  callback_add(nVSH, callbacknVSHOPT)

  nJOIN = Button(x, "• Join a game against a remote program")
  pack(nJOIN, expand=true, fill="both")
  #join a game(networking)
  callback_add(nJOIN, callback)
  #callback_add(nJOIN, callbacknJOIN)

  nHAI = Button(x, "• Host a game, using your AI as the player")
  pack(nHAI, expand=true, fill="both")
  callback_add(nHAI, callbacknHAIOPT)

  nHH = Button(x, "• Host a game, with a human as the player")
  pack(nHH, expand=true, fill="both")
  callback_add(nHH, callbacknHHOPT)

  nE = Button(x, "• Start a new game over email")
  pack(nE, expand=true, fill="both")
  callback_add(nE, callbacknEOPT)
#=
As play by email, I meant: Produce a game file that the other side would read.
Your program does not do the sending. Just create a game file database
and tell the user where it is. The user does the emailing.
=#

end

function callbacko(path)
  x = Toplevel("New Game", 400, 300)
  tcl("pack", "propagate", x, false) ## or pack_stop_propagate(w)

  mb = Menu(x)

  oLOCAL = Button(x, "• Continue a local game")
  pack(oLOCAL, expand=true, fill="both")
  #open local game
  callback_add(oLOCAL, callback)

  oEMAIL = Button(x, "• • Take a turn in an email game")
  pack(oEMAIL, expand=true, fill="both")
  #open email game
  callback_add(oEMAIL, callback)

end

function callbackr(path)
  a=1
  #open saved file
end

callback_add(n, callbackn)   ## generic way to add callback for most common event
callback_add(o, callbacko)
callback_add(r, callbackr)
callback_add(q, callbackq)

#=

TheTimeLimit = 0
if  TIMEvalue == true
  if parse(Int64,LIMITvalue) >= 1
    TheTimeLimit = parse(Int64,LIMITvalue)
    println("the time limit is :$TheTimeLimit")
  end
end

JRM = false
if  JRMvalue == true
    JRM = JRMvalue
    println("japanese roulette mode? :$JRM")
end

shvar = " "
if  VARvalue != "no shogi"
    shvar = VARvalue
    println("shogi variant :$shvar")
else
  shvar = "yes shogi"
  println("shogi variant :$shvar")
end

dfvar = " "
if  DIFFvalue != "not normal"
    dfvar = DIFFvalue
    println("difficulty :$dfvar")
else
  dfvar = "normal"
  println("difficulty :$dfvar")
end

CHEAT = false
if  CHEATvalue == true
    CHEAT = CHEATvalue
    println("cheating? :$CHEAT")
end

FIRST = false
if  FIRSTvalue == true
    FIRST = FIRSTvalue
    println("going first? :$FIRST")
end

=#

#send extracted data to gamefile

max_stars_repo_name: kinoroy/Shogi

max_issues_repo_name: kinoroy/Shulia

max_forks_repo_name: kinoroy/Shulia


text:
"""
    index_map, num_segments = felzenszwalb(edges, num_vertices, k, min_size=0)

Segment an image represented as Region Adjacency Graph(RAG) using Felzenszwalb's segmentation algorithm. Each pixel/region
corresponds to a node in the graph and weights on each edge measure the dissimilarity between pixels.
The function returns the number of segments and index mapping from nodes of the RAG to segments.

Parameters:
- `edges`:        Array of edges in RAG. Each edge is represented as `ImageEdge`.
- `num_vertices`: Number of vertices in RAG
- `k`:            Threshold for region merging step. Larger threshold will result in bigger segments.
- `min_size`:     Minimum segment size (in # pixels)
"""
function felzenszwalb(edges::Array{ImageEdge}, num_vertices::Int, k::Float64, min_size::Int = 0)

    num_edges = length(edges)
    G = IntDisjointSets(num_vertices)
    set_size = ones(num_vertices)
    threshold = fill(convert(Float64,k), num_vertices)

    sort!(edges, lt = (x,y)->(x.weight<y.weight))

    for edge in edges
        w = edge.weight
        a = find_root!(G, edge.index1)
        b = find_root!(G, edge.index2)
        if a!=b
            if w <= min(threshold[a], threshold[b])
                merged_root = union!(G, a, b)
                set_size[merged_root] = set_size[a] + set_size[b]
                threshold[merged_root] = w + k/set_size[merged_root]
            end
        end
    end

    #merge small segments
    for edge in edges
        a = find_root!(G, edge.index1)
        b = find_root!(G, edge.index2)
        if a!=b && (set_size[a] < min_size || set_size[b] < min_size)
            union!(G, a, b)
        end
    end

    segments = OrderedSet{Int}()
    for i in 1:num_vertices
        push!(segments, find_root!(G, i))
    end

    num_sets = length(segments)
    segments2index = Dict{Int, Int}()
    for (i, s) in enumerate(segments)
        segments2index[s] = i
    end

    index_map = Vector{Int}(undef, num_vertices)
    for i in 1:num_vertices
        index_map[i] = segments2index[find_root!(G, i)]
    end

    return index_map, num_sets
end
felzenszwalb(edges::Array{ImageEdge}, num_vertices::Integer, k::Real, min_size::Integer = 0) =
    felzenszwalb(edges, convert(Int, num_vertices)::Int, convert(Float64, k)::Float64, convert(Int, min_size)::Int)

meantype(::Type{T}) where T = typeof(zero(accum_type(T))/2)

"""
    segments = felzenszwalb(img, k, [min_size])

Segment an image using Felzenszwalb's segmentation algorithm and returns the result as `SegmentedImage`.
The algorithm uses euclidean distance in color space as edge weights for the region adjacency graph.

Parameters:
- `img`:      input image
- `k`:        Threshold for region merging step. Larger threshold will result in bigger segments.
- `min_size`: Minimum segment size (in # pixels)
"""
function felzenszwalb(img::AbstractArray{T}, k::Real, min_size::Int = 0) where T<:Union{Real,Color}

    sz = size(img)
    num_vertices = prod(sz)

    R = CartesianIndices(img)
    L = LinearIndices(img)
    Ibegin, Iend = first(R), last(R)
    I1 = _oneunit(Ibegin)

    # Compute the number of entries per pixel (other than at the image edges)
    num_edges = 0
    for I in _colon(-I1, I1)
        I >= zero(I1) && continue
        num_edges += 1
    end
    num_edges *= num_vertices   # now the number for the whole image
    edges = Vector{ImageEdge}(undef, num_edges)

    num = 0
    for I in R
        imgI = img[I]
        for J in _colon(max(Ibegin, I-I1), min(Iend, I+I1))
            if I >= J
                continue
            end
            edges[num+=1] = ImageEdge(L[I], L[J], sqrt(_abs2(imgI-meantype(T)(img[J]))))
        end
    end
    deleteat!(edges, num+1:num_edges)   # compensate for the ones we were missing at the image edges

    index_map, num_segments = felzenszwalb(edges, num_vertices, k, min_size)

    result              = similar(img, Int)
    labels              = Array(1:num_segments)
    region_means        = Dict{Int, meantype(T)}()
    region_pix_count    = Dict{Int, Int}()

    for I in R
        result[I] = index_map[L[I]]
        region_pix_count[result[I]] = get(region_pix_count, result[I], 0) + 1
        region_means[result[I]] = get(region_means, result[I], zero(meantype(T))) + (img[I] - get(region_means, result[I], zero(meantype(T))))/region_pix_count[result[I]]
    end

    return SegmentedImage(result, labels, region_means, region_pix_count)
end

ext: jl


text:
using SQLite

@testset "SQLite DBUtils" begin

    conn = SQLite.DB()
    PubMed.create_tables!(conn)

    #check collection of tables
    tables_query = DBUtils.select_all_tables(conn)
    tables = ["basic","author_ref","mesh_desc","mesh_heading","mesh_qual","pub_type","abstract_structured","abstract_full","file_meta", "sqlite_sequence"]
    @test sort(tables) == sort(tables_query)

    #check minimum insert
    DBUtils.insert_row!(conn, "basic", Dict(:pmid => 1234, :title=>"Test Article", :pub_year=>nothing))

    sel = DBUtils.db_select(conn, ["pmid"], "basic", Dict(:title=>"Test Article", :pmid=>1234))

    @test length(sel[1]) == 1
    @test sel[1][1] == 1234
end


text:
module VaderSentiment
export VaderSentiment
include("vader_sentiment.jl")
end #module

max_stars_repo_name: nusretipek/JuliaVader

max_issues_repo_name: nusretipek/JuliaVader


text:
using Pkg

Pkg.activate(@__DIR__)
Pkg.update()
Pkg.instantiate()

using Literate

if isempty(ARGS)
    flist = ["sphere_scattering.jl", "singular_integrals.jl"]
else
    flist = ARGS
end

for file in flist
    fname =  joinpath(@__DIR__,file)
    Literate.markdown(fname,@__DIR__,documenter=true)
    # Literate.notebook(fname,@__DIR__;documenter=true,execute=false)
end

max_stars_repo_name: WaveProp/WaveProp

max_issues_repo_name: WaveProp/WaveProp

max_forks_repo_name: WaveProp/WaveProp


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

# weak key dictionaries

"""
    WeakKeyDict([itr])

`WeakKeyDict()` constructs a hash table where the keys are weak
references to objects which may be garbage collected even when
referenced in a hash table.

See [`Dict`](@ref) for further help.  Note, unlike [`Dict`](@ref),
`WeakKeyDict` does not convert keys on insertion, as this would imply the key
object was unreferenced anywhere before insertion.
"""
mutable struct WeakKeyDict{K,V} <: AbstractDict{K,V}
    ht::Dict{WeakRef,V}
    lock::ReentrantLock
    finalizer::Function
    dirty::Bool

    # Constructors mirror Dict's
    function WeakKeyDict{K,V}() where V where K
        t = new(Dict{Any,V}(), ReentrantLock(), identity, 0)
        t.finalizer = k -> t.dirty = true
        return t
    end
end
function WeakKeyDict{K,V}(kv) where V where K
    h = WeakKeyDict{K,V}()
    for (k,v) in kv
        h[k] = v
    end
    return h
end
WeakKeyDict{K,V}(p::Pair) where V where K = setindex!(WeakKeyDict{K,V}(), p.second, p.first)
function WeakKeyDict{K,V}(ps::Pair...) where V where K
    h = WeakKeyDict{K,V}()
    sizehint!(h, length(ps))
    for p in ps
        h[p.first] = p.second
    end
    return h
end
WeakKeyDict() = WeakKeyDict{Any,Any}()

WeakKeyDict(kv::Tuple{}) = WeakKeyDict()
copy(d::WeakKeyDict) = WeakKeyDict(d)

WeakKeyDict(ps::Pair{K,V}...)           where {K,V} = WeakKeyDict{K,V}(ps)
WeakKeyDict(ps::Pair{K}...)             where {K}   = WeakKeyDict{K,Any}(ps)
WeakKeyDict(ps::(Pair{K,V} where K)...) where {V}   = WeakKeyDict{Any,V}(ps)
WeakKeyDict(ps::Pair...)                            = WeakKeyDict{Any,Any}(ps)

function WeakKeyDict(kv)
    try
        Base.dict_with_eltype((K, V) -> WeakKeyDict{K, V}, kv, eltype(kv))
    catch
        if !isiterable(typeof(kv)) || !all(x->isa(x,Union{Tuple,Pair}),kv)
            throw(ArgumentError("WeakKeyDict(kv): kv needs to be an iterator of tuples or pairs"))
        else
            rethrow()
        end
    end
end

function _cleanup_locked(h::WeakKeyDict)
    if h.dirty
        h.dirty = false
        idx = skip_deleted_floor!(h.ht)
        while idx != 0
            if h.ht.keys[idx].value === nothing
                _delete!(h.ht, idx)
            end
            idx = skip_deleted(h.ht, idx + 1)
        end
    end
    return h
end

sizehint!(d::WeakKeyDict, newsz) = sizehint!(d.ht, newsz)
empty(d::WeakKeyDict, ::Type{K}, ::Type{V}) where {K, V} = WeakKeyDict{K, V}()

IteratorSize(::Type{<:WeakKeyDict}) = SizeUnknown()

islocked(wkh::WeakKeyDict) = islocked(wkh.lock)
lock(wkh::WeakKeyDict) = lock(wkh.lock)
unlock(wkh::WeakKeyDict) = unlock(wkh.lock)
lock(f, wkh::WeakKeyDict) = lock(f, wkh.lock)
trylock(f, wkh::WeakKeyDict) = trylock(f, wkh.lock)

function setindex!(wkh::WeakKeyDict{K}, v, key) where K
    !isa(key, K) && throw(ArgumentError("$(limitrepr(key)) is not a valid key for type $K"))
    # 'nothing' is not valid both because 'finalizer' will reject it,
    # and because we therefore use it as a sentinel value
    key === nothing && throw(ArgumentError("`nothing` is not a valid WeakKeyDict key"))
    lock(wkh) do
        _cleanup_locked(wkh)
        k = getkey(wkh.ht, key, nothing)
        if k === nothing
            finalizer(wkh.finalizer, key)
            k = WeakRef(key)
        else
            k.value = key
        end
        wkh.ht[k] = v
    end
    return wkh
end
function get!(wkh::WeakKeyDict{K}, key, default) where {K}
    v = lock(wkh) do
        if key !== nothing && haskey(wkh.ht, key)
            wkh.ht[key]
        else
            wkh[key] = default
        end
    end
    return v
end
function get!(default::Callable, wkh::WeakKeyDict{K}, key) where {K}
    v = lock(wkh) do
        if key !== nothing && haskey(wkh.ht, key)
            wkh.ht[key]
        else
            wkh[key] = default()
        end
    end
    return v
end

function getkey(wkh::WeakKeyDict{K}, kk, default) where K
    k = lock(wkh) do
        k = getkey(wkh.ht, kk, nothing)
        k === nothing && return nothing
        return k.value
    end
    return k === nothing ? default : k::K
end

map!(f, iter::ValueIterator{<:WeakKeyDict})= map!(f, values(iter.dict.ht))

function get(wkh::WeakKeyDict{K}, key, default) where {K}
    key === nothing && throw(KeyError(nothing))
    lock(wkh) do
        return get(wkh.ht, key, default)
    end
end
function get(default::Callable, wkh::WeakKeyDict{K}, key) where {K}
    key === nothing && throw(KeyError(nothing))
    lock(wkh) do
        return get(default, wkh.ht, key)
    end
end
function pop!(wkh::WeakKeyDict{K}, key) where {K}
    key === nothing && throw(KeyError(nothing))
    lock(wkh) do
        return pop!(wkh.ht, key)
    end
end
function pop!(wkh::WeakKeyDict{K}, key, default) where {K}
    key === nothing && return default
    lock(wkh) do
        return pop!(wkh.ht, key, default)
    end
end
function delete!(wkh::WeakKeyDict, key)
    key === nothing && return wkh
    lock(wkh) do
        delete!(wkh.ht, key)
    end
    return wkh
end
function empty!(wkh::WeakKeyDict)
    lock(wkh) do
        empty!(wkh.ht)
    end
    return wkh
end
function haskey(wkh::WeakKeyDict{K}, key) where {K}
    key === nothing && return false
    lock(wkh) do
        return haskey(wkh.ht, key)
    end
end
function getindex(wkh::WeakKeyDict{K}, key) where {K}
    key === nothing && throw(KeyError(nothing))
    lock(wkh) do
        return getindex(wkh.ht, key)
    end
end
isempty(wkh::WeakKeyDict) = length(wkh) == 0
function length(t::WeakKeyDict)
    lock(t) do
        _cleanup_locked(t)
        return length(t.ht)
    end
end

function iterate(t::WeakKeyDict{K,V}, state...) where {K, V}
    return lock(t) do
        while true
            y = iterate(t.ht, state...)
            y === nothing && return nothing
            wkv, state = y
            k = wkv[1].value
            GC.safepoint() # ensure `k` is now gc-rooted
            k === nothing && continue # indicates `k` is scheduled for deletion
            kv = Pair{K,V}(k::K, wkv[2])
            return (kv, state)
        end
    end
end

filter!(f, d::WeakKeyDict) = filter_in_one_pass!(f, d)

max_stars_repo_name: jonas-schulze/julia

max_issues_repo_name: jonas-schulze/julia

max_forks_repo_name: jonas-schulze/julia


text:
export          OrthoPoly,
                MultiOrthoPoly,
                LegendreOrthoPoly,
                JacobiOrthoPoly,
                LaguerreOrthoPoly,
                genLaguerreOrthoPoly,
                HermiteOrthoPoly,
                genHermiteOrthoPoly,
                GaussOrthoPoly,
                Beta01OrthoPoly,
                GammaOrthoPoly,
                LogisticOrthoPoly,
                Uniform01OrthoPoly,
                Uniform_11OrthoPoly,
                MeixnerPollaczekOrthoPoly,
                InconsistencyError,
                Quad

struct InconsistencyError <: Exception
    var::String
end

Base.showerror(io::IO, err::InconsistencyError) = print(io, err.var)

function _checkConsistency(deg::Int, Nrec::Int)
    deg < 0 && throw(DomainError(deg, "degree has to be non-negative"))
    Nrec < deg + 1 && throw(DomainError(Nrec, "not enough recurrence coefficients specified (need >= $(deg + 1))"))
end

struct OrthoPoly{V<:AbstractVector{<:Real}, M, Q} <: AbstractOrthoPoly{M, Q}
    name::String
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q
    # inner constructor
    function OrthoPoly(name::String, deg::Int, α::AbstractVector{<:Real}, β::AbstractVector{<:Real}, measure::AbstractMeasure; addQuadrature::Bool = true)
        deg < 0 && throw(DomainError(deg, "degree has to be non-negative"))
        !(length(α) == length(β)) && throw(InconsistencyError("Inconsistent lengths"))
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        # @show types = promote_type(eltype(α), eltype(β)), promote_type(typeof(α), typeof(β)), typeof(measure), typeof(quadrature)
        new{promote_type(typeof(α), typeof(β)), typeof(measure), typeof(quadrature)}(lowercase(name), deg, α, β, measure, quadrature)
    end
end

# constructor for known Measure
function OrthoPoly(name::String, deg::Int, measure::AbstractMeasure; Nrec=deg+1, Nquad=10*Nrec, quadrature::Function=clenshaw_curtis, discretization::Function=stieltjes, addQuadrature::Bool=true)
    _checkConsistency(deg, Nrec)
    name = lowercase(name)
    α, β = rm_compute(measure, Nrec, Nquad, quadrature=quadrature, discretization=discretization)
    return OrthoPoly(name, deg, α, β, measure, addQuadrature=addQuadrature)
end

# general constructor
function OrthoPoly(name::String, deg::Int, w::Function, s::Tuple{<:Real,<:Real}, symm::Bool, d::Dict=Dict(); Nrec=deg+1, Nquad=10*Nrec, quadrature::Function=clenshaw_curtis, discretization::Function=stieltjes, addQuadrature::Bool = true)
  name = lowercase(name)
  measure = Measure(name, w, s, symm, d)
  OrthoPoly(name, deg, measure; Nrec=Nrec, Nquad=Nquad, quadrature=quadrature, discretization=discretization, addQuadrature=addQuadrature)
end

##############################################

struct LegendreOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function LegendreOrthoPoly(deg::Int; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_legendre(Nrec)
        quadrature = addQuadrature ?  Quad(length(α)-1, α, β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), LegendreMeasure, typeof(quadrature)}(deg, α, β, LegendreMeasure(), quadrature)
    end
end

struct JacobiOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int            # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function JacobiOrthoPoly(deg::Int, shape_a::Real, shape_b::Real; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_jacobi(Nrec, shape_a, shape_b)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), JacobiMeasure, typeof(quadrature)}(deg, α, β, JacobiMeasure(shape_a, shape_b), quadrature)
    end
end

struct LaguerreOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function LaguerreOrthoPoly(deg::Int;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_laguerre(Nrec)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), LaguerreMeasure, typeof(quadrature)}(deg, α, β, LaguerreMeasure(), quadrature)
    end
end

struct genLaguerreOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function genLaguerreOrthoPoly(deg::Int, shape::Real ;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_laguerre(Nrec, shape)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), genLaguerreMeasure, typeof(quadrature)}(deg, α, β, genLaguerreMeasure(shape), quadrature)
    end
end

struct HermiteOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function HermiteOrthoPoly(deg::Int; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_hermite(Nrec)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), HermiteMeasure, typeof(quadrature)}(deg, α, β, HermiteMeasure(), quadrature)
    end
end

struct genHermiteOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function genHermiteOrthoPoly(deg::Int, mu::Real;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_hermite(Nrec, mu)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), genHermiteMeasure, typeof(quadrature)}(deg, α, β, genHermiteMeasure(mu), quadrature)
    end
end

struct MeixnerPollaczekOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function MeixnerPollaczekOrthoPoly(deg::Int, λ::Real, ϕ::Real; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = rm_meixner_pollaczek(Nrec, λ, ϕ)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), MeixnerPollaczekMeasure, typeof(quadrature)}(deg, α, β, MeixnerPollaczekMeasure(λ,ϕ), quadrature)
    end
end

struct GaussOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function GaussOrthoPoly(deg::Int;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale(1/sqrt(2pi), rm_hermite_prob(Nrec)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), GaussMeasure, typeof(quadrature)}(deg, α, β, GaussMeasure(), quadrature)
    end
end

struct Uniform01OrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function Uniform01OrthoPoly(deg::Int;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale(1., rm_legendre01(Nrec)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), Uniform01Measure, typeof(quadrature)}(deg, α, β, Uniform01Measure(), quadrature)
    end
end

struct Uniform_11OrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V  # recurrence coefficients
    β::V  # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function Uniform_11OrthoPoly(deg::Int;Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale(0.5, rm_legendre(Nrec)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), Uniform_11Measure, typeof(quadrature)}(deg, α, β, Uniform_11Measure(), quadrature)
    end
end

struct Beta01OrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function Beta01OrthoPoly(deg::Int, shape_a::Real, shape_b::Real; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale(1/beta(shape_a, shape_b), rm_jacobi01(Nrec, shape_b-1., shape_a-1.)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), Beta01Measure, typeof(quadrature)}(deg, α, β, Beta01Measure(shape_a, shape_b), quadrature)
    end
end

struct GammaOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function GammaOrthoPoly(deg::Int, shape::Real, rate::Real; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale((rate^shape)/gamma(shape), rm_laguerre(Nrec, shape-1.)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), GammaMeasure, typeof(quadrature)}(deg, α, β, GammaMeasure(shape, rate), quadrature)
    end
end

struct LogisticOrthoPoly{V, M, Q} <: AbstractCanonicalOrthoPoly{V, M, Q}
    deg::Int          # maximum degree
    α::V # recurrence coefficients
    β::V # recurrence coefficients
    measure::M
    quad::Q

    # inner constructor
    function LogisticOrthoPoly(deg::Int; Nrec::Int=deg+1, addQuadrature::Bool = true)
        _checkConsistency(deg, Nrec)
        α, β = r_scale(1., rm_logistic(Nrec)...)
        quadrature = addQuadrature ?  Quad(length(α)-1,α,β) : EmptyQuad()
        new{promote_type(typeof(α), typeof(β)), LogisticMeasure, typeof(quadrature)}(deg, α, β, LogisticMeasure(), quadrature)
    end
end

# #####################################################
# #####################################################
# #####################################################

struct MultiOrthoPoly{M, Q, V<:AbstractVector} <: AbstractOrthoPoly{M, Q}
name::Vector{String}
deg::Int
dim::Int
ind::Matrix{Int} # multi-index
measure::ProductMeasure
uni::V
    function MultiOrthoPoly(uniOrthoPolys, deg::Int)
      degs = [ op.deg for op in uniOrthoPolys ]
      deg > minimum(degs) && throw(DomainError(deg, "Requested degree $deg is greater than smallest univariate degree $(minimum(degs))."))

      w(t) = prod([op.measure.w(t) for op in uniOrthoPolys])
      measures = [ op.measure for op in uniOrthoPolys ]
      measure = ProductMeasure(w, measures)

      names =
        if VERSION >= v"1.2.0"
            [ hasfield(typeof(op), :name) ? op.name : string(typeof(op)) for op in uniOrthoPolys ]
        else
            [ :name in fieldnames(typeof(op)) ? op.name : string(typeof(op)) for op in uniOrthoPolys ]
        end
      ind = calculateMultiIndices(length(uniOrthoPolys), deg)
      dim = size(ind,1)

      new{typeof(measure), typeof(first(uniOrthoPolys).quad), typeof(uniOrthoPolys)}(names, deg, dim, ind, measure, uniOrthoPolys)
    end
end

function _hasfield(op::AbstractOrthoPoly, name::Symbol)
    name in fieldnames(op)
end


text:
export OneDGrid

struct OneDGrid

    dev :: AbstractDevice
    len  :: Int
    start :: Real
    stop  :: Real
    points :: LinRange
    step :: Real

    function OneDGrid( dev, len, start, stop )

        points = LinRange( start, stop, len+1 )[1:end-1]
        step = ( stop - start ) / len
        new( dev, len, start, stop, points, step)

    end

end

max_stars_repo_name: JuliaVlasov/VlasovSolvers

max_issues_repo_name: JuliaVlasov/VlasovSolvers

max_forks_repo_name: JuliaVlasov/VlasovSolvers


text:
using Abaco
using Test

Abaco.loginit()

# My time zone: Saturday, October 30, 2021 3:43:35 PM GMT+02:00 DST
window = 900
metric_ts = 1635600600
metric_sn = "Civetta"

expected_triggers = 0
actual_triggers = 0

abaco = abaco_init(interval=window) do ts, ne, name, value, inputs
    global actual_triggers
    @info "[identity] age [$ts]: scope: [$ne] $name = $value"
    @test ts == metric_ts
    @test ne == metric_sn
    @test value == 100.0
    actual_triggers += 1
end

invalid_formula = "r = rm(\"foo.txt\")"
@test_throws Abaco.WrongFormula(invalid_formula) formula(abaco, invalid_formula)

@test actual_triggers === expected_triggers


text:
# When building data structures for use with interactive data manipuluation,
# it is often handy to have data structures that can be efficiently indexed
# by position (like a vector) as well as readably indexed by name (like a
# hash table). The NamedIndex type here maps unique names to integer
# indexes, looking up ByteString indexes in a Dict and passing other
# index types through. The NamedVector type implements a simple use case,
# with a backing vector of an arbitrary type.

module Named

using Base
#import Base.*
import Base.length, Base.isempty, Base.names, Base.copy, Base.has, Base.assign,
    Base.keys, Base.values, Base.push, Base.del, Base.ref, Base.select,
    Base.start, Base.done, Base.next, Base.show, Base.size

export NamedIndex, SimpleIndex, NamedVector,
	length, isempty, names, copy, names!, replace_names!,
	replace_names, has, assign, keys, values, push, del, ref,
	select, select_kv,
	set_group, set_groups, get_group, get_groups, isgroup,
	start, done, next, show, size

# should this go elsewhere?
promote_rule(::Type{Union(UTF8String,ASCIIString)}, ::Type{ASCIIString} ) = Union(UTF8String,ASCIIString)

# an AbstractIndex is a thing that can be used to look up ordered things by name, but that
# will also accept a position or set of positions or range or other things and pass them
# through cleanly.
# an NamedIndex is the usual implementation, with ByteString names.
# a SimpleIndex only works if the things are integer indexes, which is weird.
abstract AbstractIndex

type NamedIndex <: AbstractIndex   # an OrderedDict would be nice here...
    lookup::Dict{ByteString,Indices}      # name => names array position
	groups::Dict  					  	# name => [other names]
    names::Vector{ByteString}
end

function NamedIndex{T<:ByteString}(x::Vector{T})
	NamedIndex(Dict{ByteString, Indices}(tuple(x...), tuple([1:length(x)]...)),
			   Dict(),
               make_unique(convert(Vector{ByteString}, x)))
end
NamedIndex() = NamedIndex(Dict{ByteString,Indices}(), Dict(), ByteString[])

length(x::NamedIndex) = length(x.names)
names(x::NamedIndex) = copy(x.names)
copy(x::NamedIndex) = NamedIndex(copy(x.lookup), copy(x.groups), copy(x.names))

# note: replacing names removes any groups no longer present
function names!(x::NamedIndex, nm::Vector)
    if length(nm) != length(x)
        error("lengths don't match.")
    end
    for i in 1:length(nm)
        del(x.lookup, x.names[i])
        x.lookup[nm[i]] = i
    end
    x.names = nm
	clean_groups!(x, nm)
	x.names
end

function clean_groups!(x::NamedIndex, nm::Vector)
	for gr_key in keys(x.groups)
		new_val = intersect(x.groups[gr_key], nm)
		if length(new_val) == 0
			del(x.groups, gr_key)
		else
			x.groups[gr_key] = new_val
		end
	end
end

function replace_names!(x::NamedIndex, indexes::Vector{Int}, new_names::Vector)
    if length(indexes) != length(new_names)
        error("lengths of indexes and new_names don't match.")
    end
	# it's tricky to figure out if we're ending up with dupes without trying it...
	# for now, copy twice to avoid clobbering the caller if we end up failing
	newx = copy(x)
    for i = 1:length(indexes)
		# remove the old lookup
		del(newx.lookup, newx.names[i])
		# add the new lookup
		newx.lookup[new_names[i]] = indexes[i]
		# replace the name
		newx.names[indexes[i]] = new_names[i]
    end
	# make sure we didn't screw anything up
	if length(keys(x.lookup)) != length(x.names)
		error("replacement failed -- ended up with duplicate names")
	end
	x.names = newx.names
	x.lookup = newx.lookup
	clean_groups!(x, x.names)
    x.names
end
function replace_names!(x::NamedIndex, from::Vector, to::Vector)
    if length(from) != length(to)
        error("lengths of from and to don't match.")
    end
    for idx in 1:length(from)
        if has(x, from[idx]) && !has(x, to[idx])
            x.lookup[to[idx]] = x.lookup[from[idx]]
            x.names[x.lookup[from[idx]]] = to[idx]
            del(x.lookup, from[idx])
        end
    end
	clean_groups!(x, x.names)
    x.names
end
replace_names!(x::NamedIndex, from, to) = replace_names!(x, [from], [to])
replace_names(x::NamedIndex, from, to) = replace_names!(copy(x), from, to)

# groups work with ref(), but not has(), keys(), etc
has(x::NamedIndex, key) = has(x.lookup, key)
keys(x::NamedIndex) = names(x)
function push(x::NamedIndex, nm)
    x.lookup[nm] = length(x) + 1
    push(x.names, nm)
end
function del(x::NamedIndex, idx::Integer)
    # reset the lookup's beyond the deleted item
    for i in idx+1:length(x.names)
        x.lookup[x.names[i]] = i - 1
    end
    del(x.lookup, x.names[idx])
    del(x.names, idx)
    # fix groups
	clean_groups!(x, x.names)
end
function del(x::NamedIndex, nm)
    if !has(x.lookup, nm)
        return
    end
    idx = x.lookup[nm]
    del(x, idx)
end

# ref should properly deal with the new groups
function ref{T<:ByteString}(x::NamedIndex, idx::Vector{T})
	# expand any groups, then do the comprehension
	idx2 = T[]
	for i in idx
		if isgroup(x, i)
			push(idx2, get_group(x, i))
		else
			push(idx2, i)
		end
	end
	[[x.lookup[i] for i in idx]...]
end
function ref{T<:ByteString}(x::NamedIndex, idx::T)
	if isgroup(x, idx)
		ref(x, get_group(x, idx))
	else
		x.lookup[idx]
	end
end

# if we get a symbol or a vector of symbols, convert to strings for ref
ref(x::NamedIndex, idx::Symbol) = x[string(idx)]
ref(x::NamedIndex, idx::Vector{Symbol}) = [[x.lookup[string(i)] for i in idx]...]

# fall-throughs, when something other than the index type is passed
ref(x::AbstractIndex, idx::Int) = idx
ref(x::AbstractIndex, idx::Vector{Int}) = idx
ref(x::AbstractIndex, idx::Range{Int}) = [idx]
ref(x::AbstractIndex, idx::Range1{Int}) = [idx]
ref(x::AbstractIndex, idx::Vector{Bool}) = [1:length(x)][idx]
#ref(x::AbstractIndex, idx::AbstractDataVec{Bool}) = x[nareplace(idx, false)]
#ref(x::AbstractIndex, idx::AbstractDataVec{Int}) = x[nafilter(idx)]

type SimpleIndex <: AbstractIndex
    length::Integer
end
SimpleIndex() = SimpleIndex(0)
length(x::SimpleIndex) = x.length
names(x::SimpleIndex) = nothing

# Chris DuBois' idea: named
function set_group(idx::NamedIndex, newgroup, names)
	# confirm that the names exist
	for name in names
		if !has(idx, name)
			error("can't add group referring to non-existent name!")
		end
	end
	# add the group
	idx.groups[newgroup] = Set(names...)
end
function set_groups(idx::NamedIndex, gr::Dict)  # {ByteString,Vector{ByteString}}) breaks; duck type
    for (k,v) in gr
        set_group(idx, k, v)
    end
end
get_group(idx::NamedIndex, name) = elements(idx.groups[name]) # set -> array
get_groups(idx::NamedIndex) = idx.groups # returns a dict to sets, which may not be what you want
isgroup(idx::NamedIndex, name::ByteString) = has(idx.groups, name)

function show(io, idx::NamedIndex)
    println(io, "$(length(idx))-element NamedIndex")
    pretty_show(io, idx.groups)
    for i = 1:min(length(idx), 9)
        println(io, "$i = $(names(idx)[i])")
    end
    if length(idx) > 9
        println(io, "...")
    end
end

# special pretty-printer for groups, which are just Dicts.
function pretty_show(io, gr)
    allkeys = keys(gr)
    for k = allkeys
        print(io, "$(k): ")
        print(io, join(gr[k], ", "))
        if k == last(allkeys)
            println(io)
        else
            print(io, "; ")
        end
    end
end

############ NamedVector #############

type NamedVector{V} <: AbstractVector{V}
    idx::NamedIndex
    arr::Vector{V}
	#NamedVector() = new(NamedIndex(), Array(V,0))
end
NamedVector() = NamedVector(NamedIndex(), Array(Any,0))
NamedVector(T::Type) = NamedVector(NamedIndex(), Array(T,0))
#NamedVector(i::NamedIndex, a::Vector) = NamedVector(i,a)

function NamedVector{K<:ByteString,V}(a::Associative{K,V})
	ret = NamedVector(V)
	for k in keys(a)
		ret[k] = a[k]
	end
	ret
end

function NamedVector{K<:ByteString,V}(keys::Vector{K}, values::Vector{V})
	ret = NamedVector(V)
	for i = 1:length(keys)
		ret[keys[i]] = values[i]
	end
	ret
end

# similar's a little funny -- we make a similar array, but if the new vector's
# longer than what we started with, the new keys are weird
function similar(nv::NamedVector, element_type, dims)
    new_arr = similar(nv.arr, element_type, dims)
    if dims == size(nv)
        new_idx = copy(nv.idx)
    elseif dims < size(nv)
        new_idx = NamedIndex(names(nv.idx)[1:dims[1]])
    else
        new_names = vcat(names(nv.idx), fill("X", dims[1]-length(nv)))
        new_names = make_unique(new_names)
        new_idx = NamedIndex(new_names)
    end
    NamedVector(new_idx, new_arr)
end

# assignment by a string replaces or appends
function assign(id::NamedVector, v, key::ByteString)
    if has(id.idx, key)
        id.arr[id.idx[key]] = v
    else
        push(id.arr, v)
        push(id.idx, key)
    end
end
assign(id::NamedVector, v, key::Symbol) = assign(id, v, string(key))
# assignment by an integer replaces or throws an error
assign(id::NamedVector, v, pos::Integer) = id.arr[pos] = v
assign(id::NamedVector, v, pos::Real) = assign(id, v, iround(pos))
assign(id::NamedVector, v, pos) = id.arr[pos] = v

ref{V,T<:Integer}(id::NamedVector{V}, ii::AbstractArray{T,1}) = id.arr[id.idx[ii]]
ref(id::NamedVector, i::Integer) = id.arr[id.idx[i]]
ref(id::NamedVector, i::Real) = ref(id, iround(i))
ref(id::NamedVector, i) = id.arr[id.idx[i]]
# ref gives the vector value; find gives the _position_ in the underlying NamedIndex
find(id::NamedVector, v) = has(id, v) ? id.idx[v] : 0

function get{K}(id::NamedVector{K}, key, deflt)
    try
        id[key]
    catch e
        deflt
    end
end

size(nv::NamedVector) = size(nv.arr)
names(nv::NamedVector) = names(nv.idx)
keys(nv::NamedVector) = names(nv)
values(nv::NamedVector) = nv.arr
select(nv::NamedVector, r::Int64) = nv[r]
select(nv::NamedVector, r) = nv[r]
select_kv(nv::NamedVector, r) = (names(nv)[r], nv[r])

has(id::NamedVector, key) = has(id.idx, key)
isempty(id::NamedVector) = isempty(id.arr)
length(id::NamedVector) = length(id.arr)

start(id::NamedVector) = 1
done(id::NamedVector, i) = i > length(id)
next(id::NamedVector, i) = (id[i], i+1)

function show(io, id::NamedVector)
    n = names(id.idx)
    println(io, "$(length(id))-element $(eltype(id.arr)) NamedVector")
    pretty_show(io, id.idx.groups)
    for i = 1:min(length(id), 9)
        println(io, "$i, $(n[i]): $(id[i])")
    end
    if length(id) > 9
        println(io, "...")
    end
end
repl_show(io, id::NamedVector) = show(io, id::NamedVector)

# copying has to copy everything, because otherwise you can append an item,
# which can cause the vector to change length and break the original
copy(nv::NamedVector) = NamedVector(copy(nv.idx), copy(nv.arr))

function make_unique{S<:ByteString}(names::Vector{S})
    x = NamedIndex()
    names = copy(names)
    dups = Int[]
    for i in 1:length(names)
        if has(x, names[i])
            push(dups, i)
        else
            push(x, names[i])
        end
    end
    for i in dups
        nm = names[i]
        newnm = nm
        k = 1
        while true
            newnm = "$(nm)_$k"
            if !has(x, newnm)
                push(x, newnm)
                break
            end
            k += 1
        end
        names[i] = newnm
    end
    names
end

set_group(v::NamedVector, newgroup, names) = set_group(v.idx, newgroup, names)
set_groups(v::NamedVector, gr) = set_groups(v.idx, gr)

get_group(v::NamedVector, name) = get_group(v.idx, name)
get_groups(v::NamedVector) = get_groups(v.idx)
isgroup(v::NamedVector, name::ByteString) = isgroup(v.idx, name)

end #module


text:
using ArgParse

flagDebug=false; # Used to print extra information for debugging

argSettings = ArgParseSettings(description = string(
"This is a tool for systematically generating LSF jobs which write to log files. Below is an outline of the process.\n",
"\n",
"\n",
"\n",
"\n",
"\n  STAGE 1: Generate summary files for each set of input data.\n",
"\n",
"\n",
"\n    STAGE 1 INPUTS:\n",
"\nPath to protocol file (-p).\n",
"\n(Optional) Variables file supplied with the --vars (-r) option.\n",
"\n(Optional) Variables from lists file supplied with the --fvars (-f) option.\n",
"\n",
"\n",
"\n    STAGE 1 OUTPUTS:\n",
"\nSummary files generated by expanding the variables in the protocol file.\n",
"\nText file listing paths to the generated summary files.\n",
"\n",
"\n",
"\n",
"\n",
"\n   STAGE 2: Generate job files.\n",
"\n",
"\n",
"\n     STAGE 2 INPUTS:\n",
"\nText file listing the of paths to the generated summary files.  If stages 1 and 2 are run together the summary files from stage 1 will be used.\n",
"\n",
"\n",
"\n     STAGE 2 OUTPUTS:\n",
"\nJob files generated from summary files.\n",
"\nText file listing paths to the generated job files.\n",
"\n",
"\n",
"\n",
"\n",
"\n   STAGE 3: Submit jobs to the queuing system.\n",
"\n",
"\n",
"\n     STAGE 3 INPUTS:\n",
"\nText file listing paths to the generated job files.  If stages 2 and 3 are run together the job files from stage 2 will be used for stage 3.\n",
"\n",
"\n",
"\n     STAGE 3 OUTPUTS:\n",
"\nLog files written by the job files.\n",
"\nStarndard output and error output from the queuing system as well as any output produced by running the commands contained in the job files.",
"\n",
"\n",
"\n",
));
sourcePath = dirname(Base.source_path()) * "/"; # Get the path to the jsub.jl file

####### INPUTS #######

## Default job header values
# jobID="LSFjob";
# numberOfCores=1;
# numberOfHosts=1;
# wallTime="8:00";
# queue="normal"
# grantCode="prepay-houlston"

# jobHeader = string(
# "#!/bin/bash\n
# #BSUB -J \"$jobID\"\n
# #BSUB -n $numberOfCores\n
# #BSUB -R \"span[hosts=$numberOfHosts]\"\n
# #BSUB -P $grantCode\n
# #BSUB -W $wallTime\n
# #BSUB -q $queue\n
# #BSUB -o output.$jobID\n
# #BSUB -e error.$jobID\n"
# )

######################

## Hard coded variables
# First non-whitespace string indicating the start of a comment line
const comStr="#" # Note: this is expected to be a string ("#") rather than a character ('#').  Changing the string (char) used to indicate comments may cause problems further down the line.
# const dlmVars='\t' # Column delimiter for files containing variables
# const dlmProtocol=' ' # Column delimiter for the protocol file
const dlmWhitespace=[' ','\t','\n','\v','\f','\r'] # The default whitespace characters used by split
# const verbose = false;
const adapt_quotation=true; # this should be the default to avoid nasty accidents

num_suppressed = [0];

## Tags
tagsExpand = Dict(
  "header" => "#BSUB",
  "tagSummaryName" => "#JSUB<summary-name>",
  "tagSplit" => "#JGROUP",
  "tagJobName" => "#JSUB<job-id>"
)

## Paths to bash functions {"function name" => "path to file containing function"}
commonFunctions = Dict(
  "kill_this_job" => sourcePath * "common_functions/job_processing.sh",
  "process_job" => sourcePath * "common_functions/job_processing.sh",
  "version_control" => sourcePath * "common_functions/version_control.sh",
)
checkpointsDict = Dict(
  "jcheck_file_not_empty" => sourcePath * "common_functions/jcheck_file_not_empty.sh",
  "jcheck_checkpoint" => sourcePath * "common_functions/jcheck_checkpoint.sh",
)

bsubOptions = [
"-ar",
"-B",
"-H",
"-I", "-Ip", "-Is", # [-tty]
"-IS", "-ISp", "-ISs", "-IX", #[-tty]
"-K",
"-N",
"-r", "-rn",
"-ul",
"-a", "esub_application", # [([argument[,argument...]])]..."
"-app", # application_profile_name
"-b", # [[year:][month:]day:]hour:minute
"-C", # core_limit
"-c", # [hour:]minute[/host_name | /host_model]
"-clusters", # "all [~cluster_name] ... | cluster_name[+[pref_level]] ... [others[+[pref_level]]]"
"-cwd", # "current_working_directory"
"-D", # data_limit
"-E", # "pre_exec_command [arguments ...]"
"-Ep", # "post_exec_command [arguments ...]"
"-e", # error_file
"-eo", #error_file
"-ext", #[sched] "external_scheduler_options"
"-F", # file_limit
"-f", # local_file operator [remote_file]" ...
"-freq", # numberUnit
"-G", # user_group
"-g", # job_group_name
"-i", # input_file | -is input_file
"-J", # job_name | -J "job_name[index_list]%job_slot_limit"
"-Jd", # "job_description"
"-jsdl", # file_name | -jsdl_strict file_name
"-k", # "checkpoint_dir [init=initial_checkpoint_period][checkpoint_period] [method=method_name]"
"-L", # login_shell
"-Lp", # ls_project_name
"-M", # mem_limit
"-m", # "host_name[@cluster_name][[!] | +[pref_level]] | host_group[[!] | +[pref_level | compute_unit[[!] | +[pref_level]] ..."
"-mig", # migration_threshold
"-n", # min_proc[,max_proc]
"-network", # " network_res_req"
"-o", # output_file
"-oo", # output_file
"-outdir", # output_directory
"-P", # project_name
"-p", # process_limit
"-pack", # job_submission_file
"-Q", # "[exit_code ...] [EXCLUDE(exit_code ...)]"
"-q", # "queue_name ..."
"-R", # "res_req" [-R "res_req" ...]
"-rnc", # resize_notification_cmd
"-S", # stack_limit
"-s", # signal
"-sla", # service_class_name
"-sp", # priority
"-T", # thread_limit
"-t", # [[[year:]month:]day:]hour:minute
"-U", # reservation_ID
"-u", # mail_user
"-v", # swap_limit
"-W", # [hour:]minute[/host_name | /host_model]
"-We", # [hour:]minute[/host_name | /host_model]
"-w", # 'dependency_expression'
"-wa", # 'signal'
"-wt", # '[hour:]minute'
"-XF",
"-Zs",
"-h",
"-V",
];

#### FUNCTIONS FROM OTHER FIELS ####
include("./common_functions/jsub_common.jl")
####################################

######### MAIN #########
# function main(args)
## Argparse settings
@add_arg_table argSettings begin
  "-p", "--protocol"
    help = "Path to \"protocol\" file.  This file contains commands to be run with variables to be substituted using values from files supplied with \"--vars (-r)\" and/or \"--fvars (-f)\" options."

  "-v", "--verbose"
    action = :store_true
    help = "Verbose mode prints warnings and additional information to std out."

  "-s", "--generate-summaries"
    action = :store_true
    help = "Generate summary files from protocol files using variables from files where supplied (see --help for --vars & --fvars options)."

  "-j", "--generate-jobs"
    action = :store_true
    help = "Generate LSF job files from summary files."

  "-u", "--list-summaries"
    help = "Path to an text file listing summary file paths."

  "-o", "--list-jobs"
    help = "Path to an text file listing job file paths."

  "-b", "--submit-jobs"
    action = :store_true
    help = "Submit LSF job files to the queue."

  "-r", "--vars"
    help = "Path to \"variables\" file.  This file contains two columns - variable names and variable values.  Matching variable names found in the \"protocol\" will be substituted with the corresponding values.  Variable names may themselves contain variables which will be expanded if the variable name-value pair is found above (in this vars file)."

  "-f", "--fvars"
    help = "Path to \"file variables\" file.  The purpose of this file is to declare variables that are to be substituted with values taken from a list in another file.  This file consists of three columns - variable names, list file column and list file path."

  "-w", "--suppress-warnings"
    action = :store_true
    help = "Do not print warnings to std out."

  "-m", "--summary-prefix"
    help = "Prefix to summary files."

  "-n", "--process-name"
    help = "Name string to be used in all files."

  "-q", "--job-prefix"
    help = "Prefix to job file paths."

  "-t", "--timestamp-files"
    action = :store_true
    help = "Add timestamps to summary and job files."

  "-a", "--portable"
    help = "A directory to which copies of the job files as well as the submission script and relevant functions will be written.  This is done so that the jobs can be easily copied over to and run on a system where jsub.jl is not to be run directly."

  "-z", "--zip-jobs"
    action = :store_true
    help = "Create a zip file containing the jobs directory supplied to the \"--portable\" (-a) option for ease of copying."

  "-c", "--common-header"
    help = "String to be included at the start of every job file.  Default value is \"#!/bin/bash\nset -eu\n\"."

  "-l", "--header-from-file"
    help = "Path to a file containing text to be included in every job file header.  This is included after any string specified in the --common-header option."

  "-y", "--no-version-control"
    action = :store_true
    help = "Do not call the bash function which does version control inside these jobs."

  "-d", "--no-logging-timestamp"
    action = :store_true
    help = "When this flag is not present, timestamps of the format \"YYYYMMDD_HHMMSS\" are not added to the log file by job files running the process_job function."

  "-k", "--keep-superfluous-quotes"
    action = :store_true
    help = "Do not remove superfluous quotes.  For example, the string \"abc\"\"def\" will not be converted to \"abcdef\"."

  "-g", "--fvars-delimiter"
    help = "Delimiter used in the .fvars file.  The default delmiter character is a tab ('\t').  The .fvars file is expected to contain three columns (1) variable name, (2) one-indexed column number, (3) path to a text file."

  "-e", "--prefix-lsf-out"
    help = "Prefix given to the output (*.output) and error (*.error) files produced when running an LSF job."

  "-C", "--prefix-completed"
    help = "Prefix to the *.completed files generated by the LSF job."

  "-I", "--prefix-incomplete"
    help = "Prefix to the *.incomplete files generated by the LSF job."

end
parsed_args = parse_args(argSettings) # the result is a Dict{String,Any}

## Set flag states
const flagVerbose = get_argument(parsed_args, "verbose", verbose=parsed_args["verbose"], optional=true, default=false);
const SUPPRESS_WARNINGS = get_argument(parsed_args, "suppress-warnings", verbose=flagVerbose, optional=true, default=false);
const delimiterFvars = get_argument(parsed_args, "fvars-delimiter", verbose=flagVerbose, optional=true, default='\t');
requiredStages = map_flags_sjb(parsed_args["generate-summaries"], parsed_args["generate-jobs"], parsed_args["submit-jobs"])
flagVerbose && print("\nInterpreted jsub arguments as requesting the following stages: ")
(flagVerbose && requiredStages[1]=='1') && print("1 ")
(flagVerbose && requiredStages[2]=='1') && print("2 ")
(flagVerbose && requiredStages[3]=='1') && print("3 ")
flagVerbose && print("\n\n")

## Initialise shared variables
pathSubmissionScript = string(sourcePath, "/common_functions/submit_lsf_jobs.sh");
pathSubmissionFunctions = string(sourcePath, "/common_functions/job_submission_functions.sh");

## TODO: Check input file format
# Check that pathFvars contains 3 delmiterFvars separated columns

## Declare functions used to run the three stages of summary file generation (1), job file generation (2) and job submission (3).
## STAGE 1
function run_stage1_(pathProtocol, pathVars, pathFvars; processName="", summaryPrefix="", pathSummariesList="", flagVerbose=false, adapt_quotation=true, delimiterFvars='\t', tagsExpand=Dict(), keepSuperfluousQuotes=false, timestampString="")
  flagVerbose && println("\n - STAGE 1: Generating summary files using data from files supplied to the --protocol, --vars and --fvars options.");

  ## Determine what names to use for output summary files
  # The following options are used to generate a "process name"
  #   --protocol "dir/protocolFile"
  #   --vars     "dir/varsFile"
  #   --fvars    "dir/fvarsFile"
  #     OR
  #   --process-name "processName"
  if processName == ""
    stringProtocol = remove_suffix(basename(pathProtocol), ".protocol");
    stringVars = remove_suffix(basename(pathVars), ".vars");
    stringFvars = remove_suffix(basename(pathFvars), ".fvars");
    processName = stick_together(stick_together(stringProtocol, stringVars, "_"), stringFvars, "_");
  end
  flagDebug && println("Using \"$processName\" as the default recurring name but this may be overwritten by specific inputs or tags in the summary files.");

  ## Get path to summaries list file
  if pathSummariesList == ""
    pathSummariesList = string(summaryPrefix, processName, ".list-summaries");
  end

  ## Read protocol, vars and fvars files and expand variables
  namesVars = []; valuesVars = [];
  if pathVars != ""
    flagVerbose && println(string("Expanding variables line by line in data from \"--vars\" file: ", pathVars));
    # Read .vars file # Extract arrays of variable names and variable values
    namesVarsRaw, valuesVarsRaw = parse_varsfile_(pathVars, tagsExpand=tagsExpand);
    # Expand variables in each row from .vars if they were assigned in a higher row (as though they are being assigned at the command line).
    namesVars, valuesVars = expandinorder(namesVarsRaw, valuesVarsRaw, adapt_quotation=adapt_quotation);
  end
  namesFvars = []; infileColumnsFvars = []; filePathsFvars = [];
  if pathFvars != ""
    flagVerbose && println(string("Expanding variables in data from \"--fvars\" file using names and values from the \"--vars\" file: ", pathFvars));
    namesFvars, infileColumnsFvars, filePathsFvars = parse_expandvars_fvarsfile_(pathFvars, namesVars, valuesVars; dlmFvars=delimiterFvars, verbose=false, adapt_quotation=adapt_quotation, tagsExpand=tagsExpand, keepSuperfluousQuotes=keepSuperfluousQuotes);
  end

  # Previously: # Read .protocol file (of 1 column ) and expand variables from .vars
  # (flagVerbose && length(namesVars) > 0) && println("Expanding variables in protocol file using values from the --vars file.");
  # arrProtExpVars, cmdRowsProt = parse_expandvars_protocol_(pathProtocol, namesVars, valuesVars, adapt_quotation=adapt_quotation, verbose=false, tagsExpand=tagsExpand, keepSuperfluousQuotes=keepSuperfluousQuotes);

  dictListArr = Dict(); dictCmdLineIdxs = Dict();
  if pathFvars != ""
    println(string("Expanding variables from the --fvars file using values from the files listed in each row..."));
    dictListArr, dictCmdLineIdxs = parse_expandvars_listfiles_(filePathsFvars, namesVars, valuesVars, delimiterFvars; verbose=false, adapt_quotation=adapt_quotation, tagsExpand=tagsExpand, keepSuperfluousQuotes=keepSuperfluousQuotes);
    if length(keys(dictListArr)) != length(keys(dictCmdLineIdxs))
      error("Numbers of command rows (", length(keys(dictListArr)), ") and command row indices (", length(keys(dictCmdLineIdxs)), ") in list file (", filePathsFvars, ") do not match.")
    end
  end

  ## Create summary files
  # Use variable values from "list" files to create multiple summary file arrays from the single .protocol file array
  flagVerbose && println("Creating summary arrays...");
  arrArrExpFvars = [];
  arrProt, cmdRowsProt = file2arrayofarrays_(pathProtocol, comStr; cols=1, delimiter=nothing, tagsExpand=tagsExpand);
  if length(keys(dictListArr)) != 0 && length(keys(dictCmdLineIdxs)) != 0
    (flagVerbose && length(namesVars) > 0) && println("Expanding variables in protocol file using values from both the --vars and --frvars files.");

    arrArrExpFvars = protocol_to_array(arrProt, cmdRowsProt, namesVars, valuesVars, namesFvars, infileColumnsFvars, filePathsFvars, dictListArr, dictCmdLineIdxs ;
      verbose=false, adapt_quotation=adapt_quotation, keepSuperfluousQuotes=keepSuperfluousQuotes
    );

  else
    # Read .protocol file (of 1 column ) and expand variables from .vars
    (flagVerbose && length(namesVars) > 0) && println("Expanding variables in protocol file using only values from the --vars file.");
    push!(arrArrExpFvars, expand_inarrayofarrays(arrProt, cmdRowsProt, namesVars, valuesVars; verbose=false, adapt_quotation=adapt_quotation, keepSuperfluousQuotes=keepSuperfluousQuotes)); # If there is no data from list files, simply proceed using the data from the vars file
  end
  # Generate list of summary file paths.
  summaryPaths = get_summary_names(arrArrExpFvars; tag="#JSUB<summary-name>", # if an entry with this tag is found in the protocol (arrArrExpFvars), the string following the tag will be used as the name
    longName=processName, # Otherwise the string passed to longName will be used as the basis of the summary file name
    prefix=summaryPrefix,
    suffix=".summary",
    timestamp=timestampString
  );
  # Take an expanded protocol in the form of an array of arrays and produce a summary file for each entry
  outputSummaryPaths = create_summary_files_(arrArrExpFvars, summaryPaths; verbose=flagVerbose);
  println(string("Writing list of summary files to: ", pathSummariesList));
  writedlm(pathSummariesList, outputSummaryPaths);

  return pathSummariesList
end

## STAGE 2
function run_stage2_(pathSummariesList, pathJobsList; jobFilePrefix="", flagVerbose=false, tagsExpand=Dict(), checkpointsDict=Dict(), commonFunctions=Dict(), tagCheckpoint="jcheck_", doJsubVersionControl=true, stringBoolFlagLoggingTimestamp=true, headerPrefix="#!/bin/bash", headerSuffix="", prefixOutputError="", prefixCompleted="", prefixIncomplete="", timestampString="")
  flagVerbose && println("\n - STAGE 2: Using summary files to generate LSF job files.");
  summaryPaths2 = readdlm(pathSummariesList); # Read paths to summary files from list file
  summaryFilesData = map((x) -> file2arrayofarrays_(x, "#", cols=1, tagsExpand=tagsExpand), summaryPaths2 ); # Note: file2arrayofarrays_ returns a tuple of file contents (in an array) and line number indices (in an array)

  flagVerbose && println("Importing bash functions from files...");
  arrDictCheckpoints = map((x) -> identify_checkpoints(x[1], checkpointsDict; tagCheckpoint="jcheck_"), summaryFilesData );
  arrBashFunctions = map((x) -> get_bash_functions(commonFunctions, x), arrDictCheckpoints);

  flagVerbose && println("Splitting summary file contents into separate jobs...");
  summaryArrDicts = map((x) -> split_summary(x[1]; tagSplit=tagsExpand["tagSplit"]), summaryFilesData);

  ## The job file name and the job ID passed to lsf's bsub command are determined by the contents of the array arrJobIDs from which values are passed to the create_jobs_from_summary_ function.
  ## Get job ID and check that the list is unique
  jobIDTag = "#JSUB<job-id>";
  flagVerbose && println("Getting job ID prefixes from summary file lines starting with: ", jobIDTag);
  preArrJobIDs = map((x) -> get_taggedunique(x[1], jobIDTag), summaryFilesData );
  # Create an array of summary file basenames concatenated with a padded index
  replaceWith = map((x, y) -> stick_together(basename(remove_suffix(x, ".summary")), dec(y, length(dec(length(summaryPaths2)))), "_"),
    summaryPaths2, collect(1:length(summaryPaths2))
  );
  flagDebug && (println("Replaceing blank jobID entries with the values from array replaceWith:"); println(replaceWith);)
  # Use this array to replace any empty strings in the jobIDs array
  arrJobIDs = replace_empty_strings(preArrJobIDs, replaceWith);
  (length(arrJobIDs) != length(unique(arrJobIDs))) && error(" in run_stage2_ the array of job IDs contains non-qunique entries:\n", arrJobIDs);
  flagDebug && (println("Final array of job IDs is arrJobIDs:"); println(arrJobIDs);)

  ## Create directories required by prefixes
  (dirname(jobFilePrefix) != "") && mkpath(dirname(jobFilePrefix)); # Create directory for job files if it does not already exist
  (dirname(prefixOutputError) != "") && mkpath(dirname(prefixOutputError)); # Create directory for .error and .output files
  (dirname(prefixCompleted) != "") && mkpath(dirname(prefixCompleted)); # Create directory for .completed files
  (dirname(prefixIncomplete) != "") && mkpath(dirname(prefixIncomplete)); # Create directory for .incomplete files

  ## Write job files
  arrDictFilePaths = map((summaryFilePath, dictSummaries, jobID) -> create_jobs_from_summary_(summaryFilePath, dictSummaries, commonFunctions, checkpointsDict;
      jobFilePrefix=jobFilePrefix, jobID=jobID, jobDate=timestampString,
      doJsubVersionControl=doJsubVersionControl, stringBoolFlagLoggingTimestamp=stringBoolFlagLoggingTimestamp, headerPrefix=headerPrefix, headerSuffix=headerSuffix, verbose=flagVerbose, bsubOptions=bsubOptions, prefixOutputError=prefixOutputError, prefixCompleted=prefixCompleted, prefixIncomplete=prefixIncomplete
    ),
    summaryPaths2, summaryArrDicts, arrJobIDs,
  );

  ## Get an array of job priorities
  arrDictPriorities = map((dictSummaries, dictFilePaths, jobID) -> get_priorities(dictSummaries, dictFilePaths), summaryArrDicts, arrDictFilePaths, arrJobIDs)

  ## Re-order job paths list according to job priority
  arrArrOrderedJobPaths = map((ranksDict, pathsDict) -> order_by_dictionary(ranksDict, pathsDict), arrDictPriorities, arrDictFilePaths)

  ## Write ordered list of job paths to file
  string2file_(pathJobsList, join(map(x -> join(x, '\n'), arrArrOrderedJobPaths), '\n') * "\n");

  return pathJobsList
end

## STAGE 3
function run_stage3_(pathJobsList, pathPortable, pathSubmissionScript, pathSubmissionFunctions; flagVerbose=false, flagZip=false)
  flagVerbose && println("\n - STAGE 3: Submitting LSF jobs.");

  ## Call the job submission script or copy it to the jobs directory
  if (pathPortable == "")
    SUPPRESS_WARNINGS ? arg2 = "suppress-warnings" : arg2 = "";
    flagVerbose && println("Submitting jobs to LSF queuing system using...");
    flagVerbose && println("bash $pathSubmissionScript $pathJobsList $arg2");
    subRun = "";
    if checkforlsf_()
      try
        run(`bash $pathSubmissionScript $pathJobsList $arg2`);
      catch
        println(subRun);
      end
    else
      println("The LSF queuing system does not appear to be available on this system.")
      # println("If this is incorrect consider amending line 423 of jsub.jl or submitting manually using:")
      println("Finished running jsub stage 3 without submitting any jobs.")
    end
    ## Point out that the zip option currently only works in combination with the portable option
    if flagZip == true
      (!SUPPRESS_WARNINGS) && println("WARNING (in jsub.jl): The --zip-jobs (-z) flag was supplied but an argument to the --portable (-a) option was not supplied.  Currently the -z option only works together with -a so it will have no effect here.");
    end
  else
    # Get the target directory from the the portable option or use default
    pathPortableZip = get_zip_dir_path(pathPortable);
    mkpath(pathPortable); # Create the portable directory if needed

    # Parse list of job paths and copy them to the portable directory (if it's not the same directory)
    flagVerbose && println(string("Copying file listing jobs (", basename(pathJobsList), ") to the directory: ", pathPortable));
    cp(pathJobsList, string(pathPortable, "/", basename(pathJobsList)), remove_destination=true); # Copy list of jobs
    arrJobPaths = split(readall(pathJobsList), '\n')
    for jobFile in arrJobPaths
      if !ispath(jobFile)
        (!SUPPRESS_WARNINGS) && println("WARNING (in jsub.jl): The list file $pathJobsList contains a non-valid path: $jobFile");
      elseif (dirname(jobFile) != pathPortable)
        flagVerbose && println(string("Copying job file \"$jobFile\" to directory specificed by the --portable (-a) option: ", pathPortable));
        cp(jobFile, string(pathPortable, "/", basename(jobFile)), remove_destination=true);
      end
    end

    flagVerbose && println(string("Writing a copy of the submission script and functions file to the job file directory: ", pathPortable));
    # println("pathSubmissionScript = ", pathSubmissionScript);
    cp(pathSubmissionScript, string(pathPortable, "/", basename(pathSubmissionScript)), remove_destination=true);
    # println("pathSubmissionFunctions = ", pathSubmissionFunctions);
    cp(pathSubmissionFunctions, string(pathPortable, "/", basename(pathSubmissionFunctions)), remove_destination=true);
    flagVerbose && println(string("The jobs can be submitted to the queuing system by running the shell script: ", basename(pathSubmissionScript)));

    ## Zip jobs directory if requested
    if flagZip == true
      flagVerbose && println("Zipping jobs directory: ", pathPortable);
      flagVerbose && println("             into file: ", pathPortableZip);
      flagVerbose ? zipVerbose = "v" : zipVerbose = ""
      subZip = "";
      try
        subZip = run(`tar -zc$[zipVerbose]f $pathPortableZip $pathPortable`);
      catch
        println(subZip);
      end
    end
  end
  flagVerbose && println("");
end

## RUN ##
if requiredStages[1] == '1'
  flagDebug && println(" --- Starting STAGE 1.\n")
  pathExistingSummariesList = run_stage1_(
    get_argument(parsed_args, "protocol"; verbose=flagVerbose, optional=(requiredStages[1]=='0'), default=""),
    get_argument(parsed_args, "vars"; verbose=flagVerbose, optional=true, default=""),
    get_argument(parsed_args, "fvars"; verbose=flagVerbose, optional=true, default="");
    processName=get_argument(parsed_args, "process-name"; verbose=flagVerbose, optional=true, default=""),
    summaryPrefix=get_argument(parsed_args, "summary-prefix", verbose=flagVerbose, optional=true, default=""),
    pathSummariesList=get_argument(parsed_args, "list-summaries"; verbose=flagVerbose, optional=true, default=""),
    keepSuperfluousQuotes=get_argument(parsed_args, "keep-superfluous-quotes", verbose=flagVerbose, optional=true, default=false),
    flagVerbose=flagVerbose, adapt_quotation=adapt_quotation, delimiterFvars=delimiterFvars, tagsExpand=tagsExpand,
    timestampString=( (get_argument(parsed_args, "timestamp-files", verbose=flagVerbose, optional=true, default=false) ? get_timestamp_(nothing) : "") ), # Get summary timestamp string,
  );
  flagDebug && println(" --- Completed STAGE 1.  Produced summary list file: $pathExistingSummariesList\n")
elseif requiredStages[2] == '1'
  flagDebug && println("Not running stage 1, so the path to *.list-summaries has to come from an argument.")
  pathExistingSummariesList = get_argument(parsed_args, "list-summaries"; verbose=flagVerbose, optional=(requiredStages[2] != '1'), default=""); # Optional if jobs are not being generated from summaries
end
if requiredStages[2] == '1'
  pathCommonHeader = get_argument(parsed_args, "header-from-file"; verbose=flagVerbose, optional=true, default="");
  inputJobsPrefix = get_argument(parsed_args, "job-prefix", verbose=flagVerbose, optional=true, default="")
  flagDebug && println(" --- Starting STAGE 2.\n")
  pathExistingJobsList = run_stage2_(
    pathExistingSummariesList,
    string(inputJobsPrefix, basename(remove_suffix(pathExistingSummariesList, ".list-summaries")), ".list-jobs"); # Determine path to the *.list-jobs file
    flagVerbose=flagVerbose, tagsExpand=tagsExpand, checkpointsDict=checkpointsDict, commonFunctions=commonFunctions,
    jobFilePrefix=inputJobsPrefix,
    doJsubVersionControl=( get_argument(parsed_args, "no-version-control"; verbose=flagVerbose, optional=true, default=false) ? "false" : "true" ),
    stringBoolFlagLoggingTimestamp=( get_argument(parsed_args, "no-logging-timestamp"; verbose=flagVerbose, optional=true, default=false) ? "false" : "true" ), # Indicates if bash scripts should create a timestamp in the logging file, default is "true" (this is a string because it is written into a bash script)
    headerPrefix=get_argument(parsed_args, "common-header"; verbose=flagVerbose, optional=true, default="#!/bin/bash\nset -eu\n"),
    prefixOutputError=get_argument(parsed_args, "prefix-lsf-out", verbose=flagVerbose, optional=true, default=""), # Get prefix for *.error and *.output files (written by the LSF job)
    prefixCompleted=get_argument(parsed_args, "prefix-completed", verbose=flagVerbose, optional=true, default=""),
    prefixIncomplete=get_argument(parsed_args, "prefix-incomplete", verbose=flagVerbose, optional=true, default=""),
    timestampString=(get_argument(parsed_args, "timestamp-files", verbose=flagVerbose, optional=true, default=false) ? get_timestamp_(nothing) : ""), # Get job timestamp string
    headerSuffix=(pathCommonHeader == "" ? "" : readall(pathCommonHeader)), # String from file to be added to the header (after common-header string) of every job file
  )
  flagDebug && println(" --- Completed STAGE 2. Produced jobs list file: $pathExistingJobsList\n")
elseif requiredStages[3] == '1'
  flagDebug && println("Not running stage 2, so the path to *.list-jobs has to come from an argument.")
  pathExistingJobsList = get_argument(parsed_args, "list-jobs"; verbose=flagVerbose, optional=(requiredStages[3] != '1'), default=""); # Optional if jobs are not being submitted
end
if requiredStages[3] == '1'
  flagDebug && println(" --- Starting STAGE 3.\n")
  run_stage3_(
    pathExistingJobsList,
    get_argument(parsed_args, "portable", verbose=flagVerbose, optional=true, default=""),
    pathSubmissionScript,
    pathSubmissionFunctions;
    flagVerbose=flagVerbose,
    flagZip=get_argument(parsed_args, "zip-jobs"; verbose=flagVerbose, optional=true, default=false),
  )
  flagDebug && println(" --- Completed STAGE 3.\n")
end
#########

# Report if there were any suppressed warnings
if num_suppressed[1] > 0
  println("Suppressed ", num_suppressed[1], " warnings.");
end
########################
# EOF

max_stars_repo_name: olenive/jsub

max_issues_repo_name: olenive/jsub

max_forks_repo_name: olenive/jsub


text:
_isreducible(m::Module, v) = (getoperator(m.dfg, v).type != typeof(SBitstream)) &&
                             all(isconstant, getinputs(m.dfg, v))

function _reducenode!(m::Module, v)
    # determine constant
    val = value.(getoutputs(m.dfg, v))

    # double check result
    isreal(val) || error("""
        Tried to reduce node, but received non-numeric result: $val
            args: $(value.(getinputs(m.dfg, v)))
            op: $(getoperator(m.dfg, v))
        """)

    # delete node and propagate constant
    children = outneighbors(m.dfg, v)
    outputs = getoutputs(m.dfg, v)
    for child in children
        inputs = getinputs(m.dfg, child)
        for (j, output) in enumerate(outputs)
            for (i, input) in enumerate(inputs)
                if name(input) == name(output)
                    inputs[i] = Net(val[j]; name = string(val[j]), class = :constant)
                end
            end
            set_prop!(m.dfg, child, :inputs, inputs)
        end
    end
    rem_vertex!(m.dfg, v)

    return m
end

function constantreduction!(m::Module)
    # if there are no nodes to reduce then return
    nodes = filter(v -> _isreducible(m, v), getroots(m.dfg))
    isempty(nodes) && return m

    # reduce one node then try again
    _reducenode!(m, nodes[1])
    constantreduction!(m)

    return m
end


text:
export EntModel

mutable struct EntModel <: Model
    tokens::BOW
    config::TextConfig
end

function smooth_factor(dist::AbstractVector)::Float64
    s = sum(dist)
    s < length(dist) ? 1.0 : 0.0
end

"""
    fit(::Type{EntModel}, model::DistModel, smooth::Function=smooth_factor; lower=0.001)

Fits an EntModel using the already fitted DistModel; the `smooth` function is called to compute the smoothing factor
for a given histogram. It accepts only symbols with a final weight higher or equal than `lower`.
"""
function fit(::Type{EntModel}, model::DistModel, smooth::Function=smooth_factor; lower=0.001)
    tokens = BOW()
    nclasses = length(model.sizes)
    maxent = log2(nclasses)

    @inbounds for (token, dist) in model.tokens
        b = smooth(dist)
        e = 0.0
        pop = b * nclasses + sum(dist)

        for j in 1:nclasses
            pj = (dist[j] + b) / pop

            if pj > 0.0
                e -= pj * log2(pj)
            end
        end
        e = 1.0 - e / maxent
        if e >= lower
            tokens[token] = e
        end
    end

    EntModel(tokens, model.config)
end

function fit(::Type{EntModel}, config::TextConfig, corpus, y; nclasses=0, norm_by=minimum, smooth=smooth_factor, lower=0.001)
    dmodel = fit(DistModel, config, corpus, y, nclasses=nclasses, norm_by=minimum)
    fit(EntModel, dmodel, smooth, lower=lower)
end

"""
    prune(model::EntModel, lower)

Prunes the model accepting only those symbols with a weight higher than `lower`

"""
function prune(model::EntModel, lower)
    tokens = BOW()
    for (t, ent) in model.tokens
        if ent >= lower
            tokens[t] = ent
        end
    end

    EntModel(tokens, model.config)
end

abstract type EntTfModel end
abstract type EntTpModel end

"""
    vectorize(model::EntModel, data, modify_bow!::Function=identity)::BOW
    vectorize(model::EntModel, ::Type, data, modify_bow!::Function=identity)::BOW

Computes a weighted bow for a given `data`
"""
function vectorize(model::EntModel, scheme::Type{T}, data, modify_bow!::Function=identity; normalize=true)::BOW where T <: Union{EntTfModel,EntTpModel,EntModel}
    bow, maxfreq = compute_bow(model.config, data)
    len = 0
    for v in values(bow)
        len += v
    end

    bow = modify_bow!(bow)
    for (token, freq) in bow
        w = get(model.tokens, token, 0.0)
        w = _weight(scheme, w, freq,  len)
        if w > 0.0
            bow[token] = w
        else
            delete!(bow, token)
        end
    end

    normalize && normalize!(bow)
    bow
end

vectorize(model::EntModel, data, modify_bow!::Function=identity; normalize=true) = vectorize(model, EntTpModel, data, modify_bow!, normalize=normalize)

_weight(::Type{EntTpModel}, ent, freq, n) = ent * freq / n
_weight(::Type{EntTfModel}, ent, freq, n) = ent * freq
_weight(::Type{EntModel}, ent, freq, n) = ent

function broadcastable(model::EntModel)
    (model,)
end