text:
using Symbolics: get_variables, make_array, SerialForm, Func, toexpr, _build_and_inject_function, @__MODULE__, MultithreadedForm, tosymbol, Sym
using SymbolicUtils.Code: MakeArray
# using Rotations

export AbstractMaterial, Material, RotatedMaterial, get_model, generate_fn, Δₘ_factors, Δₘ
export rotate, mult, unique_axes, plot_data, nn̂g, nĝvd, nn̂g_model, nn̂g_fn, nĝvd_model, nĝvd_fn, ε_fn
export n²_sym_fmt1, n_sym_cauchy, has_model, χ⁽²⁾_fn, material_name, plot_model!, n_model, ng_model, gvd_model

# RuntimeGeneratedFunctions.init(@__MODULE__)

import Symbolics: substitute, simplify
Symbolics.substitute(A::AbstractArray{Num},d::Dict) = Symbolics.substitute.(A,(d,))
Symbolics.simplify(A::AbstractArray{Num}) = Symbolics.simplify.(A)

# add Symbolics.get_variables for arrays of `Num`s
import Symbolics.get_variables
function Symbolics.get_variables(A::AbstractArray{Num})
	unique(vcat(get_variables.(A)...))
end

# # add minimal Unitful+Symbolics interoperability
# import Base:*
# *(x::Unitful.AbstractQuantity,y::Num) =  Quantity(x.val*y, unit(x))
# *(y::Num,x::Unitful.AbstractQuantity) = x*y

# adjoint/rrule for SymbolicUtils.Code.create_array
# https://github.com/JuliaSymbolics/SymbolicUtils.jl/pull/278/files
# function ChainRulesCore.rrule(::typeof(SymbolicUtils.Code.create_array), A::Type{<:AbstractArray}, T, u::Val{j}, d::Val{dims}, elems...) where {dims, j}
#   y = SymbolicUtils.Code.create_array(A, T, u, d, elems...)
#   function create_array_pullback(Δ)
#     dx = Δ
#     (NO_FIELDS, DoesNotExist(), DoesNotExist(), DoesNotExist(), DoesNotExist(), dx..., ntuple(_ -> DoesNotExist(), length(elems) - prod(dims) + j)...)
#   end
#   y, create_array_pullback
# end

function generate_array_fn(args::Vector{Num},A::AbstractMatrix; expr_module=@__MODULE__(), parallel=SerialForm())
	return fn = _build_and_inject_function(expr_module,toexpr(Func(args,[],make_array(parallel,args,A,Matrix))))
end

function generate_array_fn(arg::Num,A::AbstractMatrix; expr_module=@__MODULE__(), parallel=SerialForm())
	return fn = generate_array_fn([arg,], A; expr_module, parallel)
end

function generate_array_fn(args::Vector{Num},A::SArray; expr_module=@__MODULE__(), parallel=SerialForm())
	return fn = _build_and_inject_function(expr_module,toexpr(Func(args,[],make_array(parallel,args,A,SArray))))
end

function generate_array_fn(arg::Num,A::SArray; expr_module=@__MODULE__(), parallel=SerialForm())
	return fn = generate_array_fn([arg,], A; expr_module, parallel)
end

function generate_array_fn(args::Vector{Num},A::TA; expr_module=@__MODULE__(), parallel=SerialForm()) where TA<:AbstractArray
	return fn = _build_and_inject_function(expr_module,toexpr(Func(args,[],make_array(parallel,args,A,TA))))
end

function generate_array_fn(arg::Num,A::TA; expr_module=@__MODULE__(), parallel=SerialForm()) where TA<:AbstractArray
	return fn = generate_array_fn([arg,], A; expr_module, parallel)
end

@non_differentiable generate_array_fn(arg,A)

"""
################################################################################
#																			   #
#							   	   Materials							   	   #
#																			   #
################################################################################
"""

abstract type AbstractMaterial end

struct Material <: AbstractMaterial
	models::Dict
	defaults::Dict
	name::Symbol
	color::Color
end

# constructor adding random color when color is not specified
Material(models::Dict,defaults::Dict,name::Symbol) = Material(models,defaults,name,RGBA(rand(3)...,1.0))

import Base: nameof
Base.nameof(mat::AbstractMaterial) = getfield(mat, :name)

material_name(x::Real) = Symbol("Const_Material_$x")
material_name(x::AbstractVector) = Symbol("Const_Material_$(x[1])_$(x[2])_$(x[3])")
material_name(x::AbstractMatrix) = Symbol("Const_Material_$(x[1,1])_$(x[2,2])_$(x[3,3])")

Material(x) = Material(Dict([ε_tensor(x),]),Dict([]),material_name(x))
Material(mat::AbstractMaterial) = mat

function get_model(mat::AbstractMaterial,model_name::Symbol,args...)
	model = mat.models[model_name]
	missing_var_defaults = filter(x->!in(first(x),tosymbol.(args)),mat.defaults)
	subs =  Dict([(Sym{Real}(k),v) for (k,v) in missing_var_defaults])
	if typeof(model)<:AbstractArray
		model_subs = substitute.(model, (subs,))
	else
		model_subs = substitute(model, subs)
	end
	return model_subs
end

function generate_fn(mat::AbstractMaterial,model_name::Symbol,args...; expr_module=@__MODULE__(), parallel=SerialForm())
	model = get_model(mat,model_name,args...)
	if typeof(model)<:AbstractArray
		# fn = generate_array_fn([Num(Sym{Real}(arg)) for arg in args],model; expr_module, parallel)
		fn = build_function(model,args...;expression=Val{false})[2]
	else
		fn = build_function(model,args...;expression=Val{false})
	end
	return fn
end

function has_model(mat::AbstractMaterial,model_name::Symbol)
	return haskey(mat.models,model_name)
end

@non_differentiable generate_fn(mat::AbstractMaterial,model_name::Symbol,args...)

"""
################################################################################
#																			   #
#							   	  Rotations							   		   #
#																			   #
################################################################################
"""

struct RotatedMaterial{TM,TR} <: AbstractMaterial
	parent::TM
	rotation::TR
	rotation_defaults::Dict
	name::Symbol
	color::Color
end

function mult(χ::AbstractArray{T,3},v₁::AbstractVector,v₂::AbstractVector) where T<:Real
	@tullio v₃[i] := χ[i,j,k] * v₁[j] * v₂[k]
end

function mult(χ::AbstractArray{T,4},v₁::AbstractVector,v₂::AbstractVector,v₃::AbstractVector) where T<:Real
	@tullio v₄[i] := χ[i,j,k,l] * v₁[j] * v₂[k] * v₃[l]
end

function rotate(χ::AbstractMatrix,𝓡::AbstractMatrix)
	@tullio χᵣ[i,j] := 𝓡[a,i] * 𝓡[b,j] * χ[a,b]  fastmath=true
end

function rotate(χ::AbstractArray{T,3},𝓡::AbstractMatrix) where {T<:Real}
	@tullio χᵣ[i,j,k] := 𝓡[a,i] * 𝓡[b,j] * 𝓡[c,k] * χ[a,b,c]  fastmath=true
end

function rotate(χ::AbstractArray{T,4},𝓡::TR) where {T<:Real, TR<:StaticMatrix{3,3}}
	@tullio χᵣ[i,j,k,l] := 𝓡[a,i] * 𝓡[b,j] * 𝓡[c,k] * 𝓡[d,l] * χ[a,b,c,d]  fastmath=true
end

rotate(χ::Real,𝓡::StaticMatrix{3,3}) = χ

function rotate(mat::TM,𝓡::TR;name=nothing,color=mat.color) where {TM<:AbstractMaterial,TR<:AbstractMatrix}
	if eltype(𝓡)<:Num
		vars = get_variables(𝓡)
		defs = Dict{Symbol,Real}([ tosymbol(var) => 0.0 for var in vars])
	else
		defs = Dict{Symbol,Real}([])
	end
	if isnothing(name)
		name = Symbol(String(mat.name)*"_Rotated")
	end
	RotatedMaterial{TM,TR}(mat,𝓡,defs,name,color)
end

function rotate(mat::TM,𝓡::TR,defs::Dict;name=nothing,color=mat.color) where {TM<:AbstractMaterial,TR<:AbstractMatrix}
	if isnothing(name)
		name = Symbol(String(mat.name)*"_Rotated")
	end
	RotatedMaterial{TM,TR}(mat,𝓡,defs,name,color)
end

function get_model(mat::RotatedMaterial,model_name::Symbol,args...)
	model = rotate(mat.parent.models[model_name],mat.rotation)
	defs = merge(mat.parent.defaults,mat.rotation_defaults)
	missing_var_defaults = filter(x->!in(first(x),tosymbol.(args)),defs)
	subs =  Dict([(Sym{Real}(k),v) for (k,v) in missing_var_defaults])
	if typeof(model)<:AbstractArray
		model_subs = substitute.(model, (subs,))
	else
		model_subs = substitute(model, subs)
	end
	return model_subs
end

has_model(mat::RotatedMaterial,model_name::Symbol) = haskey(mat.parent.models,model_name)

# material_name(mat::RotatedMaterial) = material_name(mat.parent)
# Base.nameof(mat::RotatedMaterial) = getfield(mat.parent, :name)
"""
################################################################################
#																			   #
#					  Dispersion (group index, GVD) models					   #
#																			   #
################################################################################
"""

function n²_sym_fmt1( λ ; A₀=1, B₁=0, C₁=0, B₂=0, C₂=0, B₃=0, C₃=0, kwargs...)
    λ² = λ^2
    A₀  + ( B₁ * λ² ) / ( λ² - C₁ ) + ( B₂ * λ² ) / ( λ² - C₂ ) + ( B₃ * λ² ) / ( λ² - C₃ )
end

function n_sym_cauchy( λ ; A=1, B=0, C=0, B₂=0, kwargs...)
    A   +   B / λ^2    +   C / λ^4
end

# Miller's Delta scaling
function Δₘ_factors(λs,ε_sym)
	λ = Num(first(get_variables(sum(ε_sym))))
	diagε_m1 = Vector(diag(ε_sym)) .- 1
	mapreduce(lm->substitute.( diagε_m1, ([λ=>lm],)), .*, λs)
end

function Δₘ(λs::AbstractVector,ε_sym, λᵣs::AbstractVector, χᵣ::AbstractArray{T,3}) where T
	dm = Δₘ_factors(λs,ε_sym) ./ Δₘ_factors(λᵣs,ε_sym)
	@tullio χ[i,j,k] := χᵣ[i,j,k] * dm[i] * dm[j] * dm[k] fastmath=true
end

# Symbolic Differentiation
function ng_model(n_model::Num, λ::Num)
	Dλ = Differential(λ)
	return n_model - ( λ * expand_derivatives(Dλ(n_model)) )
end

function gvd_model(n_model::Num, λ::Num)
	Dλ = Differential(λ)
	return λ^3 * expand_derivatives(Dλ(Dλ(n_model)))
end

ng_model(n_model::AbstractArray{Num}, λ::Num) = ng_model.(n_model,(λ,))
gvd_model(n_model::AbstractArray{Num}, λ::Num) = gvd_model.(n_model,(λ,))

function ng_model(mat::AbstractMaterial; symbol=:λ)
	λ = Num(Sym{Real}(symbol))
	n_model = sqrt.(get_model(mat,:ε,symbol))
	return ng_model(n_model,λ)
end

function gvd_model(mat::AbstractMaterial; symbol=:λ)
	λ = Num(Sym{Real}(symbol))
	n_model = sqrt.(get_model(mat,:ε,symbol))
	return gvd_model(n_model,λ)
end

function nn̂g_model(mat::AbstractMaterial; symbol=:λ)
	λ = Num(Sym{Real}(symbol))
	Dλ = Differential(λ)
	n_model = sqrt.(get_model(mat,:ε,symbol))
	return ng_model(n_model,λ) .* n_model
end

function nĝvd_model(mat::AbstractMaterial; symbol=:λ)
	λ = Num(Sym{Real}(symbol))
	Dλ = Differential(λ)
	n_model = sqrt.(get_model(mat,:ε,symbol))
	return gvd_model(n_model,λ) .* n_model
end

ε_fn(mat::AbstractMaterial) = generate_array_fn([Num(Sym{Real}(:λ)) ,],get_model(mat,:ε,:λ))
nn̂g_fn(mat::AbstractMaterial) =  generate_array_fn([Num(Sym{Real}(:λ)) ,],nn̂g_model(mat))
nĝvd_fn(mat::AbstractMaterial) =  generate_array_fn([Num(Sym{Real}(:λ)) ,],nĝvd_model(mat))

function χ⁽²⁾_fn(mat::AbstractMaterial)
	if has_model(mat,:χ⁽²⁾)
		return generate_array_fn([Num(Sym{Real}(:λs₁)), Num(Sym{Real}(:λs₂)), Num(Sym{Real}(:λs₃))],get_model(mat,:χ⁽²⁾,:λs₁,:λs₂,:λs₃))
	else
		return (lm1,lm2,lm3) -> zero(SArray{Tuple{3,3,3}})
	end
end

nn̂g(mat::AbstractMaterial,lm::Real) = SMatrix{3,3}(nn̂g_fn(mat)(lm))
nĝvd(mat::AbstractMaterial,lm::Real) = SMatrix{3,3}(nĝvd_fn(mat)(lm))

"""
################################################################################
#																			   #
#							    Utility methods					   			   #
#																			   #
################################################################################
"""

function unique_axes(mat::AbstractMaterial;model=:ε)
	e11,e22,e33 = diag(get_model(mat,model,:λ))
	if isequal(e11,e22)
		isequal(e11,e33) ? (return ( [1,], [""] )) : (return ( [1,3], ["₁,₂","₃"] )) # 1 == 2 == 3 (isotropic) : 1 == 2 != 3 (uniaxial)
	elseif isequal(e22,e33)
		return ( [1,2], ["₁","₂,₃"] )	# 1 != 2 == 3 (uniaxial)
	else
		isequal(e11,e33) ? (return ( [1,2], ["₁,₃","₂"] )) : (return ( [1,2,3], ["₁","₂","₃"] )) # 1 == 3 != 2 (uniaxial) : 1 != 2 != 3 (biaxial)
	end
end

"""
################################################################################
#																			   #
#							   Plotting methods					   			   #
#																			   #
################################################################################
"""

function plot_data(mats_in::AbstractVector{<:AbstractMaterial};model=:n)
	if isequal(model,:n)
		mats = filter(x->has_model(x,:ε),mats_in)
		# fes = generate_fn.(mats,(:ε,),(:λ,))
		fes = ε_fn.(mats)
		axind_axstr_unq = unique_axes.(mats)
		axind_unq = getindex.(axind_axstr_unq,1)
		axstr_unq = getindex.(axind_axstr_unq,2)
		fns = vcat(map((ff,as)->[(x->sqrt(ff(x)[a,a])) for a in as ], fes, axind_unq)...)
		mat_names = String.(nameof.(mats))
		names = "n" .* vcat([.*(axstr_unq[i], " (", mat_names[i],")") for i=1:length(mats)]...) # "n, n_i or n_i,j (Material)" for all unique axes and materials
	else
		mats = filter(x->has_model(x,model),mats_in)
		# fgs = generate_fn.(mats,(model,),(:λ,))
		fgs = generate_array_fn.(([Num(Sym{Real}(:λ)) ,],),get_model.(mats,(model,),(:λ,)))
		axind_axstr_unq = unique_axes.(mats)
		axind_unq = getindex.(axind_axstr_unq,1)
		axstr_unq = getindex.(axind_axstr_unq,2)
		fns = vcat(map((ff,as)->[(x->ff(x)[a,a]) for a in as ], fgs, axind_unq)...)
		mat_names = String.(nameof.(mats))
		names = String(model) .* vcat([.*(axstr_unq[i], " (", mat_names[i],")") for i=1:length(mats)]...)
	end
	colors = vcat( [ [ mat.color for i=1:ll ] for (mat,ll) in zip(mats,length.(getindex.(axind_axstr_unq,(1,)))) ]...)
	all_linestyles	=	[nothing,:dash,:dot,:dashdot,:dashdotdot]
	linestyles  =	vcat( [ getindex.((all_linestyles,),1:ll) for ll in length.(getindex.(axind_axstr_unq,(1,))) ]... )
	return fns, names, colors, linestyles
end
plot_data(mat::AbstractMaterial ; model=:n) = plot_data([mat,]; model)
plot_data(mats::NTuple{N,<:AbstractMaterial} where N ; model=:n) = plot_data([mats...]; model)

function uplot(x::Union{AbstractMaterial, AbstractVector{<:AbstractMaterial}, NTuple{N,<:AbstractMaterial} };
		model=:n, xlim=[0.5,1.8], xlabel="λ [μm]", ylabel="n", kwargs...)  where N
	fns, name, colors, styles = plot_data(x;model)
	UnicodePlots.lineplot(fns, xlim[1], xlim[2];
	 	xlim,
		ylim=map((a,b)->a(b,digits=1),(floor,ceil),ylims(fns;xlims=xlim)),
		name,
		xlabel,
		ylabel,
		width=75,
		height=35,
		kwargs...
		)
end

function uplot!(plt::UnicodePlots.Plot,x::Union{Material, AbstractVector{<:Material}, NTuple{N,<:Material} };
		xlim=[0.5,1.8], xlabel="λ [μm]", ylabel="n")  where N
	fns, name, colors, styles = plot_data(x)
	UnicodePlots.lineplot!(plt, fns; name ) #, xlim[1], xlim[2];
	 	# xlim,
		# ylim=round.( ylims(plt,ylims(fns;xlims=xlim)) ,digits=1),
		# name,
		# xlabel,
		# ylabel,
		# )
end

function plot_model!(ax, mats::AbstractVector{<:AbstractMaterial};model=:n,xrange=nothing,kwargs...)
	if isnothing(xrange)
		xmin = ax.limits[].origin[1]
		xmax = xmin + ax.limits[].widths[1]
	end
	lns = [lines!(ax, xmin..xmax, fn; label=lbl, color=clr, linestyle=ls, kwargs...) for (fn,lbl,clr,ls) in zip(plot_data(mats; model)...)]
end
plot_model(ax, mat::AbstractMaterial ; model=:n, xrange=nothing, kwargs...) = plot_model([mat,]; model, xrange, kwargs...)
plot_model(ax, mats::NTuple{N,<:AbstractMaterial} where N ; model=:n, xrange=nothing, kwargs...) = plot_model([mats...]; model, xrange, kwargs...)

import Base: show
Base.show(io::IO, ::MIME"text/plain", mat::AbstractMaterial) = uplot(mat) #print(io, "Examplary instance of Material\n", m.x, " ± ", m.y)
Base.show(io::IO, mat::AbstractMaterial) = uplot(mat) #print(io, m.x, '(', m.y, ')')
Base.show(io, ::MIME"text/plain", mat::AbstractMaterial) = uplot(mat)
################################################################################
#                                Load Materials                                #
################################################################################
include("material_lib/MgO_LiNbO3.jl")
include("material_lib/SiO2.jl")
include("material_lib/Si3N4.jl")
include("material_lib/αAl2O3.jl")
include("material_lib/LiB3O5.jl")
# include("material_lib/silicon.jl")
# include("material_lib/GaAs.jl")
# include("material_lib/MgF2.jl")
# include("material_lib/HfO2.jl")


text:
#==== Move ====================================================================#

function htmove(h::Symbol, Y, C::Constants, S::State, X::Auxiliary, U::Counts)
    if h == :hmc
        ht_hmc(Y,C,S,X,U)
    else
        ht_gibbs(Y,C,S,X,U)
    end
end

#==== Gibbs ===================================================================#

function ht_gibbs(Y, C::Constants, S::State, X::Auxiliary, U::Counts)

    k,α,β = S.k, C.α, C.β
    @views locs,hts = S.locs[1:k+2], S.hts[1:k+1]

    @views ix = X.ix[1:k+1] # heights will be
    shuffle!(ix)            # updated in random order.
    for j in ix
        s,sn = locs[j], locs[j+1]
        h = hts[j]
        x = h*exp(rand()-.5)

        lratio = llhdratio(Y,s,sn,sn,x,x,h) + α*log(x/h) - (x-h)β

        ratio = exp(max(-30,min(0,lratio)))
        if rand() < ratio
            hts[j] = x
            U.haccepts += 1
        end
        U.hprops += 1
    end
    ht_init(Y,S,X)        # leave the auxiliary
    ht_unconstrained(k,X) # variables in a
    ix .= 1:k+1           # consistent state.

end

#==== HMC =====================================================================#

function ht_hmc(Y, C::Constants, S::State, X::Auxiliary, U::Counts)

    k = S.k
    ε = .8C.heps + rand()*.4C.heps # base step size ± 20%.
    ht_init(Y,S,X)
    ht_unconstrained(k,X)
    ht_momenta(k,X)
    ep = ht_potential(k,C,X) # previous energy; also computes the gradient.
    kp = ht_kinetic(k,X)

    ht_pstep(.5ε,k,X)
    for i = 1:C.hleaps
        ht_qstep(ε,k,X)
        if i != C.hleaps
            ht_potential(k,C,X)
            ht_pstep(ε,k,X)
        end
    end
    en = ht_potential(k,C,X)
    ht_pstep(.5ε,k,X)
    kn = ht_kinetic(k,X)

    p = rand()
    l = exp(ep+kp-en-kn)

    if p < l
        @views S.hts[1:k+1] .= X.hts[1:k+1]
        U.haccepts += 1
    else
        ht_init(Y,S,X)        # leave the auxiliary variables
        ht_unconstrained(k,X) # in a consistent state.
    end
    U.hprops += 1

end

function ht_pstep(s, k, X::Auxiliary)
    @. @views X.plhts[1:k+1] -= s * X.∇lhts[1:k+1]
end

function ht_qstep(s, k, X::Auxiliary)
    @. @views X.lhts[1:k+1] += s * X.plhts[1:k+1]
    ht_constrained(k,X)
end

function ht_potential(k, C::Constants, X::Auxiliary)
    R,α,β = C.R, C.α, C.β
    @views locs,hts,lhts = X.locs[2:k+1], X.hts[1:k+1], X.lhts[1:k+1]
    @views widths,counts = X.widths[1:k+1], X.counts[1:k+1]

    e = sum(@. (widths+β)*hts - (counts+α)*lhts - log(widths)) -
        sum(@. log(locs*(R-locs)))
    @. X.∇lhts[1:k+1] = (widths+β)*hts - (counts+α)
    return e
end

ht_kinetic(k, X::Auxiliary) = @views .5 * sum(X.plhts[1:k+1].^2)

#==== Auxiliary ===============================================================#

function ht_init(Y, S::State, X::Auxiliary) # hts.
    k = S.k
    @views X.hts[1:k+1] .= S.hts[1:k+1]
end

function ht_unconstrained(k, X::Auxiliary) # lhts.
    @. @views X.lhts[1:k+1] = log(X.hts[1:k+1])
end

function ht_constrained(k, X::Auxiliary) # hts.
    @. @views X.hts[1:k+1] = exp(X.lhts[1:k+1])
end

function ht_momenta(k, X::Auxiliary) # plhts
    @views randn!(X.plhts[1:k+1])
end

#===== I/O =====================================================================#

function ht_println(k, X::Auxiliary)
    println("X: locs = $(string(X.locs[1:k+2])),",
              " hts = $(string(X.hts[1:k+1]))\n",
            "   lhts = $(string(X.lhts[1:k+1])),",
              " ∇lhts = $(string(X.∇lhts[1:k+1])),",
              " plhts = $(string(X.plhts[1:k+1]))")
end

ext: jl

max_stars_repo_name: kevdur/rjcp

max_issues_repo_name: kevdur/rjcp

max_forks_repo_name: kevdur/rjcp


text:
module Tokenizer
using JSON

#default_caption_files = ["data/Flickr30k/results_20130124.token", "data/MsCoco/captions_val2014.json", "data/MsCoco/captions_train2014.json"]
default_caption_files = ["data/Flickr30k/results_20130124.token"];
function tokenize(vocab, caption_dicts; data_files=default_caption_files)
  word_counts = Dict{String,Int}();
  for i=1:length(data_files)
      file = open(data_files[i])
      tokenization_type = split(data_files[i],'.')[2]
      if tokenization_type == "token"
        #make vocab unchanged in every random val/test selection
        dict1 = tokenize_flicker_captions(readlines(file));
        #push!(caption_dicts, dict1)
        vocab = get_vocab(dict1, vocab, word_counts)
        #end
         file = open(data_files[i])
         dict1,dict2,dict3 = tokenize_flicker_captions2(readlines(file));
         push!(caption_dicts, dict1, dict2, dict3)
      elseif tokenization_type == "json"
        dict = tokenize_mscoco_captions(readlines(file));
        push!(caption_dicts, dict)
        vocab = get_vocab(dict, vocab, word_counts)
      else
        println("invalid caption file: ", data_files[i])
      end
      close(file)
  end

  filtervocab(5,vocab,word_counts)
  return vocab, caption_dicts ;
end

function tokenize_flicker_captions(captions)
  captions_dict = Array{Tuple{Tuple{Int64,Array{String,1}},Int64},1}()
  for i=1:length(captions)
      words = map(lowercase, split(captions[i],[' ','\t','#','.', '\n']))
      id = words[1]
      id = parse(Int64,id)
      count = 4;
      for i=4:length(words)
        words[count] = lowercase(strip(words[count],[' ', '.', ',','#', '\'', ')', '(', '!', '/', '?', '\t', '`']))
        if length(words[count]) < 1
          deleteat!(words,count)
          count-=1;
        end
        count += 1;
      end
      push!(captions_dict, ((id,words[4:end]),count-4));
  end
  captions_dict = sort(captions_dict, by = tuple -> last(tuple), rev=false)
  return captions_dict
end

function tokenize_flicker_captions2(captions)
  val_size = 1000; test_size = 1000;
  #to keep same val/test set
  srand(5);
  c = shuffle(1:5:length(captions))
  srand(ceil(Int,time()))
  val_line_numbers  = c[1:val_size];
  test_line_numbers = c[val_size+1:val_size+test_size];
  val_caption_idexes = [val_line_numbers;val_line_numbers+1;
                        val_line_numbers+2;val_line_numbers+3;
                        val_line_numbers+4];
 sort!(val_caption_idexes)
 test_caption_idexes = [test_line_numbers;test_line_numbers+1;
                        test_line_numbers+2;test_line_numbers+3;
                        test_line_numbers+4]
 sort!(test_caption_idexes)

 val_captions = captions[val_caption_idexes];
 test_captions = captions[test_caption_idexes];

 deleted = [val_caption_idexes;test_caption_idexes]
 sort!(deleted);
 deleteat!(captions,deleted);

 captions_dict = Array{Tuple{Tuple{Int64,Array{String,1}},Int64},1}()
 validations_dict = Array{Tuple{Tuple{Int64,Array{String,1}},Int64},1}()
 test_dict = Array{Tuple{Tuple{Int64,Array{String,1}},Int64},1}()
 flicker(captions, captions_dict)
 flicker(val_captions,validations_dict)
 flicker(test_captions, test_dict);
 return captions_dict, validations_dict, test_dict;
end

function flicker(captions, captions_dict;sort=true)
  for i=1:length(captions)
      words = map(lowercase, split(captions[i],[' ','\t','#','.', '\n']))
      id = words[1]
      id = parse(Int64,id)
      count = 4;
      for i=4:length(words)
        words[count] = lowercase(strip(words[count],[' ', '.', ',','#', '\'', ')', '(', '!', '/', '?', '\t', '`']))
        if length(words[count]) < 1
          deleteat!(words,count)
          count-=1;
        end
        count += 1;
      end
      push!(captions_dict, ((id,words[4:end]),count-4));
  end
  if sort
    captions_dict = sort!(captions_dict, by = tuple -> last(tuple), rev=false)
  end
  return captions_dict
end

function tokenize_mscoco_captions(captions)
    captions_dict = Array{Tuple{Tuple{Int64,Array{String,1}},Int64},1}()
    data = JSON.parse(captions[1])["annotations"]
    for obj in data
      words = split(obj["caption"],' ');
      count = 1;
      for i=1:length(words)
        words[count] = lowercase(strip(words[count],[' ', '.', ',', '#', '\'', ')', '(', '!', '/', '?', '\t', '`']))
        if length(words[count]) < 1
          deleteat!(words,count)
          count-=1;
        end
        count += 1;
      end
      id = obj["image_id"]
      push!(captions_dict, ((id,words),count-1));
    end
    captions_dict = sort(captions_dict, by = tuple -> last(tuple), rev=false)
    return captions_dict
end

function get_vocab(dict, vocab, word_counts)

  for data in dict
    for word in data[1][2]
          if length(word) > 0
            get!(vocab, word, 1+length(vocab));
            count = get!(word_counts,word,0)
            word_counts[word] = count+1;
          end
    end
  end
  return vocab

end

function filtervocab(threshold,vocab,word_counts)
  for (word,count) in word_counts
    if count < threshold
      delete!(vocab, word)
    end
  end

  temp = copy(vocab)
  empty!(vocab)

  get!(vocab,"~~",1+length(vocab)) #eos
  get!(vocab,"``",1+length(vocab)) #bos
  get!(vocab,"##",1+length(vocab))  #unk

  for (key,value) in temp
    get!(vocab, key, 1+length(vocab));
  end

  temp = 0
end

#vocab = get_vocab(dict1, vocab)

#println(length(vocab))
#vocab = get_vocab(dict2, vocab)
#println(length(vocab))
#vocab = get_vocab(dict3, vocab)
#println(length(vocab))
end

max_issues_repo_name: ekinakyurek/LRCN

max_forks_repo_name: ekinakyurek/LRCN


text:
import Base: insert!, length, delete!
export EventQueue, fetch_event!, insert_event!
using DataStructures

struct EventQueue{T}
    tree::AVLTree{Point{T}}
    segments::Dict{Point{T},Set{Segment{T}}} # maps upper point -> list of segments
end

length(Q::EventQueue) = length(Q.tree)

function delete!(Q::EventQueue, p::Point)
    delete!(Q.tree, p)
    delete!(Q.segments, p)
end

EventQueue() =
    EventQueue(AVLTree{Point{Float64}}(), Dict{Point{Float64},Set{Segment{Float64}}}())

function insert!(Q::EventQueue, segment::Segment)
    p = segment.p # guaranteed to be the upper point
    q = segment.q
    if p ∈ Q.tree
        push!(Q.segments[p], segment)
    else
        push!(Q.tree, p)
        Q.segments[p] = Set([segment])
    end
    if q ∉ Q.tree
        push!(Q.tree, q)
        Q.segments[q] = Set(Segment[]) # we only store for the upper
    end
end

function insert!(Q::EventQueue, p::Point)
    if p ∉ Q.tree
        push!(Q.tree, p)
    end
end

function EventQueue(segments::Vector{Segment{T}}) where {T<:AbstractFloat}
    Q = EventQueue()
    for segment in segments
        insert!(Q, segment)
    end
    return Q
end

function fetch_event!(Q::EventQueue)
    p = Q.tree[1]
    if p in keys(Q.segments)
        segments = Q.segments[p]
    else
        segments = Set{Segment{Float64}}()
    end
    delete!(Q, p)
    return Event(p, segments)
end