text:
###########
## ranks ##
###########

function ranks(x::Vector)
    ## transform observations to unit interval
    nObs = size(x, 1)

    y = ones(nObs)
    vals = ([1:nObs])/(nObs + 1)

    idx = sortperm(x)
    y[idx] = vals[:]
    return y
end

function ranks(x::Array)
    ## transform observations to unit interval
    nObs = size(x, 1)
    nVars = size(x, 2)

    y = ones(nObs, nVars)
    vals = ([1:nObs])/(nObs + 1)

    for ii=1:nVars
        idx = sortperm(x[:, ii])
        y[idx, ii] = vals[:]
    end
    return y
end

function ranks(da::DataVector)
    indNA = isna(da)
    vals = Econometrics.ranks(da[!indNA].data)
    daOut = DataArray(Float64, size(da, 1))
    daOut[!indNA] = vals
    return daOut
end

function ranks(tn::AbstractTimenum)
    df = DataFrame()
    for (nam, da) in eachcol(tn.vals)
        df[nam] = ranks(da)
    end
    return Timenum(df, idx(tn))
end

###############
## plotDistr ##
###############

function wstHist(x::Vector, nbins = 20; linewidth = 8)
    p = FramedPlot()
    setattr(p.y1, "draw_ticklabels", false)
    setattr(p.x1, "draw_ticklabels", false)
    ## setattr(p.x1, "ticks", 2)
    ## setattr(p.x1, "tickdir", -1)
    add(p, Histogram(hist(x, nbins)..., linewidth = linewidth))
    return p
end

function scatter2d(arr::Array{Float64, 2}; pointsize = 0.2)
    p = FramedPlot(aspect_ratio = 1)
    setattr(p.y1, "draw_ticklabels", false)
    setattr(p.x1, "draw_ticklabels", false)
    add(p, Points(arr[:, 1], arr[:, 2], kind="filled circle",
                  size=pointsize))
    return p
end

function plotVarName(smb::Symbol; textsize = 4)
    nam = string(smb)
    p = FramedPlot(aspect_ratio = 1,
                   xrange=(0,100),
                   yrange=(0,100))
    setattr(p.y1, "draw_ticklabels", false)
    setattr(p.x1, "draw_ticklabels", false)
    add(p, PlotLabel(.5, .5, nam, textangle = 45, size=textsize))
    return p
end

## function plotVarName(nam::String; textsize = 4)
##     p = FramedPlot(aspect_ratio = 1,
##                    xrange=(0,100),
##                    yrange=(0,100))
##     setattr(p.y1, "draw_ticklabels", false)
##     setattr(p.x1, "draw_ticklabels", false)
##     add(p, PlotLabel(.5, .5, nam, textangle = 45, size=textsize))
##     return p
## end

## function plotmatrix(arr::Array{Float64, 2},
##                     nams = [string(ii) for ii=1:size(arr,2)];
##                     textsize = 4, pointsize = 0.2, linewidth = 8)
##     nVars = size(arr, 2)
##     t = Table(nVars+1, nVars+1)
##     for ii=1:nVars
##         t[ii+1, ii+1] = wstHist(arr[!isnan(arr[:, ii]), ii],
##                                 linewidth = linewidth)
##     end
##     for ii=1:(nVars-1)
##         for jj=(ii+1):nVars
##             t[ii+1, jj+1] = scatter2d([arr[:, ii] arr[:, jj]],
##                                       pointsize = pointsize)
##             t[jj+1, ii+1] = scatter2d([arr[:, ii] arr[:, jj]],
##                                       pointsize = pointsize)
##         end
##     end
##     for ii=1:nVars
##         t[1, ii+1] = plotVarName(nams[ii], textsize = textsize)
##         t[ii+1, 1] = plotVarName(nams[ii], textsize = textsize)
##     end
##     setattr(t, "cellspacing", 0.1)
##     return t
## end

## lowlevel: data for both margins and dependency
##-----------------------------------------------

function plotDistr(arr::Array{Float64, 2}, U::Array{Float64, 2},
                   nams::Array{Symbol, 1};
                   pointsize = 0.2, textsize = 4, linewidth = 8)
    nVars = size(arr, 2)
    t = Table(nVars+1, nVars+1)
    for ii=1:nVars
        t[ii+1, ii+1] = wstHist(arr[!isnan(arr[:, ii]), ii],
                                linewidth = linewidth)
    end
    for ii=1:(nVars-1)
        for jj=(ii+1):nVars
            t[ii+1, jj+1] = scatter2d([U[:, ii] U[:, jj]],
                                      pointsize = pointsize)
            t[jj+1, ii+1] = scatter2d([U[:, ii] U[:, jj]],
                                      pointsize = pointsize)
        end
    end
    for ii=1:nVars
        t[1, ii+1] = plotVarName(nams[ii], textsize = textsize)
        t[ii+1, 1] = plotVarName(nams[ii], textsize = textsize)
    end
    setattr(t, "cellspacing", 0.1)
    return t
end

## functions with one data argument only
##--------------------------------------

## function plotDistr(arr::Array{Float64, 2},
##                    nams = [string(ii) for ii=1:size(arr,2)],
##                    args...)
##     ## calculate copula data within function
##     U = ranks(arr)
##     return plotDistr(arr, U, args...)
## end

## function plotDistr(tm::Timematr; args...)
##     ## calculate copula data within function
##     vals = asArr(tm, Float64)
##     U = ranks(vals)
##     return plotDistr(vals, U, names(tm), args...)
## end

function plotDistr(tn::Timenum; args...)
    ## calculate copula data within function
    Utmp = ranks(tn) # contains NAs
    U = asArr(Utmp, Float64, NaN)
    vals = asArr(tn, Float64, NaN)
    return plotDistr(vals, U, names(tn); args...)
end

## plotDistr(tn, textsize = 2, pointsize = 0.15, linewidth = 12)

## ## plotDistr(randn(1000, 20))
## plotmatrix(rand(100, 8), ["dklfskdfdkflshw", "sk", "sdk", "dskf",
##                           "sk", "sl", "lk", "kj"])

## import Winston.plot
## function plot(tc::Timecop)
##     ## plot bi-variate copula data
##     if size(tc, 2) != 2
##         error("copula plot is defined for bivariate data only")
##     end

##     vals = core(tc)
##     ## aspect ratio
##     p = FramedPlot(
##                    aspect_ratio=1,
##                    xrange=(0,1),
##                    yrange=(0,1))
##     plot(vals[:, 1], vals[:, 2], ".b")
## end


text:
module Experiments

export Experiment

using ..AlphaZero

using Base: @kwdef

"""
    Experiment

A structure that contains the information necessary to replicate a training session.

# Constructor

    Experiment(gspec, params, mknet, netparams, benchmarks)

- `gspec` is the specification of the game to be played
- `params` has type [`Params`](@ref)
- `mknet` is a neural network constructor taking arguments `(netparams, gspec)`
- `netparams` are the neural network hyperparameters
- `benchmark` is a vector of [`Benchmark.Duel`](@ref) to be used as a benchmark to track
  training progress.

"""
@kwdef struct Experiment
  name :: String
  gspec :: AbstractGameSpec
  params :: Params
  mknet :: Any
  netparams :: Any
  benchmark :: Vector{<:Benchmark.Evaluation}
end

end

ext: jl


text:
using Gadfly

n = 10
plot(x=rand(n), y=rand(n), color=sinh(rand(n)), Scale.color_asinh)


text:
"""
    ThreadedScans.linear!(op, xs::AbstractVector) -> xs

Compute inclusive scan. Spawn, wait, and propagation of intermediate reductions
are all done sequentially.

# Keyword arguments
* `ntasks = Threads.nthreads()`: number of tasks
"""
function ThreadedScans.linear!(
    op,
    xs;
    ntasks::Integer = Threads.nthreads() + 1,
    spin::Union{Nothing,Integer} = nothing,
)
    ntasks = min(length(xs), ntasks)
    if ntasks <= 1
        scan!(op, xs)
        return xs
    end

    it = partitionto(xs, ntasks)
    chunk0, state = iterate(it)
    y = iterate(it, state)
    if y === nothing
        scan!(op, xs)
        return xs
    end

    tasks = Task[]
    fromleft = OneWayPromise{eltype(xs)}()
    @assert !isempty(chunk0)
    t = Threads.@spawn try
        put!($fromleft, scan!(op, $chunk0))
    catch
        close($fromleft)
        rethrow()
    end
    push!(tasks, t)
    chunk, state = y
    while true
        y = iterate(it, state)
        y === nothing && break
        @assert !isempty(chunk)
        toright = OneWayPromise{eltype(xs)}()
        t = Threads.@spawn try
            local accr = foldl(op, $chunk)
            local accl = take!($fromleft, spin)
            put!($toright, op(accl, accr))
            scan!(op, accl, $chunk)
        catch
            close($fromleft)
            close($toright)
            rethrow()
        end
        push!(tasks, t)
        fromleft = toright
        chunk, state = y
    end
    try
        scan!(op, take!(fromleft, spin), chunk)
    finally
        close(fromleft)
        foreach(wait, tasks)
    end
    return xs
end


text:
module trial
import Parameters: @with_kw
import DataFrames: DataFrameRow, DataFrame
import JSONTables: jsontable

import jcontrol: Track, closest_point_idx, unwrap, kinematics_from_position, movingaverage
import ..comparisons: TrackSegment

export Trial, trimtrial, get_varvalue_at_frameidx

"""
Compute values of `s` based on position along a trial
"""
function get_trial_s(trial::DataFrameRow, track::Track)
    x, y = trial.body_x, trial.body_y
    n = length(x)

    S::Vector{Float64} = []
    for i in 1:n
        # get the closest track point
        idx = closest_point_idx(track.X, x[i], track.Y, y[i])
        push!(S, track.S[idx])
    end

    return S
end

"""
Store trial information
"""
struct Trial
    x::Vector{Float64}
    y::Vector{Float64}
    s::Vector{Float64}
    θ::Vector{Float64}
    ω::Vector{Float64}
    speed::Vector{Float64}
    u::Vector{Float64}
    v::Vector{Float64}
    duration::Float64
end

Base.show(io::IO, trial::Trial) = print(io, "Trial: $(round(trial.duration; digits=2))s")

"""
Construct Trial out of a dataframe entry.
"""
function Trial(trial::DataFrameRow, track::Track)
    s = get_trial_s(trial, track)

    # get orientation
    θ = unwrap(
        atan.(trial.snout_y .- trial.tail_base_y, trial.snout_x .- trial.tail_base_x)
    )
    θ = movingaverage(θ, 3)

    # get velocities
    speed, u, v, ω = kinematics_from_position(
        trial.body_x, trial.body_y, θ; fps=60, smooth=true, smooth_wnd=0.05
    )

    # remove artifacts
    ω = ω
    ω[abs.(ω) .> 15] .= 0

    # trim start to when speed is high enough
    start = findfirst(speed .> 15)
    start = isnothing(start) ? 1 : start

    return Trial(
        trial.body_x[start:end],    # x
        trial.body_y[start:end],    # y
        s[start:end],               # s
        θ[start:end],               # θ
        ω[start:end],               # ω
        speed[start:end],           # speed
        u[start:end],               # u
        v[start:end],               # v
        trial.duration,             # duration
    )
end

function Trial(filepath::String)
    open(filepath) do f
        data = jsontable(read(f))
        return Trial(
                data.x,             # x
                data.y,             # y
                data.s,             # s
                data.θ,             # θ
                data.ω,             # ω
                data.speed,         # speed
                data.u,             # u
                data.v,             # v
                data.duration[1]    # duration
            )
    end
end

# ----------------------------------- utils ---------------------------------- #
"""
Cut a trial to keep only the data between two s-values.
"""
function trimtrial(trial::Trial, s0, s1; by=:space)
    if by == :space
        start = findfirst(trial.s .>= s0)
        isnothing(start) && return nothing

        stop = findlast(trial.s .<= s1)
        isnothing(stop) && return nothing
    else
        start, stop = s0, s1
    end

    return Trial(
        trial.x[start:stop],        # x
        trial.y[start:stop],        # y
        trial.s[start:stop],        # s
        trial.θ[start:stop],        # θ
        trial.ω[start:stop],        # ω
        trial.speed[start:stop],    # speed
        trial.u[start:stop],        # u
        trial.v[start:stop],        # v
        (stop-start)/60,            # duration
    )
end

trimtrial(trial::Trial, seg::TrackSegment) = trimtrial(trial,260 * ( seg.s₀ - .01), 260 * (seg.s₁ - .01))

"""
    Get the value of each trial's variable at a frame index
"""
get_varvalue_at_frameidx(trials::Vector{Trial}, variable::Symbol, frame::Int) = map(t->getfield(t, variable)[frame], trials)

"""
    Get the value of each trial's variable at the last frame
"""
get_varvalue_at_frameidx(trials::Vector{Trial}, variable::Symbol) = map(t->getfield(t, variable)[end], trials)

end

max_stars_repo_name: FedeClaudi/LocomotionControl

max_issues_repo_name: FedeClaudi/LocomotionControl

max_forks_repo_name: FedeClaudi/LocomotionControl


text:
"""
Make Yt matrix for timestep t.
"""
function make_Yt_matrix end

"""
    make_Yt_matrix(t, data, f2i)

Make Yt matrix for timestep t. A given column of the Yt matrix specifies the support
for the corresponding n-gram predicted for timestep t for each of the observations (rows of Yt).

# Obligatory Arguments
- `t::Int64`: the timestep t
- `data::DataFrame`: the dataset
- `f2i::Dict`: the dictionary returning indices given features

# Optional Arguments
- `tokenized::Bool=false`: if true, the dataset target is assumed to be tokenized
- `sep_token::Union{Nothing, String, Char}=nothing`: separator token
- `verbose::Bool=false`: if verbose, more information will be printed

# Examples
```julia
latin = DataFrame(CSV.File(joinpath("data", "latin_mini.csv")))
JudiLing.make_Yt_matrix(2, latin)
```
"""
function make_Yt_matrix(
    t,
    data,
    f2i;
    grams = 3,
    target_col = "Words",
    tokenized = false,
    keep_sep = false,
    sep_token = nothing,
    start_end_token = "#",
)

    # split tokens from words or other columns
    if tokenized && !isnothing(sep_token)
        tokens = split.(data[:, target_col], sep_token)
    else
        tokens = split.(data[:, target_col], "")
    end

    # making ngrams from tokens
    # make_ngrams function are below
    ngrams = make_ngrams.(tokens, grams, keep_sep, sep_token, start_end_token)

    ngrams_l = length.(ngrams)

    # the number of non-zero values is equal to the number of utterances have
    # larger length than timestep t
    n_f = length(ngrams_l[ngrams_l.>=t])

    m = size(data, 1)
    n = length(f2i)
    I = zeros(Int64, n_f)
    J = zeros(Int64, n_f)
    V = ones(Int64, n_f)

    cnt = 0
    for (i, v) in enumerate(ngrams)
        if t <= length(v)
            cnt += 1
            I[cnt] = i
            J[cnt] = f2i[v[t]]
        end
    end

    sparse(I, J, V, m, n)

end


text:
using Base.Test
using AutoEnvs

function test_simple_ctor()
    srand(2)
    filepath = Pkg.dir("NGSIM", "data", "trajdata_i80_trajectories-0400-0415.txt")
    params = Dict(
        "trajectory_filepaths"=>[filepath],
        "H"=>40,
        "primesteps"=>5
    )
    env = NGSIMEnv(params)
    x = reset(env)
    nx, r, terminal, _ = step(env, [0.,0.])
    nx, r, terminal, _ = step(env, [0.,0.])
end

function test_basics()
    # ctor
    filepath = Pkg.dir("NGSIM", "data", "trajdata_i80_trajectories-0400-0415.txt")
    params = Dict("trajectory_filepaths"=>[filepath], "H"=>200)
    env = NGSIMEnv(params)

    # reset, step
    x = reset(env)
    a = [0., 0.]
    nx, r, terminal, infos = step(env, a)
    @test x != nx
    @test terminal == false
    nnx, r, terminal, infos = step(env, a)
    @test nx != nnx

    # obs spec
    shape, spacetype, infos = observation_space_spec(env)
    @test spacetype == "Box"
    @test in("high", keys(infos))
    @test in("low", keys(infos))

    # does accel reflect applied value
    fns = obs_names(env)
    acc_idx = [i for (i,n) in enumerate(fns) if "accel" == n][1]
    tur_idx = [i for (i,n) in enumerate(fns) if "turn_rate_global" == n][1]
    a = [0., 1.]
    for _ in 1:10
        nx, _, _, _ = step(env, a)
    end

    @test abs(nx[acc_idx]) <= 1e-1
    @test abs(nx[tur_idx] - 1) <= 1e-1

    # test infos
    reset(env)
    _, _, _, infos = step(env, [1.,1.])
    @test infos["rmse_pos"] != 0.

    # test reset with egoid
    x = reset(env; offset=250, egoid=3194, start=8886)
    nx = reset(env; offset=250, egoid=3194, start=8886)
    @test x == nx

end

function test_all_roadways()
    srand(2)
    filenames = [
        "trajdata_i80_trajectories-0400-0415.txt",
        "trajdata_i80_trajectories-0500-0515.txt",
        "trajdata_i80_trajectories-0515-0530.txt",
        "trajdata_i101_trajectories-0805am-0820am.txt",
        "trajdata_i101_trajectories-0820am-0835am.txt",
        "trajdata_i101_trajectories-0750am-0805am.txt"
    ]
    for fn in filenames
        filepath = Pkg.dir("NGSIM", "data", fn)
        params = Dict(
            "trajectory_filepaths"=>[filepath],
            "H"=>50,
            "primesteps"=>5
        )
        env = NGSIMEnv(params)
        x = reset(env)
    end
end

function test_render()
    srand(2)
    filepath = Pkg.dir("NGSIM", "data", "trajdata_i101_trajectories-0805am-0820am.txt")
    params = Dict(
        "trajectory_filepaths"=>[filepath],
        "H"=>200,
        "primesteps"=>50
    )
    env = NGSIMEnv(params)

    x = reset(env)
    imgs = []
    for _ in 1:100
        a = [1.,0.]
        img = render(env)
        x, r, terminal, _ = step(env, a)
        if terminal
            break
        end
    end
end

# @time test_simple_ctor()
@time test_basics()
# manual tests
# @time test_all_roadways()
# @time test_render()


text:
function train(traindata::Vector, testdata::Vector)
    for epoch = 1:20
        println("epoch: $(epoch)")
        loss = 0.0
        for i in randperm(length(traindata))
            #for x in traindata
            x = traindata[i]
            sy = State(x, scorefun)
            sz = State(x, scorefun)
            y = beamsearch(sy, 1, expand_gold)[end][1]
            z = beamsearch(sz, 1, expand_pred)[end][1]
            loss += z.score - y.score
            max_violation!(y, z, train_gold, train_pred)
        end
        println("loss: $(loss)")
        golds, preds = Int[], Int[]
        for x in testdata
            s = State(x, scorefun)
            z = beamsearch(s, 1, expand_pred)[end][1]
            append!(golds, map(t -> t.headid, x))
            append!(preds, toheads(z))
        end
        acc = accuracy(golds, preds)
        println("test acc: $(acc)")
        println("")
    end
end

max_stars_repo_name: hshindo/JukaiNLP

max_issues_repo_name: hshindo/JukaiNLP

max_forks_repo_name: hshindo/JukaiNLP


text:
@testset "optimal transport" begin

    P = randn(4, 5).^2 |> x -> x .= x / sum(x)

    M = 0.1randn(size(P))

    a, b = marginals(P)

    @test isprobability(P)
    @test !isprobability(P .+ 0.1)
    @test isprobability(a)
    @test isprobability(b)

    @test entropy(P[1,:] |> OTSIN.normalize) ≈ entropy(P[1,:], norm=true)

    @testset "free OT" begin
        Q = optimaltransport(M, λ=1.0)
        @test isprobability(Q)
        @test entropy(Q) > entropy(optimaltransport(M, λ=10.0))
        @test relative_entropy(P, Q) > 0
        @test Q ≈ optimaltransport(M, nothing, nothing, λ=1.0)
    end

    @testset "one marginal OT" begin
        Q = optimaltransport(M, a, λ=1.0)
        @test isprobability(Q)
        @test marginals(Q)[1] ≈ a
        @test Q ≈ optimaltransport(M, a, nothing, λ=1.0)
    end

    @testset "Sinkhorn" begin
        Q = optimaltransport(M, a, b, λ=1.0, ϵ=1e-10)
        @test isprobability(Q)
        @test all(marginals(P) .≈ marginals(Q))
        @test optimaltransport(M, a, b, λ=0) ≈ optimaltransport(a, b)
        @test Q ≈ optimaltransport(M, a, b, λ=1.0, ϵ=1e-10)
    end

    #  check if some elements of a and b can be zero
    @testset "Sinkhorn sparse" begin
        Q = optimaltransport(M, [0.5, 0.2, 0.0, 0.3],
                    [0.1, 0.1, 0.1, 0.0, 0.7], λ=1.0, ϵ=1e-10)
        a_, b_ = marginals(Q)
        @assert a_[3] < 1e-6
        @assert b_[4] < 1e-6
        @assert b_[1] ≈ 0.1
        @assert sum(Q) ≈ 1.0
    end

    @testset "perturbation" begin
        ∇af, ∇bf = perturbation(Q -> sum(Q.^2), M, a, b)
        @test ∇af ⋅ ones(size(∇af)) < 1e-6
        @test ∇bf ⋅ ones(size(∇bf)) < 1e-6
    end

end

max_stars_repo_name: PoisotLab/OTSIN

max_issues_repo_name: PoisotLab/OTSIN

max_forks_repo_name: PoisotLab/OTSIN


text:
# ---
# title: 1017. Convert to Base -2
# id: problem1017
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/convert-to-base-2/description/>
# hidden: true
# ---
#
# Given a number `N`, return a string consisting of `"0"`s and `"1"`s that
# represents its value in base `**-2**` (negative two).
#
# The returned string must have no leading zeroes, unless the string is `"0"`.
#
#
#
# **Example 1:**
#
#
#
#     Input: 2
#     Output: "110"
#     **Explantion:** (-2) ^ 2 + (-2) ^ 1 = 2
#
#
# **Example 2:**
#
#
#
#     Input: 3
#     Output: "111"
#     **Explantion:** (-2) ^ 2 + (-2) ^ 1 + (-2) ^ 0 = 3
#
#
# **Example 3:**
#
#
#
#     Input: 4
#     Output: "100"
#     **Explantion:** (-2) ^ 2 = 4
#
#
#
#
# **Note:**
#
#   1. `0 <= N <= 10^9`
#
#
## @lc code=start
using LeetCode

function base_neg2(N::Int)
    cnt = 0
    res = ""
    while N > 0
        bit = (N & 1)
        N -= bit * (-1) ^ cnt
        N >>= 1
        cnt += 1
        res = string(bit) * res
    end
    res
end
## @lc code=end