text:
export IntensityMap, fov, imagepixels, pixelsizes, stokes

"""
    $(TYPEDEF)
Image array type. This is an Matrix with a number of internal fields
to describe the field of view, pixel size, and the `pulse` function that
makes the image a continuous quantity.

To use it you just specify the array and the field of view/pulse
``julia
img = IntensityMap(zeros(512,512), 100.0, 100.0)
```
"""
struct IntensityMap{T,S<:AbstractMatrix, F, K<:Pulse} <: AbstractIntensityMap{T,S}
    im::S
    fovx::F
    fovy::F
    psizex::F
    psizey::F
    pulse::K
end

function IntensityMap(im, fovx, fovy, pulse=DeltaPulse())
    ny,nx = size(im)
    psizex=fovx/max(nx-1,1)
    psizey=fovy/max(ny-1,1)
    F = promote_type(typeof(fovx), typeof(fovy))
    return IntensityMap{eltype(im), typeof(im), F, typeof(pulse)}(im,
                       convert(typeof(psizex),fovx),
                       convert(typeof(psizey),fovy),
                       psizex,
                       psizey,
                       pulse)
end

# IntensityMap will obey the Comrade interface. This is so I can make easy models
visanalytic(::Type{<:IntensityMap}) = NotAnalytic()
imanalytic(::Type{<:IntensityMap}) = IsAnalytic()
isprimitive(::Type{<:IntensityMap}) = IsPrimitive()

function intensity_point(m::IntensityMap, x, y)
    dx, dy = pixelsizes(m)
    xitr,yitr = imagepixels(m)
    sum = zero(eltype(m))
    #The kernel is written in terms of pixel number so we convert x to it
    @inbounds for (i,xx) in pairs(xitr), (j,yy) in pairs(yitr)
        Δx = (x-xx)/dx
        Δy = (y-yy)/dy
        k = intensity_point(m.pulse, Δx, Δy)
        #println(Δx," ", Δy, " ",   k)
        sum += m[j, i]*k/(dx*dy)
    end
    return sum
end

function IntensityMap(im::Matrix{<:StokesVector}, fovx, fovy, pulse)
    return IntensityMap(StructArray(im), fovx, fovy, pulse)
end

function IntensityMap(im::IntensityMap{<:StokesVector}, fovx, fovy, pulse)
    return IntensityMap(im.im, fovx, fovy, pulse)
end

function stokes(im::IntensityMap{<:StokesVector}, p::Symbol)
    @assert p ∈ propertynames(im.im) "$p is not a valid stokes parameter"
    return IntensityMap(getproperty(im.im, p), im.fovx, im.fovy, im.pulse)
end

function IntensityMap(I::AbstractIntensityMap,
                      Q::AbstractIntensityMap,
                      U::AbstractIntensityMap,
                      V::AbstractIntensityMap
                      )
    @assert I.fovx == Q.fovx == U.fovx == V.fovx "Must have matching fov in RA"
    @assert I.fovy == Q.fovy == U.fovy == V.fovy "Must have matching fov in DEC"
    simg = StructArray{StokesVector{eltype(I)}}((I,Q,U,V))
    return IntensityMap(simg, I.fovx, I.fovy, I.pulse)
end

# function ChainRulesCore.rrule(::Type{<:IntensityMap}, im, fovx, fovy, pulse)
#     y = IntensityMap(im, fovx, fovy, pulse)
#     intensity_pullback(Δy) = (NoTangent(), IntensityMap(Δy.im, fovx, fovy, pulse), Δy.fovx, Δy.fovy, Δy.pulse)
#     return y, intensity_pullback
# end

fov(m::AbstractIntensityMap) = (m.fovx, m.fovy)

"""
    $(SIGNATURES)
Computes the flux of a intensity map
"""
function flux(im::AbstractIntensityMap{T,S}) where {T,S}
    f = sum(im.im)#*(flux(im.pulse))^2
    return f#*prod(pixelsizes(im))
end

"""
    $(SIGNATURES)
Computes the flux of a intensity map
"""
function flux(im::AbstractIntensityMap{T,S}) where {F,T<:StokesVector{F},S}
    I = stokes(im, :I)
    flux(I)
end

# Define the array interface
Base.IndexStyle(::Type{<: IntensityMap{T,S,K}}) where {T,S,K} = Base.IndexStyle(S)
Base.size(im::AbstractIntensityMap) = size(im.im)
Base.@propagate_inbounds Base.getindex(im::AbstractIntensityMap, i::Int) = getindex(im.im, i)
Base.@propagate_inbounds Base.getindex(im::AbstractIntensityMap, I...) = getindex(im.im, I...)
Base.@propagate_inbounds Base.setindex!(im::AbstractIntensityMap, x, i::Int) = setindex!(im.im, x, i)
Base.@propagate_inbounds Base.setindex!(im::IntensityMap, x, i::Int) = setindex!(im.im, x, i)

function Base.similar(im::IntensityMap, ::Type{T}) where{T}
    sim = similar(im.im, T)
    return IntensityMap(sim, im.fovx, im.fovy, im.pulse)
end

#function Base.similar(im::AbstractIntensityMap, ::Type{T}, dims::Dims) where {T}
#    fovx = im.psizex*last(dims)
#    fovy = im.psizey*first(dims)
#    sim = similar(im.im, T, dims)
#    return IntensityMap(sim, fovx, fovy, im.psizex, im.psizey)
#end

#Define the broadcast interface
struct IntensityMapStyle <: Broadcast.AbstractArrayStyle{2} end
IntensityMapStyle(::Val{2}) = IntensityMapStyle()

Base.BroadcastStyle(::Type{<:AbstractIntensityMap}) = IntensityMapStyle()
function Base.similar(bc::Broadcast.Broadcasted{IntensityMapStyle}, ::Type{ElType}) where ElType
    #Scan inputs for IntensityMap
    #print(bc.args)
    Im = _find_sim(bc)
    #fovxs = getproperty.(Im, Ref(:fovx))
    #fovys = getproperty.(Im, Ref(:fovy))
    #@assert all(i->i==first(fovxs), fovxs) "IntensityMap fov must be equal to add"
    #@assert all(i->i==first(fovys), fovys) "IntensityMap fov must be equal to add"
    return IntensityMap(similar(Array{ElType}, axes(bc)), Im.fovx, Im.fovy, Im.pulse)
end

#Finds the first IntensityMap and uses that as the base
#TODO: If multiply IntensityMaps maybe I should make sure they are consistent?
_find_sim(bc::Base.Broadcast.Broadcasted) = _find_sim(bc.args)
_find_sim(args::Tuple) = _find_sim(_find_sim(args[1]), Base.tail(args))
_find_sim(x) = x
_find_sim(::Tuple{}) = nothing
_find_sim(a::AbstractIntensityMap, rest) = a
_find_sim(::Any, rest) = _find_sim(rest)

#Guards to prevent someone from adding two Images with different FOV's
function Base.:+(x::AbstractIntensityMap, y::AbstractIntensityMap)
    @assert fov(x) == fov(y) "IntensityMaps must share same field of view"
    return x .+ y
end

@inline function pixelsizes(im::AbstractIntensityMap)
    ny,nx = size(im)
    return im.fovx/nx, im.fovy/ny
end

@inline function imagepixels(fovx, fovy, nx::Int, ny::Int)
    px = fovx/nx; py = fovy/ny
    return range(-fovx/2+px/2, step=px, length=nx),
           range(-fovy/2+py/2, step=py, length=ny)
end

@inline function imagepixels(im::AbstractIntensityMap)
    ny,nx = size(im)
    return imagepixels(im.fovx, im.fovy, nx, ny)
end

ext: jl


text:
#############################################################################
# Joulia
# A Large-Scale Spatial Power System Model for Julia
# See https://github.com/JuliaEnergy/Joulia.jl
#############################################################################
# This file contains functions and structs for storages

mutable struct Storages
	id
	node
	technology
	power
	storage
	efficiency
	vc

	function Storages(df::DataFrame)
		d = df_to_dict_with_id(df)

		new(df[:id], d[:Node], d[:Technology], d[:Power], d[:Storage],
			d[:Efficiency], d[:VariableCost])
	end

end


text:
"""
Generate the low-level `src/AWSServices.jl` file with all definitions.
"""
function _generate_low_level_wrappers(
    service_files::AbstractArray{ServiceFile}, auth::GitHub.Authorization
)
    service_definitions = sort!(_generate_low_level_definitions(service_files, auth))

    template = """
    # This file is auto-generated by AWSMetadata.jl

    module AWSServices

    using AWS
    using OrderedCollections: LittleDict

    $(join(service_definitions, "\n"))

    end
    """

    open(services_path, "w") do f
        print(f, template)
    end

    return template
end

"""
Get the low-level definitions for all AWS Services.
"""
function _generate_low_level_definitions(
    service_files::AbstractArray{ServiceFile}, auth::GitHub.Authorization
)
    low_level_defs = Vector{String}(undef, length(service_files))

    Threads.@threads for i in eachindex(service_files)
        service_file = service_files[i]
        service_name = service_file.name
        @info "Generating low level wrapper for $service_name"

        service = service_definition(service_file; auth=auth)
        service_metadata = service["metadata"]

        low_level_defs[i] = _generate_low_level_definition(service_metadata)
    end

    return low_level_defs
end

"""
Get the low-level definition for an AWS Service.
"""
function _generate_low_level_definition(service::AbstractDict)
    protocol = service["protocol"]
    endpoint_prefix = service["endpointPrefix"]
    signing_name =
        haskey(service, "signingName") ? service["signingName"] : service["endpointPrefix"]
    service_id = replace(lowercase(service["serviceId"]), ' ' => '_')
    api_version = service["apiVersion"]

    service_specifics = LittleDict{String,String}()

    if service_id == "glacier"
        service_specifics[service_id] = "LittleDict(\"x-amz-glacier-version\" => \"$(service["apiVersion"])\")"
    end

    if protocol == "rest-xml"
        return "const $service_id = AWS.RestXMLService(\"$signing_name\", \"$endpoint_prefix\", \"$api_version\")"
    elseif protocol in ("ec2", "query")
        return "const $service_id = AWS.QueryService(\"$signing_name\", \"$endpoint_prefix\", \"$api_version\")"
    elseif protocol == "rest-json" && haskey(service_specifics, service_id)
        return "const $service_id = AWS.RestJSONService(\"$signing_name\", \"$endpoint_prefix\", \"$api_version\", $(service_specifics[service_id]))"
    elseif protocol == "rest-json"
        return "const $service_id = AWS.RestJSONService(\"$signing_name\", \"$endpoint_prefix\", \"$api_version\")"
    elseif protocol == "json"
        json_version = service["jsonVersion"]
        target = service["targetPrefix"]
        return "const $service_id = AWS.JSONService(\"$signing_name\", \"$endpoint_prefix\", \"$api_version\", \"$json_version\", \"$target\")"
    else
        throw(
            ProtocolNotDefined(
                "$service_id is using a new protocol; $protocol which is not supported."
            ),
        )
    end
end


text:
module StructureLearner

using LogicCircuits
using ..Utils
using ..Probabilistic
using ..CSDD
using ..IO

export
# ChowLiuTree
learn_chow_liu_tree, parent_vector, print_tree, CLT,

# CircuitBuilder
compile_prob_circuit_from_clt, learn_probabilistic_circuit, BaseCache, ⊤, LitCache, learn_credal_circuit,

# PSDDInitializer
learn_struct_prob_circuit,
learn_vtree_from_clt, compile_psdd_from_clt,compile_fully_factorized_psdd_from_vtree,
learn_struct_credal_circuit

include("ChowLiuTree.jl")
include("CircuitBuilder.jl")
include("PSDDInitializer.jl")

end

max_stars_repo_name: IDSIA/cjuice

max_issues_repo_name: IDSIA/cjuice

max_forks_repo_name: IDSIA/cjuice


text:
using Test, GeoDatasets

@testset "landseamask" begin
    lon,lat,data = GeoDatasets.landseamask(;resolution='c',grid=5)
    @test size(data,1) == length(lon)
    @test size(data,2) == length(lat)

    @test_throws ErrorException GeoDatasets.landseamask(;resolution='c',grid=-999)
    @test_throws ErrorException GeoDatasets.landseamask(;resolution='g',grid=5)
end

@testset "gshhg" begin
    res = 'c'
    @test length(GeoDatasets.gshhg(res,[1,5])) > 300
    @test length(GeoDatasets.WDBII_river(res,[1,2])) > 1000
    @test length(GeoDatasets.WDBII_border(res,[1])) >  100

    # resolution 'x' does not exists
    @test_throws ErrorException GeoDatasets.gshhg('x',[1,5])
end


text:
abstract type Process{T} end

struct DateProcess <: Process{Date}
end

struct ConstProcess{T} <: Process{T}
    val::T
end

struct CondProcess{T} <: Process{T}
    cond::Process{Bool}
    a::Process{T}
    b::Process{T}
end

proctype(::Process{T}) where {T} = T
proctype(::Tuple{Process{T1}}) where {T1} = Tuple{T1}
proctype(::Tuple{Process{T1},Process{T2}}) where {T1,T2} = Tuple{T1,T2}

struct LiftProc{F,A,R} <: Process{R}
    f::F
    a::A
    function LiftProc{F,A,R}(f,a...) where {F,A,R}
        new{F,A,R}(f,a)
    end
end

function LiftProc(f::Function, a::Process...)
    RR = Base.return_types(f,proctype(a))
    R = length(RR) > 1 ? Any : RR[1]
    LiftProc{typeof(f),typeof(a),R}(f,a...)
end

struct DiscountProc{T,Pr<:Process{Float64},Pp<:Process{Bool},Pv<:Process} <: Process{T}
    r::Pr
    p::Pp
    v::Pv
end
DiscountProc(r::Process{Float64},p::Process{Bool},v::Process{T}) where {T} =  DiscountProc{T,typeof(r),typeof(p),typeof(v)}(r,p,v)


text:
module MyMod

export hi, inc, fac, fib;

hi() = "Hello"
hi(who::String) = "Hello, $who"

inc(x::Number) = x + 1

function fac(n::Integer)
  @assert n > 0
  (n == 1) ? 1 : n*fac(n-1);
end

function fib_helper(a ::Integer, b::Integer, n::Integer)
   (n > 0) ? fib_helper(b, a+b, n-1) : a
end

function fib(n::Integer)
   @assert n > 0
   fib_helper(0, 1, n)
end

end

max_stars_repo_path: Module 3/Chapter10/MyMod.jl

max_issues_repo_path: Module 3/Chapter10/MyMod.jl

max_forks_repo_path: Module 3/Chapter10/MyMod.jl


text:
##############################################################################
##
## The following function follows the command ranktest (called by ivreg2)
## RANKTEST: Stata module to test the rank of a matrix using the Kleibergen-Paap rk statistic
## Authors: Frank Kleibergen, Mark E Schaffer
## IVREG2: Stata module for extended instrumental variables/2SLS and GMM estimation
## Authors: Christopher F Baum, Mark E Schaffer, Steven Stillman
## More precisely, it corresponds to the Stata command:  ranktest  (X) (Z), wald full
##############################################################################

function ranktest!(X::Matrix{Float64},
                    Z::Matrix{Float64},
                    Pi::Matrix{Float64},
                    vcov_method_data::AbstractVcovMethodData,
                    df_small::Int,
                    df_absorb::Int)

    K = size(X, 2)
    L = size(Z, 2)

    crossz = cholfact!(At_mul_B(Z, Z), :U)
    crossx = cholfact!(At_mul_B(X, X), :U)

    Fmatrix = crossz[:U]
    Gmatrix = crossx[:U]
    theta = Fmatrix * (Gmatrix' \ Pi')'

    svddecomposition = svdfact(theta, thin = false)
    u = svddecomposition.U
    vt = svddecomposition.Vt

    # compute lambda
    if K == 1
        a_qq = sqrtm(A_mul_Bt(u, u))
        b_qq = sqrtm(A_mul_Bt(vt, vt))
    else
        u_12 = u[1:(K-1),(K:L)]
        v_12 = vt[1:(K-1),K]
        u_22 = u[(K:L),(K:L)]
        v_22 = vt[K,K]
        a_qq = vcat(u_12, u_22) * (u_22 \ sqrtm(A_mul_Bt(u_22, u_22)))
        b_qq = sqrtm(A_mul_Bt(v_22, v_22)) * (v_22' \ vcat(v_12, v_22)')
    end
    kronv = kron(b_qq, a_qq')
    lambda = kronv * vec(theta)

    # compute vhat
    if typeof(vcov_method_data) == VcovSimpleData
        vhat= eye(L * K) / size(X, 1)
    else
        temp1 = convert(Matrix{eltype(Gmatrix)}, Gmatrix)
        temp2 = convert(Matrix{eltype(Fmatrix)}, Fmatrix)
        k = kron(temp1, temp2)'
        vcovmodel = VcovData(Z, k, X, size(Z, 1) - df_small - df_absorb)
        matrix_vcov2 = shat!(vcov_method_data, vcovmodel)
        vhat = k \ (k \ matrix_vcov2)'
    end

    # return statistics
    # why do I need to add Hermitian? (since 0.5)
    if VERSION > v"0.5.0-dev+961"
        vlab = cholfact!(Hermitian(A_mul_Bt(kronv * vhat, kronv)))
    else
        vlab = cholfact!(A_mul_Bt(kronv * vhat, kronv))
    end
    r_kp = lambda' * (vlab \ lambda)
    p_kp = ccdf(Chisq((L-K+1 )), r_kp[1])
    F_kp = r_kp[1] / size(Z, 2)
    return F_kp, p_kp
end


text:
"""
Print information about Julia threads, e.g. on which cpu threads (i.e. cores if hyperthreading is disabled) they are running.

Keyword arguments:
* `color` (default: `true`): Toggle between colored and black-and-white output.
* `blocksize` (default: `32`): Wrap to a new line after `blocksize` many cpu threads.
* `hyperthreading` (default: `true`): If `true`, we (try to) highlight cpu threads associated with hyperthreading in the `color=true` output.
* `blas` (default: `false`): Show information about BLAS threads as well.
* `hints` (default: `false`): Give some hints about how to improve the threading related settings.
* `groupby` (default: `:sockets`): Options are `:sockets`, `:numa`, or `:none`.
"""
function threadinfo(; blas=false, hints=false, color=true, kwargs...)
    maybe_gather_sysinfo()
    # general info
    jlthreads = Threads.nthreads()
    thread_cpuids = getcpuids()
    occupied_cputhreads = length(unique(thread_cpuids))
    cputhreads = Sys.CPU_THREADS
    # visualize current pinning
    println()
    _visualize_affinity(; thread_cpuids, color, kwargs...)
    print("Julia threads: ")
    if color
        printstyled(jlthreads, "\n"; color=jlthreads > cputhreads ? :red : :green)
    else
        printstyled(jlthreads, jlthreads > cputhreads ? "(!)" : "", "\n")
    end
    print("├ Occupied CPU-threads: ")
    if color
        printstyled(
            occupied_cputhreads, "\n"; color=occupied_cputhreads < jlthreads ? :red : :green
        )
    else
        printstyled(occupied_cputhreads, occupied_cputhreads < jlthreads ? "(!)" : "", "\n")
    end
    print("└ Mapping (Thread => CPUID):")
    # print("   ")
    for (tid, core) in pairs(thread_cpuids)
        print(" $tid => $core,")
        if tid == 5
            print(" ...")
            break
        end
    end
    println("\n")
    if blas
        libblas = BLAS_lib()
        println("BLAS: ", libblas)
        if contains(libblas, "openblas")
            print("└ openblas_get_num_threads: ")
            if color
                printstyled(
                    BLAS.get_num_threads(), "\n"; color=_color_openblas_num_threads()
                )
            else
                printstyled(
                    BLAS.get_num_threads(),
                    _color_openblas_num_threads() == :red ? "(!)" : "",
                    "\n",
                )
            end
            println()
            _color_openblas_num_threads(; hints)
        elseif contains(libblas, "mkl")
            print("├ mkl_get_num_threads: ")
            if color
                printstyled(BLAS.get_num_threads(), "\n"; color=_color_mkl_num_threads())
            else
                printstyled(
                    BLAS.get_num_threads(),
                    _color_mkl_num_threads() == :red ? "(!)" : "",
                    "\n",
                )
            end
            println("└ mkl_get_dynamic: ", Bool(mkl_get_dynamic()))
            println()
            _color_mkl_num_threads(; hints)
        end
    end
    hints && _general_hints()
    return nothing
end

function _visualize_affinity(;
    thread_cpuids=getcpuids(),
    blocksize=16,
    color=true,
    groupby=:sockets,
    hyperthreading=hyperthreading_is_enabled(),
)
    ncpuids = Sys.CPU_THREADS
    cpuids_grouped = if groupby in (:sockets, :socket)
        cpuids_per_socket()
    elseif groupby == :numa
        cpuids_per_numa()
    else
        [cpuids_all()]
    end
    printstyled("| "; bold=true)
    for (i, cpuids) in pairs(cpuids_grouped)
        for (k, cpuid) in pairs(cpuids)
            if color
                if cpuid in thread_cpuids
                    printstyled(
                        cpuid;
                        bold=true,
                        color=if (hyperthreading && ishyperthread(cpuid))
                            :light_magenta
                        else
                            :yellow
                        end,
                    )
                else
                    printstyled(
                        cpuid; color=if (hyperthreading && ishyperthread(cpuid))
                            :light_black
                        else
                            :default
                        end
                    )
                end
            else
                if cpuid in thread_cpuids
                    printstyled(cpuid; bold=true)
                else
                    print("_")
                end
            end
            if !(cpuid == last(cpuids))
                print(",")
                mod(k, blocksize) == 0 && print("\n  ")
            end
        end
        # print(" | ")
        if ncpuids > 32
            printstyled(" |"; bold=true)
            if !(i == length(cpuids_grouped))
                println()
                printstyled("| "; bold=true)
            end
        else
            printstyled(" | "; bold=true)
        end
    end
    println()
    # legend
    println()
    if color
        printstyled("#"; bold=true, color=:yellow)
    else
        printstyled("#"; bold=true)
    end
    print(" = Julia thread, ")
    if hyperthreading
        printstyled("#"; color=:light_black)
        print(" = HT, ")
        printstyled("#"; bold=true, color=:light_magenta)
        print(" = Julia thread on HT, ")
    end
    if groupby in (:sockets, :socket)
        printstyled("|"; bold=true)
        print(" = Socket seperator")
    elseif groupby == :numa
        printstyled("|"; bold=true)
        print(" = NUMA seperator")
    end
    println("\n")
    return nothing
end

function _color_mkl_num_threads(; hints=false)
    jlthreads = Threads.nthreads()
    cputhreads = Sys.CPU_THREADS
    cputhreads_per_jlthread = floor(Int, cputhreads / jlthreads)
    blasthreads_per_jlthread = BLAS.get_num_threads()
    if blasthreads_per_jlthread == 1
        if jlthreads < Sys.CPU_THREADS
            hints && @info(
                "blasthreads_per_jlthread == 1 && jlthreads < cputhreads. You should set BLAS.set_num_threads($cputhreads_per_jlthread) or try to increase the number of Julia threads to $cputhreads."
            )
            return :yellow
        elseif jlthreads == cputhreads
            return :green
        else
            hints && @warn(
                "jlthreads > cputhreads. You should decrease the number of Julia threads to $cputhreads."
            )
            return :red
        end
    elseif blasthreads_per_jlthread < cputhreads_per_jlthread
        hints && @info(
            "blasthreads_per_jlthread < cputhreads_per_jlthread. You should increase the number of MKL threads, i.e. BLAS.set_num_threads($cputhreads_per_jlthread)."
        )
        return :yellow
    elseif blasthreads_per_jlthread == cputhreads_per_jlthread
        return :green
    else
        hints && @warn(
            "blasthreads_per_jlthread > cputhreads_per_jlthread. You should decrease the number of MKL threads, i.e. BLAS.set_num_threads($cputhreads_per_jlthread)."
        )
        return :red
    end
end

function _color_openblas_num_threads(; hints=false)
    # BLAS uses `blasthreads` many threads in total
    cputhreads = Sys.CPU_THREADS
    blasthreads = BLAS.get_num_threads()
    jlthreads = Threads.nthreads()
    if jlthreads != 1
        if blasthreads == 1
            return :green
        else
            # Not sure about this case...
            if blasthreads < jlthreads
                hints && @warn(
                    "jlthreads != 1 && blasthreads < jlthreads. You should set BLAS.set_num_threads(1)."
                )
                return :red
            elseif blasthreads < cputhreads
                hints && @info(
                    "jlthreads != 1 && blasthreads < cputhreads. You should either set BLAS.set_num_threads(1) (recommended!) or at least BLAS.set_num_threads($cputhreads)."
                )
                return :yellow
            elseif blasthreads == cputhreads
                hints && @info(
                    "For jlthreads != 1 we strongly recommend to set BLAS.set_num_threads(1)."
                )
                return :green
            else
                hints && @warn(
                    "jlthreads != 1 && blasthreads > cputhreads. You should set BLAS.set_num_threads(1) (recommended!) or at least BLAS.set_num_threads($cputhreads)."
                )
                return :red
            end
        end
    else
        # single Julia thread
        if blasthreads < cputhreads
            hints && @info(
                "blasthreads < cputhreads. You should increase the number of OpenBLAS threads, i.e. BLAS.set_num_threads($cputhreads)."
            )
            return :yellow
        elseif blasthreads == cputhreads
            return :green
        else
            hints && @warn(
                "blasthreads > cputhreads. You should decrease the number of OpenBLAS threads, i.e. BLAS.set_num_threads($corse)."
            )
            return :red
        end
    end
end

function _general_hints()
    jlthreads = Threads.nthreads()
    cputhreads = Sys.CPU_THREADS
    thread_cpuids = getcpuids()
    if jlthreads > cputhreads
        @warn(
            "jlthreads > cputhreads. You should decrease the number of Julia threads to $cputhreads."
        )
    elseif jlthreads < cputhreads
        @info(
            "jlthreads < cputhreads. Perhaps increase number of Julia threads to $cputhreads?"
        )
    end
    if length(unique(thread_cpuids)) < jlthreads
        @warn("Overlap: Some Julia threads are running on the same cpu threads")
    end
    return nothing
end


text:
"""
Contents: Functions for comparing a table to a schema that are used for more than one table type.
"""
module common

export assess_singlecolumn_primarykey!, assess_multicolumn_primarykey!,
       parserow!,
       assess_row!, assess_missing_value!, assess_nonmissing_value!,
       datacols_match_schemacols!, test_intrarow_constraints!,
       init_outdata, init_issues, construct_issues_table

using DataAPI
using DataFrames
using CategoricalArrays
using Tables

using ..handle_validvalues
using ..types

import Base.getindex
getindex(cr::Tables.ColumnsRow, nm::Symbol) = getproperty(cr, nm)  # For testing intrarow constraints

################################################################################
# Detecting incomplete and/or duplicated primary keys

"Modified: all_issues."
function assess_singlecolumn_primarykey!(all_issues, pk_issues, pk_n_missing::Int, pk_n_notunique::Int)
    pk_incomplete = pk_issues[:n_missing]   > pk_n_missing    # If true then the current inputrow has missing primary key
    pk_duplicated = pk_issues[:n_notunique] > pk_n_notunique  # If true then the current inputrow has duplicated primary key
    if pk_incomplete
        all_issues[:primarykey_incomplete] += 1
    end
    if pk_duplicated
        all_issues[:primarykey_duplicates] += 1
    end
    pk_incomplete, pk_duplicated
end

"Modified: all_issues, primarykey, pkvalues."
function assess_multicolumn_primarykey!(all_issues, primarykey, pk_colnames, pkvalues, sorted_by_primarykey, row)
    if sorted_by_primarykey
        pk_incomplete, pk_duplicated = populate_and_assess_sorted_primarykey!(primarykey, pk_colnames, row)
    else
        pk_incomplete, pk_duplicated = populate_and_assess_unsorted_primarykey!(primarykey, pk_colnames, row, pkvalues)
    end
    if pk_incomplete
        all_issues[:primarykey_incomplete] += 1
    end
    if pk_duplicated
        all_issues[:primarykey_duplicates] += 1
    end
    pk_incomplete, pk_duplicated
end

"""
Modified: pk_prev

Returns: pk_incomplete::Bool, pk_duplicated::Bool.

Used when the input data is sorted by its primary key and the primary key has more than 1 column.

Compares the current row's primary key to the previous row's primary key.
Checks for completeness and duplication.
After doing the checks, pk_prev is populated with the current row's primary key.
"""
function populate_and_assess_sorted_primarykey!(pk_prev::Vector{Union{String, Missing}}, pk_colnames::Vector{Symbol}, currentrow)
    pk_incomplete = false
    pk_duplicated = true
    for (j, colname) in enumerate(pk_colnames)
        val = getproperty(currentrow, colname)
        if ismissing(val)
            pk_incomplete = true
            pk_duplicated = false
        else
            val = string(val)
        end
        @inbounds prev_val = pk_prev[j]
        if ismissing(prev_val) || val != prev_val
            pk_duplicated = false
        end
        @inbounds pk_prev[j] = val
    end
    pk_incomplete, pk_duplicated
end

"""
Modified: primarykey and pkvalues (if the row's primary is complete and hasn't been seen in an eariler row).

Returns: pk_incomplete::Bool, pk_duplicated::Bool.

Used when the input data is not sorted by its primary key and the primary key has more than 1 column.
"""
function populate_and_assess_unsorted_primarykey!(primarykey::Vector{Union{String, Missing}}, pk_colnames::Vector{Symbol}, row, pkvalues)
    pk_incomplete = false
    pk_duplicated = true
    for (j, colname) in enumerate(pk_colnames)
        val = getproperty(row, colname)
        if ismissing(val)
            @inbounds primarykey[j] = missing
            pk_incomplete = true
            pk_duplicated = false
        else
            @inbounds primarykey[j] = string(val)
        end
    end
    if !pk_incomplete
        pkvalue = join(primarykey)
        if !in(pkvalue, pkvalues)  # This pkvalue hasn't already been seen in an earlier row
            push!(pkvalues, pkvalue)
            pk_duplicated = false
        end
    end
    pk_incomplete, pk_duplicated
end

################################################################################
# Transforming input values to output values

"""
Modified: outputrow

Parse inputrow into outputrow according to the colschema.
"""
function parserow!(outputrow, inputrow, colname2colschema)
    for (colname, colschema) in colname2colschema
        @inbounds outputrow[colname] = parsevalue(colschema.datatype, colschema.parser, getproperty(inputrow, colname))
    end
end

"Values that cannot be parsed are set to missing."
parsevalue(datatype::DataType, parser::Function, value) = value isa datatype ? value : parser(value)  # Default method
parsevalue(datatype::DataType, parser::Function, value::Missing)          = missing
parsevalue(datatype::DataType, parser::Function, value::String)           = value isa datatype ? value : parser(value)
parsevalue(datatype::DataType, parser::Function, value::AbstractString)   = parsevalue(datatype, parser, String(value))
parsevalue(datatype::DataType, parser::Function, value::CategoricalValue) = parsevalue(datatype, parser, DataAPI.unwrap(value))
parsevalue(datatype::DataType, parser::Function, value::CategoricalValue{String, <:Integer}) = parsevalue(datatype, parser, DataAPI.unwrap(value))

function parsevalue(datatype::DataType, parser::Function, value::Integer)
    value isa datatype && return value
    if datatype <: Signed      # Int
        value <= typemax(datatype) && return convert(datatype, value)  # Example: convert(Int32, 123)
        return missing
    elseif datatype <: Integer # UInt or Bool
        value >= 0 && value <= typemax(datatype) && return convert(datatype, value)  # Example: convert(UInt, 123)
        return missing
    elseif datatype <: AbstractFloat
        value <= typemax(datatype) && return convert(datatype, value)  # Example: convert(Float64, 123)
        return missing
    else
        return missing
    end
end

function parsevalue(datatype::DataType, parser::Function, value::AbstractFloat)
    value isa datatype && return value
    if datatype <: Integer
        value != round(value; digits=0) && return missing  # InexactError
        value >= 0.0 && value <= typemax(datatype) && return convert(datatype, value)
        datatype <: Signed && value <= typemax(datatype) && return convert(datatype, value)
        return missing  # value < 0 and datatype <: Unsigned...not possible
    elseif datatype <: AbstractFloat
        value <= typemax(datatype) && return convert(datatype, value)  # Example: convert(Float32, 12.3)
        return missing
    else
        return missing
    end
end

################################################################################
# Cell-level non-compliance issues

"Records issues with the row but doesn't mutate the row."
function assess_row!(columnissues, row, colname2colschema, uniquevalues)
    for (colname, colschema) in colname2colschema
        val = getproperty(row, colname)
        if ismissing(val)
            !colschema.isrequired && continue  # Checks are done before the function call to avoid unnecessary dict lookups
            assess_missing_value!(columnissues[colname])
        elseif value_is_valid(val, colschema.validvalues)
            !colschema.isunique && continue
            assess_nonmissing_value!(columnissues[colname], val, uniquevalues[colname])
        else
            columnissues[colname][:n_invalid] += 1
            !colschema.isunique && continue
            assess_nonmissing_value!(columnissues[colname], val, uniquevalues[colname])
        end
    end
end

assess_missing_value!(columnissues::Dict{Symbol, Int}) = columnissues[:n_missing] += 1  # Ensure no missing data

function assess_nonmissing_value!(columnissues::Dict{Symbol, Int}, value, uniquevalues_colname)
    if in(value, uniquevalues_colname)  # Ensure unique data
        columnissues[:n_notunique] += 1
    else
        push!(uniquevalues_colname, value)
    end
end

function assess_nonmissing_value!(columnissues::Dict{Symbol, Int}, value::T, uniquevalues_colname) where {T <: CategoricalValue}
    assess_nonmissing_value!(columnissues, get(value), uniquevalues_colname)
end

function assess_nonmissing_value!(columnissues::Dict{Symbol, Int}, value::T, uniquevalues_colname) where {T <: CategoricalValue{String, <:Integer}}
    assess_nonmissing_value!(columnissues, get(value), uniquevalues_colname)
end

################################################################################
# Column-level non-compliance issues

"Ensure the set of columns in the data matches that in the schema."
function datacols_match_schemacols!(issues, tableschema::TableSchema, colnames_data::Set{Symbol})
    tablename       = String(tableschema.name)
    colnames_schema = Set(tableschema.columnorder)
    cols = setdiff(colnames_data, colnames_schema)
    if length(cols) > 0
        issues[:data_extra_cols] = sort!([x for x in cols])
    end
    cols = setdiff(colnames_schema, colnames_data)
    if length(cols) > 0
        issues[:data_missing_cols] = sort!([x for x in cols])
    end
end

################################################################################
# Detecting for breaches of intra-row constraints

"""
Modified: constraint_issues.
"""
function test_intrarow_constraints!(constraint_issues::Dict{String, Int}, tableschema::TableSchema, row)
    for (msg, f) in tableschema.intrarow_constraints
        ok = @eval $f($row)        # Hack to avoid world age problem.
        ismissing(ok) && continue  # This case is picked up at the column level
        ok && continue
        if haskey(constraint_issues, msg)
            constraint_issues[msg] += 1
        else
            constraint_issues[msg] = 1
        end
    end
end

################################################################################
# Initialise output data

"Returns: A table with unpopulated columns with name, type, length and order matching the table schema."
function init_outdata(tableschema::TableSchema, n::Int)
    result = DataFrame()
    for colname in tableschema.columnorder
        colschema = tableschema.colname2colschema[colname]
        result[!, colname] = missings(colschema.datatype, n)
    end
    result
end

"""
Initialises outdata for streaming tables.
Estimates the number of required rows using: filesize = bytes_per_row * nrows
"""
function init_outdata(tableschema::TableSchema, input_data_file::String)
    bytes_per_row = 0
    for (colname, colschema) in tableschema.colname2colschema
        bytes_per_row += colschema.datatype == String ? 30 : 8  # Allow 30 bytes for Strings, 8 bytes for all other data types
    end
    nr = Int(ceil(filesize(input_data_file) / bytes_per_row))
    init_outdata(tableschema, min(nr, 1_000_000))
end

################################################################################
# Reporting non-compliance issues

function init_issues(tableschema::TableSchema)
    result = Dict(:primarykey_duplicates => 0, :primarykey_incomplete => 0, :data_extra_cols => Symbol[], :data_missing_cols => Symbol[])
    result[:intrarow_constraints] = Dict{String, Int}()
    result[:columnissues] = Dict{Symbol, Dict{Symbol, Int}}()
    for (colname, colschema) in tableschema.colname2colschema
        d = Dict(:n_notunique => 0, :n_missing => 0, :n_invalid => 0, :different_datatypes => 0, :data_not_categorical => 0, :data_is_categorical => 0)
        result[:columnissues][colname] = d
    end
    result
end

"""
Returns: Converts issues object to a formatted and sorted DataFrame.
"""
function construct_issues_table(issues, tableschema, ntotal)
    result    = NamedTuple{(:entity, :id, :issue), Tuple{String, String, String}}[]
    tablename = tableschema.name
    if !isempty(issues[:data_extra_cols])
        push!(result, (entity="table", id="$(tablename)", issue="The data has columns that the schema doesn't have ($(issues[:data_extra_cols]))."))
    end
    if !isempty(issues[:data_missing_cols])
        push!(result, (entity="table", id="$(tablename)", issue="The data is missing some columns that the Schema has ($(issues[:data_missing_cols]))."))
    end
    if issues[:primarykey_incomplete] > 0
        num = issues[:primarykey_incomplete]
        p   = make_pct_presentable(100.0 * num / ntotal)
        push!(result, (entity="table", id="$(tablename)", issue="$(p)% ($(num)/$(ntotal)) of rows contain missing values in the primary key."))
    end
    if issues[:primarykey_duplicates] > 0
        nd = issues[:primarykey_duplicates]
        p  = make_pct_presentable(100.0 * nd / ntotal)
        push!(result, (entity="table", id="$(tablename)", issue="$(p)% ($(nd)/$(ntotal)) of rows contain duplicated values of the primary key."))
    end
    for (constraint, nr) in issues[:intrarow_constraints]
        p = make_pct_presentable(100.0 * nr / ntotal)
        msg = "$(p)% ($(nr)/$(ntotal)) of rows don't satisfy the constraint: $(constraint)"
        push!(result, (entity="table", id="$(tablename)", issue=msg))
    end
    for (colname, colschema) in tableschema.colname2colschema
        !haskey(issues[:columnissues], colname) && continue
        d = issues[:columnissues][colname]
        if d[:different_datatypes] > 0
            push!(result, (entity="column", id="$(tablename).$(colname)", issue="The schema requires the data to have type $(colschema.datatype)."))
        end
        if d[:data_not_categorical] > 0
            push!(result, (entity="column", id="$(tablename).$(colname)", issue="The data is not categorical."))
        end
        if d[:data_is_categorical] > 0
            push!(issues, (entity="column", id="$(tablename).$(colname)", issue="The data is categorical but shouldn't be."))
        end
        store_column_issue!(result, tablename, colname, d[:n_missing],   ntotal, "have missing data")
        store_column_issue!(result, tablename, colname, d[:n_notunique], ntotal, "are duplicates")
        store_column_issue!(result, tablename, colname, d[:n_invalid],   ntotal, "contain invalid values")
    end
    result = DataFrame(result)
    sort!(result, [:entity, :id, :issue], rev=(true, false, false))
end

"Example: 25% (250/1000) of rows contain invalid values."
function store_column_issue!(issues, tablename, colname, num, ntotal, msg_suffix)
    num == 0 && return
    p = make_pct_presentable(100.0 * num / ntotal)
    push!(issues, (entity="column", id="$(tablename).$(colname)", issue="$(p)% ($(num)/$(ntotal)) of rows $(msg_suffix)."))
end

function make_pct_presentable(p)
    p >= 100.0 && return "100"
    p >= 1.0   && return "$(round(Int, p))"
    p >= 0.1   && return "$(round(p; digits=1))"
    "<0.1"
end

end


text:
# usage example for package developer:
#
#     import EdgeDetectionAPI: AbstractEdgeDetectionAlgorithm,
#                             detect_edges, detect_edges!

"""
    AbstractEdgeDetectionAlgorithm <: AbstractImageFilter

The root type for `ImageEdgeDetection` package.

Any concrete edge detection algorithm shall subtype it to support
[`detect_edges`](@ref) and [`detect_edges!`](@ref) APIs.

# Examples

All edge detection algorithms in ImageEdgeDetection are called in the
following pattern:

```julia
# first generate an algorithm instance
f = Canny()

# then pass the algorithm to `detect_edges`
img_edges = detect_edges(img, f)

# or use in-place version `thin_edges!`
img_edges = similar(img)
detect_edges!(img_edges, img, f)
```

For more examples, please check [`detect_edges`](@ref),
[`detect_edges!`](@ref) and concrete algorithms.
"""
abstract type AbstractEdgeDetectionAlgorithm <: AbstractImageFilter end

detect_edges!(out::Union{GenericGrayImage, AbstractArray{<:Color3}},
          img,
          f::AbstractEdgeDetectionAlgorithm,
          args...; kwargs...) =
    f(out, img, args...; kwargs...)

# TODO: Relax this to all color types
function detect_edges!(img::Union{GenericGrayImage, AbstractArray{<:Color3}},
                   f::AbstractEdgeDetectionAlgorithm,
                   args...; kwargs...)
    tmp = copy(img)
    f(img, tmp, args...; kwargs...)
    return img
end

function detect_edges(::Type{T},
                  img,
                  f::AbstractEdgeDetectionAlgorithm,
                  args...; kwargs...) where T
    out = similar(Array{T}, axes(img))
    detect_edges!(out, img, f, args...; kwargs...)
    return out
end

detect_edges(img::AbstractArray{T},
                 f::AbstractEdgeDetectionAlgorithm,
                 args...; kwargs...) where T <: Colorant =
         detect_edges(T, img, f, args...; kwargs...)

# Do not promote Number to Gray{<:Number}
detect_edges(img::AbstractArray{T},
                 f::AbstractEdgeDetectionAlgorithm,
                 args...; kwargs...) where T <: Number =
        detect_edges(T, img, f, args...; kwargs...)

# Handle instance where the input is a sequence of images.
detect_edges!(out_sequence::Vector{T},
          img_sequence,
          f::AbstractEdgeDetectionAlgorithm,
          args...; kwargs...) where T <: Union{GenericGrayImage, AbstractArray{<:Color3}} =
    f(out_sequence, img_sequence, args...; kwargs...)

# TODO: Relax this to all color types
function detect_edges!(img_sequence::Vector{T},
                   f::AbstractEdgeDetectionAlgorithm,
                   args...; kwargs...) where T <: Union{GenericGrayImage, AbstractArray{<:Color3}}
    tmp = copy(img_sequence)
    f(img_sequence, tmp, args...; kwargs...)
    return img_sequence
end

function detect_edges(::Type{T},
                  img_sequence::Vector{<:AbstractArray},
                  f::AbstractEdgeDetectionAlgorithm,
                  args...; kwargs...) where T
    N  = length(img_sequence)
    out_sequence = [similar(Array{T}, axes(img_sequence[n])) for n = 1:N]
    detect_edges!(out_sequence, img_sequence, f, args...; kwargs...)
    return out_sequence
end

detect_edges(img_sequence::Vector{<:AbstractArray{T}},
                 f::AbstractEdgeDetectionAlgorithm,
                 args...; kwargs...) where T <: Colorant =
         detect_edges(T, img_sequence, f, args...; kwargs...)

# Do not promote Number to Gray{<:Number}
detect_edges(img_sequence::Vector{<:AbstractArray{T}},
                 f::AbstractEdgeDetectionAlgorithm,
                 args...; kwargs...) where T <: Number =
        detect_edges(T, img_sequence, f, args...; kwargs...)

### Docstrings

"""
    detect_edges!([out,] img, f::AbstractEdgeDetectionAlgorithm, args...; kwargs...)

Detect edges of `img` using algorithm `f`.

# Output

If `out` is specified, it will be changed in place. Otherwise `img` will be changed in place.

# Examples

Just simply pass an algorithm to `detect_edges!`:

```julia
img_edges = similar(img)
detect_edges!(img_edges, img, f)
```

For cases you just want to change `img` in place, you don't necessarily need to manually
allocate `img_edges`; just use the convenient method:

```julia
detect_edges!(img, f)
```

See also: [`detect_edges`](@ref)
"""
detect_edges!

"""
    detect_edges([T::Type,] img, f::AbstractEdgeDetectionAlgorithm, args...; kwargs...)

Detect edges of `img` using algorithm `f`.

# Output

The return image `img_edges` is an `Array{T}`.

If `T` is not specified, then it's inferred.
# Examples

Just simply pass the input image and algorithm to `detect_edges`

```julia
img_edges = detect_edges(img, f)
```

This reads as "`detect_edges` of image `img` using algorithm `f`".

You can also explicitly specify the return type:

```julia
img_edges_float32 = detect_edges(Gray{Float32}, img, f)
```

See also [`detect_edges!`](@ref) for in-place edge detection.
"""
detect_edges


text:
### Fitting by histogram

function calcrange(::Type{Histogram}, x::AbstractVector, binNumber)
    histExtrema = extrema(x)
    histRange = histExtrema[2] - histExtrema[1]
    binSize = histRange/binNumber
    edges = collect(histExtrema[1]:binSize:histExtrema[2]+eps())
    edges[1] -= eps()
    edges[end] += eps()
    return edges
end

function fit(::Type{EmpiricalLinearManifold}, ::Type{Histogram},
             μ::AbstractVector{T}, B::AbstractMatrix{T}, X::AbstractMatrix{T};
             kvargs...) where {T <: Real}

    N, n = size(X)
    K = size(B,2)

    nbins = sturges(n)
    for (k,v) in kvargs
        if k == :nbins
            nbins = Int(v)
        end
    end

    estimates = Array{Histogram}(undef, K+1)

    zk = B'*(X .- μ)
    for k in 1:K
        histEdges = calcrange(Histogram, zk[k,:], nbins)
        estimates[k] = normalize(fit(Histogram, zk[k,:], histEdges, closed=:left), mode=:probability)
    end

    d = LMCLUS.distance_to_manifold(X, μ, B)
    histEdges = calcrange(Histogram, d, nbins)
    estimates[K+1] = normalize(fit(Histogram, d, histEdges, closed=:left), mode=:probability)

	return EmpiricalLinearManifold(μ, B, estimates)
end

fit(::Type{EmpiricalLinearManifold}, ::Type{Histogram},
    LMC::LMCLUS.Manifold, X::AbstractMatrix{T}; kvargs...) where {T <: Real} =
    fit(EmpiricalLinearManifold, Histogram, LMCLUS.mean(LMC), LMCLUS.projection(LMC), X; kvargs...)

### Evaluation

function _pdf(d::EmpiricalLinearManifold{T, Histogram}, x::AbstractVector{T}) where {T<:Real}
    # Basis subspace
    K = size(d.B,2)
    probValues = zeros(T, K+1)
    binCount = length(d.estimate[1].weights)
    z = d.B'*(x - d.μ)
    dist = LMCLUS.distance_to_manifold(x, d.μ, d.B) |> first
    for k in 1:K+1
        val = k > K ? dist : z[k] # Orthonormal subspace distances for K+1
        edges = first(d.estimate[k].edges)
        binIndex = searchsortedlast(edges, val, lt=(x,y)->x<=y)
        if 0 < binIndex <= binCount
            probValues[k] = d.estimate[k].weights[binIndex]
        end
    end
    return prod(probValues)
end

### Entropy

function entropy(d::EmpiricalLinearManifold{T, Histogram}) where {T<:Real}
    H = 0.0
    for est in d.estimate
        dp = est.weights
        filter!(x->x != zero(T), dp)
        H += entropy(dp)
        @debug "Empirical PDF" EPDF=dp H=H
    end
    @debug "Empirical LM model entropy" H=H
    return H
end

function entropy(M::MixtureModel{Q,R,EmpiricalLinearManifold{T, Histogram}}) where {Q,R,T<:Real}
    qb = first(first(components(M)).estimate).weights |> length
    c = length(components(M))
    H = zero(T)
    for i in 1:qb
        Pᵢⱼ = zero(T)
        for j in 1:c
            d = components(M)[j]
            pⱼ = probs(M)[j]
            prob = one(T)
            for est in d.estimate
                prob *= est.weights[i]/sum(est.weights)
            end
            Pᵢⱼ += pⱼ*prob
            @debug "Pᵢⱼ" i j Pᵢⱼ
        end
        H += Pᵢⱼ > zero(T) ? -Pᵢⱼ*log(Pᵢⱼ) : zero(T)
        @debug "H" i H
    end
    @debug "Empirical LM mixture model entropy" H=H
    return H
end

### Sampling

function _rand!(s::EmpiricalLinearManifoldSampler{T, Histogram}, x::AbstractVector{T}) where T <: Real
    return x
end


text:
module AdvancedPS
    using Random, StatsBase, ProgressMeter, Parameters, Distributions
    import AbstractMCMC: step!, AbstractSampler, AbstractModel
    import AbstractMCMC: bundle_samples, sample, MCMCThreads
    import MCMCChains: Chains
    export DE, Particle, DEModel, sample, MCMCThreads
    include("DEMCMC/structs.jl")
    include("DEMCMC/main.jl")
    include("DEMCMC/migration.jl")
    include("DEMCMC/crossover.jl")
    include("DEMCMC/mutation.jl")
    include("DEMCMC/utilities.jl")
end


text:
using Plots, Dates
a=["26.499215232018486" "14:51:29.572";
"15.55254740689574" "14:51:29.756";
"17.13196007509995" "14:51:29.757";
"18.65394894162646" "14:51:29.759";
"16.99354281530844" a"14:51:29.76";
"15.44265943931051" "14:51:29.762";
"14.202979799645252" "14:51:29.764";
"13.94164689332408" "14:51:29.765";
"13.965497112338758" "14:51:29.767";
"12.91434617813082" "14:51:29.768";
"11.797702046326163" "14:51:29.77";
"12.218421281575896" "14:51:29.772";
"11.907514045943785" "14:51:29.773";
"10.938190388230767" "14:51:29.775";
"10.849927020398853" "14:51:29.777";
"10.613975795278899" "14:51:29.778";
"10.115634852501415" "14:51:29.78";
"9.563886446850532" "14:51:29.782";
"8.932171005727948" "14:51:29.784";
"8.95993543921891" "14:51:29.786";
"8.799574078750084" "14:51:29.787";
"7.977678113767442" "14:51:29.789";
"7.722033743726618" "14:51:29.79";
"7.784828285660197" "14:51:29.792";
"7.748450272711257" "14:51:29.794";
"7.122765838322947" "14:51:29.807";
"6.765122519497216" "14:51:29.809";
"6.7779211077149135" "14:51:29.811";
"6.703199427053434" "14:51:29.812";
"6.422419169605139" "14:51:29.814";
"6.226020967500306" "14:51:29.816";
"6.324532646930777" "14:51:29.818";
"6.156923801015428" "14:51:29.82";
"6.047302028223021" "14:51:29.822";
"5.8787075703967675" "14:51:29.823";
"5.949343026407251" "14:51:29.825";
"5.872221052923468" "14:51:29.827";
"5.844333412350692" "14:51:29.829";
"5.820032083364488" "14:51:29.831";
"5.960252432042049" "14:51:29.832";
"6.110604478421559" "14:51:29.834";
"5.9228937883864266" "14:51:29.836";
"5.511681843845812" "14:51:29.838";
"5.391871698992283" "14:51:29.84";
"5.56577439661891" "14:51:29.841";
"5.550248941771409" "14:51:29.843";
"5.316228754019622" "14:51:29.844";
"5.182062876369177" "14:51:29.846";
"5.18351289774542" "14:51:29.848";
"5.206282049042052" "14:51:29.849";
"73.02742" "14:53:57.044";
"86.28058" "14:53:57.548";
"93.30249" "14:53:57.945";
"96.1194" "14:53:58.286";
"97.13043" "14:53:58.584";
"97.07106" "14:53:58.849";
"96.214134" "14:53:59.145";
"94.259125" "14:53:59.531";
"91.73479" "14:53:59.855";
"91.02319" "14:54:00.192";
"92.03532" "14:54:00.551";
"94.1892" "14:54:00.883";
"96.04568" "14:54:01.24";
"97.27583" "14:54:01.547";
"97.75338" "14:54:01.868";
"97.73651" "14:54:02.188";
"97.50871" "14:54:02.568";
"96.950775" "14:54:02.88";
"96.070786" "14:54:03.209";
"94.847725" "14:54:03.541";
"93.22851" "14:54:03.862";
"91.120766" "14:54:04.157";
"88.41518" "14:54:04.508";
"84.958916" "14:54:04.812";
"80.52474" "14:54:05.157";
"83.09181" "14:54:05.555";
"88.497604" "14:54:05.865";
"79.87367" "14:54:06.171";
"80.7059" "14:54:06.467";
"83.37798" "14:54:06.794";
"84.55208" "14:54:07.125";
"84.46771" "14:54:07.493";
"83.78168" "14:54:07.848";
"83.21351" "14:54:08.227";
"83.48191" "14:54:08.542";
"82.88339" "14:54:08.919";
"81.94467" "14:54:09.21";
"81.28766" "14:54:09.56";
"80.25933" "14:54:09.88";
"78.34769" "14:54:10.213";
"80.571365" "14:54:10.552";
"77.28818" "14:54:10.893";
"78.2777" "14:54:11.235";
"78.57628" "14:54:11.564";
"78.08115" "14:54:11.91";
"77.18971" "14:54:12.209";
"76.485054" "14:54:12.539";
"78.35511" "14:54:12.888";
"77.7843" "14:54:13.23";
"79.09124" "14:54:13.545";
"120.02086519777127" "14:54:13.547";
"87.23382555901625" "14:54:13.548";
"57.60968867934277" "14:54:13.549";
"33.9548217173734" "14:54:13.55";
"23.651936927136916" "14:54:13.552";
"28.05399018484942" "14:54:13.553";
"33.890138250685425" "14:54:13.554";
"36.244050672446896" "14:54:13.555";
"35.27082462029161" "14:54:13.556";
"32.075895703406346" "14:54:13.563";
"27.549637180448542" "14:54:13.565";
"22.562650741890014" "14:54:13.566";
"17.492599391254245" "14:54:13.568";
"12.769205792828402" "14:54:13.569";
"10.970373518327976" "14:54:13.571";
"13.101545048283713" "14:54:13.572";
"15.275822715649658" "14:54:13.574";
"15.84106344942932" "14:54:13.575";
"14.914592336342256" "14:54:13.576";
"13.15426617189058" "14:54:13.578";
"11.233621337968462" "14:54:13.579";
"9.622099870601996" "14:54:13.581";
"8.929089829758524" "14:54:13.582";
"9.254708185501386" "14:54:13.584";
"9.873145427463248" "14:54:13.585";
"10.341226687915723" "14:54:13.587";
"10.13993672218774" "14:54:13.588";
"9.09307792366541" "14:54:13.589";
"8.044192678840567" "14:54:13.59";
"7.61568182541545" "14:54:13.592";
"7.361236828257073" "14:54:13.593";
"7.454889419174575" "14:54:13.594";
"7.680990347292265" "14:54:13.595";
"7.688178744680857" "14:54:13.596";
"7.416913815209377" "14:54:13.598";
"6.787622596633534" "14:54:13.599";
"6.258505306082383" "14:54:13.6";
"5.95427871692019" "14:54:13.601";
"6.090535622163555" "14:54:13.602";
"6.450495602197516" "14:54:13.603";
"6.204766993981707" "14:54:13.605";
"5.6887191376032655" "14:54:13.606";
"5.385939561394959" "14:54:13.607";
"5.4163946142427735" "14:54:13.608";
"5.379109414660544" "14:54:13.609";
"5.420627167668498" "14:54:13.61";
"5.1817463721699974" "14:54:13.612";
"4.943675454637615" "14:54:13.613";
"4.845234875704597" "14:54:13.614";
"4.849045807261244" "14:54:13.615";
"46.850914" "14:54:15.97";
"52.694984" "14:54:16.472";
"56.81987" "14:54:16.979";
"59.540234" "14:54:17.524";
"59.569916" "14:54:18.042";
"55.670204" "14:54:18.532";
"59.048286" "14:54:18.96";
"54.21105" "14:54:19.397";
"55.766212" "14:54:19.85";
"54.58752" "14:54:20.316";
"50.706882" "14:54:20.743";
"49.99562" "14:54:21.175";
"50.272118" "14:54:21.632";
"48.901524" "14:54:22.085";
"47.42151" "14:54:22.535";
"44.911316" "14:54:23.093";
"43.82331" "14:54:23.585";
"43.293034" "14:54:24.085";
"40.216152" "14:54:24.554";
"39.05416" "14:54:25.007";
"37.167427" "14:54:25.483";
"34.95958" "14:54:25.987";
"33.431046" "14:54:26.483";
"31.169086" "14:54:26.997";
"28.318672" "14:54:27.496";
"26.22968" "14:54:28.011";
"24.466974" "14:54:28.585";
"22.597588" "14:54:29.101";
"20.935015" "14:54:29.576";
"17.559862" "14:54:30.078";
"10.707282" "14:54:30.721";
"9.330469" "14:54:31.251";
"7.693691" "14:54:31.865";
"7.503528" "14:54:32.427";
"5.59806" "14:54:33.178";
"5.6089206" "14:54:33.741";
"4.095738" "14:54:34.41";
"5.049217" "14:54:35.071";
"4.792138" "14:54:36.009";
"3.9050412" "14:54:36.844";
"6.551854" "14:54:37.445";
"6.5396876" "14:54:38.432";
"3.928997" "14:54:39.426";
"5.048383" "14:54:40.035";
"4.4086747" "14:54:41.002";
"3.5897787" "14:54:41.844";
"4.1762276" "14:54:42.577";
"2.8026385" "14:54:43.453";
"3.3502953" "14:54:44.093";
"2.2796144" "14:54:44.957";
"21.753600690966035" "14:54:44.958";
"20.23219456522179" "14:54:44.96";
"17.578497061320043" "14:54:44.961";
"16.10167812536832" "14:54:44.962";
"15.287545905940547" "14:54:44.963";
"11.884989886430596" "14:54:44.964";
"9.923323583431946" "14:54:44.966";
"9.915230190680978" "14:54:44.967";
"10.239861236184161" "14:54:44.968";
"9.39984673886355" "14:54:44.97";
"7.13475308855646" "14:54:44.971";
"6.967020763059508" "14:54:44.972";
"7.268046988039237" "14:54:44.973";
"6.798654634176626" "14:54:44.974";
"6.478381632437553" "14:54:44.975";
"6.198231233181829" "14:54:44.977";
"6.2579901202619475" "14:54:44.978";
"5.778927859772631" "14:54:44.979";
"6.1209066094423354" "14:54:44.98";
"5.881687666708823" "14:54:44.982";
"5.267836844673849" "14:54:44.983";
"5.014547534653412" "14:54:44.984";
"5.280159287997384" "14:54:44.985";
"4.964419172008977" "14:54:44.987";
"4.494642530497801" "14:54:44.994";
"4.523860248313952" "14:54:44.995";
"4.571356368682251" "14:54:44.997";
"4.205867184569236" "14:54:44.998";
"4.235440098621189" "14:54:45";
"3.9461275806830693" "14:54:45.001";
"3.982492833967862" "14:54:45.003";
"3.9262440665024547" "14:54:45.004";
"3.56929339415687" "14:54:45.006";
"3.6286065215812693" "14:54:45.007";
"3.5256923062286063" "14:54:45.009";
"3.215482206108975" "14:54:45.01";
"3.3121652639065444" "14:54:45.012";
"3.166169808626245" "14:54:45.013";
"3.1247009180063094" "14:54:45.015";
"2.8825151639655187" "14:54:45.016";
"2.8839146972283345" "14:54:45.018";
"2.7696057841566457" "14:54:45.019";
"2.7026232846010285" "14:54:45.021";
"2.556896168640965" "14:54:45.022";
"2.5867634283868517" "14:54:45.024";
"2.6359211246171066" "14:54:45.025";
"3.0135038641064975" "14:54:45.027";
"3.2201941078924627" "14:54:45.028";
"2.45160827601765" "14:54:45.029";
"2.181226253105567" "14:54:45.031";
"42.71606" "14:54:45.656";
"54.304455" "14:54:46.555";
"22.165562" "14:54:47.676";
"57.87572" "14:54:48.393";
"55.797794" "14:54:50.063";
"69.79438" "14:54:51.393";
"15.787255" "14:54:52.579";
"105.40192" "14:54:53.85";
"102.30041" "14:54:56.862";
"67.095436" "14:54:59.56";
"19.700068" "14:55:02.514";
"17.570517" "14:55:04.067";
"23.232357" "14:55:05.104";
"24.516844" "14:55:06.622";
"25.761795" "14:55:07.837";
"25.027925" "14:55:08.985";
"23.005068" "14:55:10.317";
"21.17099" "14:55:11.429";
"18.967398" "14:55:12.502";
"17.81275" "14:55:13.282";
"17.071012" "14:55:14.338";
"16.368814" "14:55:15.486";
"20.482412" "14:55:17.067";
"20.418335" "14:55:18.179";
"14.776248" "14:55:19.292";
"21.286222" "14:55:20.271";
"12.96673" "14:55:21.663";
"11.810612" "14:55:22.827";
"12.49811" "14:55:23.962";
"9.1051" "14:55:25.244";
"9.544827" "14:55:25.952";
"9.234246" "14:55:26.971";
"7.0311685" "14:55:28.053";
"6.3125043" "14:55:28.903";
"6.7022724" "14:55:29.732";
"5.80264" "14:55:30.809";
"5.35116" "14:55:31.764";
"5.86051" "14:55:32.462";
"5.6047306" "14:55:33.344";
"4.389505" "14:55:34.271";
"3.4503138" "14:55:35.031";
"2.9875352" "14:55:35.8";
"2.726512" "14:55:36.5";
"2.731321" "14:55:37.283";
"2.7187803" "14:55:38.17";
"2.1924343" "14:55:39.282";
"1.4248185" "14:55:40.293";
"1.340551" "14:55:41.188";
"1.5975418" "14:55:42.009";
"1.3322003" "14:55:42.999"]

function parse_dates(x)
    t=split(x,(':','.'))
    w=map(x->parse(Float64,x),t)
    Dates.Time(w...)
end
function norm_time(x)
    abs_times = x
    start_time = abs_times[1]
    norm_times = Vector(undef,length(abs_times))
    for i in 1:length(abs_times)
        norm_times[i] = abs_times[i]-start_time
    end
    norm_times
end
function parse_nanosec(x)
    w=[00,00,00,x]
    print(w)
    Dates.Time(w...)
end
loss_values = map(x->parse(Float64,x),a[:,1])
time_values = map(x->parse_dates(x),a[:,2])
norm_time_vals = norm_time(time_values)
set_dates = map(x->parse_nanosec(x),norm_time_vals)

println(norm_time_vals...)
plot(time_values[1:50],loss_values[1:50],linecolor="blue")
plot!(time_values[51:100],loss_values[51:100],linecolor="red")
plot!(time_values[101:150],loss_values[101:150],linecolor="blue")
plot!(time_values[151:150],loss_values[151:150],linecolor="red")
plot!(time_values[201:250],loss_values[201:250],linecolor="blue")
plot!(time_values[251:300],loss_values[251:300],linecolor="red")

Int64(norm_time_vals[1])

print(norm_time_vals)