text:
# Ergänze Tabellen mit Personendaten um Angaben aus der Gemeinsamen Normdatei
# der Deutschen Nationalbibliothek
#
# Projektseite zur Rechercheoberfläche
# https://lobid.org/gnd
#
# und GUI http://lobid.org/gnd/search
#
# Beschreibung des API
# http://lobid.org/gnd/api
#
# API
# http://lobid.org/gnd/search
#
# Dokumentation
# /../README.md
#

# TODO kopiere die Dateien, git commit, git branch dev, git checkout dev
# TODO ersetze gatz mit qd
# TODO prüfe die

"""
    module GND

Finde die GND-ID und weitere Daten zu den Bischöfen in
Gatz: Die Bischöfe des Heiligen Römischen Reiches 1198 bis 1448
"""
module GND

using DataFrames
using HTTP
using JSON
using Infiltrator
using Dates
using Logging

include("./Util.jl")

# Suche mit einer allgemeinen Suchfunktion noch Kombinationen von Feldern. Wenn ein
# Feld leer ist, wird es nie in die Suhe einbezogen.
# Sammle alle Treffer.
#
# Prüfe die einzelnen Felder. Lege ein Sortierreihenfolge fest.

"""
    QRecord

Element im Verzeichnis "member" in einer Anfrage an die Personendatenbank.
"""
struct QRecord
    data::Dict{String, Any}
end

const TOLERANCELEFTSDA = 0 # Toleranz für Amtsende(Gatz) - Sterbedatum(GND).

const TUNINGSCORE = QRecord(Dict("muster" => "fn vn sda ba", "zeitguete" => 0))
const GNDSTRINGCOLS = [:ID_GND_GND,
                       :pName_GND,
                       :Amt_GND,
                       :Sterbedatum_GND,
                       :Typ_GND,
                       :Abfrage_GND,
                       :Qualitaet_GND,
                       :nTreffer_GND,
                       :ID_VIAF_GND,
                       :ID_Wikidata_GND,
                       :URL_Wikipedia_GND]

const RANKMAX = 199

"""
    "Falsche" Treffer
"""
const SKIPIDS = [3815, #1043-1048 Eberhard von Augsburg 133613275 fn vn sda ba ip
           4635, #1137-1141 Udo (von Steinfurt?) 136876471 fn vn sda ba ip
                 ]

# Globale Variablen
"""
    COLNAMEID

Spaltenname der ID
"""
COLNAMEID = :ID_Bischof

"""
    occupations

Liste gültiger Ämter
"""
global occupations = ["Bischof", "Elekt", "Administrator", "Patriarch", "Metropolit"]
global rgxocc = Regex(join(occupations, "|"), "i")

function setoccupations(occ::Vector{<:AbstractString})
    global occupations = occ
    global rgxocc = Regex(join(occupations, "|"), "i")
end

setcolnameid(id::Symbol) = (global COLNAMEID = id)

global minscore = QRecord(Dict("muster" => "vn sda ao", "zeitguete" => 0))
#const minscore = QRecord(Dict("muster" => "", "zeitguete" => 0))

"""
    logpath

Wenn der Pfad nicht leer ist, werden hierhin die Log-Mitteilungen geschrieben.
"""
global logpath = ""
function setlogpath(logfile::AbstractString)
    path = splitdir(logfile)[1]
    path == "" && (path = ".")
    if ispath(path)
        global logpath = logfile
    else
        @error "Pfad nicht gefunden"
    end
end

global filelog = Logging.NullLogger()
global finfiltrate = false

import Base.getindex
getindex(r::QRecord, key::AbstractString) = getindex(r.data, key)

import Base.haskey
haskey(r::QRecord, key) = haskey(r.data, key)

function startlog(startid)
    global iolog
    iolog = open(LOGFILE, "a")
    println(iolog, now())
    println(iolog, "Beginne mit: ", startid)
end

"""
        makedrank()

Erstelle ein Ranking für die Qualität der Treffer. Das Verzeichnis wird für
Vergleiche genutzt.
"""
function makedrank()
    keys = String[]
    for mkn in ["fn vn sd",
                "fn vn sda",
                "fn sd",
                "fn sda",
                "vn sd",
                "vn sda",
                "fn vn",
                "fn",
                "vn"],
        mkocc in [true, false],
        mkp in [true, false]

        smk = mkn
        mkocc && (smk *= " ba")
        mkp && (smk *= " ao")

        push!(keys, smk)
    end
    push!(keys, "sda ba ao")

    ltf(a, b) = length(split(a)) < length(split(b))
    sort!(keys, alg = InsertionSort, lt = ltf, rev = true)
    keyips = fill("", 2 * length(keys))
    keyips[1:2:end] .= map(k -> k * " ip", keys)
    keyips[2:2:end] .= keys
    push!(keyips, "")

    Dict(k => i for (i, k) in enumerate(keyips))
end

import Base.isless

let drank = makedrank()
    global isless
    function isless(a::QRecord, b::QRecord)
        (islessinset(a.data["muster"],  b.data["muster"], drank)
         || (a.data["muster"] == b.data["muster"]
             && a.data["zeitguete"] < b.data["zeitguete"]))
    end
end

"""
    islessinset(a, b, drank)

Vergleiche Muster nach ihrer Position in `drank`
"""
function islessinset(a, b, drank)
    pa = get(drank, a, 0)
    pb = get(drank, b, 0)
    pa == 0 && @error ("Ungültiger Schlüssel: " * a)
    pb == 0 && @error ("Ungültiger Schlüssel: " * b)
    return pb < pa
end

let drank = makedrank()
    global getrank
    getrank(key) = drank[key]

    global isvalid
    isvalid(key) = haskey(drank, key)
end

"""
    getGND(searchtext)

Frage GND ab.

## Rückgabewert
Verzeichnis mit den Schlüsseln:
    "@context"
    "id"
    "totalItems"
    "member"
    "aggregation"
"""
function getGND(searchtext)
    st = encodeurl(searchtext)
    rq = HTTP.request("GET", "http://lobid.org/gnd/search?q=$(st)", readtimeout = 30, retries = 5);
    rq.status == 200 || error("Abfrage für '$searchtext' fehlgeschlagen")
    rds = String(rq.body);
    rds4parser = replace(rds, r"\n *" => "");
    rdt = JSON.parse(rds4parser)

end

"""
    getGND(row::DataFrameRow, col)

Frage GND ab.

## Rückgabewert
Verzeichnis mit den Schlüsseln:
    "@context"
    "id"
    "totalItems"
    "member"
    "aggregation"
"""
function getGND(row::DataFrameRow, colset)
    rdt = Dict{String, Any}()
    # Stelle keine leere Anfrage
    all(isequal(""), row[colset]) && return rdt
    searchtext = join(row[colset], " ")
    st = encodeurl(searchtext)
    rq = HTTP.request("GET", "http://lobid.org/gnd/search?q=$(st)", readtimeout = 30, retries = 5);
    rq.status == 200 || error("Abfrage für '$searchtext' fehlgeschlagen")
    rds = String(rq.body);
    rds4parser = replace(rds, r"\n *" => "");
    rdt = JSON.parse(rds4parser)
    # Schreibe die Abfrage mit in die Ergebnisse
    for rec in rdt["member"]
       rec["abfrage"] = join(string.(colset), ", ")
    end
    rdt
end

"""
    reconcile!(df::AbstractDataFrame,
                    qcols,
                    dfocc::AbstractDataFrame;
                    colswitch = :VOID,
                    nmsg = 40,
                    toldateofdeath = 5,
                    toloccdeath = 5)

Frage GND nacheinander nach den Werten aus den Spaltenkombinationen in `qcols`
ab. Verwende `toldateofdeath` als Toleranz für das Sterbedatum und
`toloccdeath` als Toleranz zwischen Amtsende und Sterbedatum. Bearbeite nur
Datensätze, für die `colswitch` nicht befüllt ist.

Beispiel für `qcols`
```
qcols = [[:Familienname, :Vorname], [:Famlienname, :Amt]]
```
"""
function reconcile!(df::AbstractDataFrame,
                    qcols,
                    dfocc::AbstractDataFrame;
                    colswitch = :VOID,
                    nmsg = 40,
                    toldateofdeath = 5,
                    toloccdeath = 5)

    global logpath
    global occupations
    startid = df[1, COLNAMEID]
    minkey = minscore["muster"]
    minrank = getrank(minkey)

    if logpath != ""
        logio = open(logpath, "a")
        filelog = SimpleLogger(logio)
        with_logger(filelog) do
            @info now() startid
            @info "Level:" minkey minrank
            @info "Gültige Ämter:" occupations
        end
    end

    @info "Level:" minkey minrank
    @info "Gültige Ämter" occupations

    function writematch!(row, bestrec, records)
        writerow!(row, bestrec)
        nbest = count(isequal(bestrec), records)
        if nbest > 1
            with_logger(filelog) do
                @info "Mehrere Treffer:" row[COLNAMEID] bestrec["muster"]
            end
        end
        nbest
    end

    fnotempty(v) = !ismissing(v) && !isequal(v, "")

    dbest = Dict{Int, Int}()
    irow = 1
    for row in eachrow(df)
        records = QRecord[]
        irow % nmsg == 0 && println("Datensatz: ", irow)
        irow += 1

        iep = row[COLNAMEID]
        # rette Daten
        colswitch != :VOID && row[colswitch] != "" && continue
        # überspringe Datensätze, die bekanntermaßen falsche Treffer liefern
        iep in SKIPIDS && continue

        # Ämter für row
        places = String[]
        deocs = String[]
        dflocc = Util.rowselect(dfocc, iep, COLNAMEID)
        if size(dflocc, 1) == 0
            @info "Keine Amtsdaten gefunden" iep
        else
            places = filter(fnotempty, dflocc[!, :Bistum])
            deocs = filter(fnotempty, dflocc[!, :Amtsende])
            if length(deocs) == 0
                @info "Keine Amtsendedaten gefunden" iep
            end
        end

        ffound = false
        nbest = 0
        for cols in qcols
            nbest = 0
            qres = getGND(row, cols)
            length(qres) == 0 && continue

            append!(records, evaluate!(qres, row, places, deocs, toldateofdeath, toloccdeath))

            if length(records) > 0
                bestrec, posbest = findmax(records)
                if bestrec >= TUNINGSCORE
                    nbest = writematch!(row, bestrec, records)
                    ffound = true
                    break
                end
            end
        end

        if !ffound && length(records) > 0
            bestrec, posbest = findmax(records)
            if bestrec >= minscore
                nbest = writematch!(row, bestrec, records)
            end
        end
        dbest[nbest] = get(dbest, nbest, 0) + 1
    end

    if logpath != ""
        flush(logio)
        close(logio)
    end

    dbest
end

function selectmatches(row, records)
    nbest = 0
    ffound = false
    length(records) == 0 && return nbest, ffound

    bestrec, posbest = findmax(records)
    if bestrec >= minscore
        # schreibe einen Datensatz, auch wenn er nicht eindeutig ist.
        writerow!(row, bestrec)
        nbest = count(isequal(bestrec), records)
        if nbest == 1
            ffound = true
        else
            println("Mehrere Treffer für :", row[COLNAMEID], " mit: ", bestrec["muster"])
        end
    end
    return nbest, ffound
end

"""
    evaluate!(qres::Dict{String, Any}, row, places, toldateofdeath, toloccdeath)

Bewerte die Datensätze in `qres`.
"""
function evaluate!(qres::Dict{String, Any},
                   row,
                   places,
                   deocs,
                   toldateofdeath,
                   toloccdeath)
    idep = row[COLNAMEID]
    records = QRecord[]
    nvalid = 0
    matchkey = ""
    record = Dict{String, Any}()
    sreject = String[]
    for record in qres["member"]
        # println(record["preferredName"], ": mk: ", mk, " scdod: ", scdod)

        # Die Bewertung wird in `record` in den Feldern `muster` und `zeitguete`
        # abgelegt.
        record["amuster"] = String[]
        record["zeitguete"] = 0
        isperson(record) || continue

        evaluategnfn!(record, row)
        evaluatedod!(record, idep, row[:Sterbedatum], deocs, toldateofdeath, toloccdeath)
        if record["zeitguete"] == -1
            push!(sreject, join(record["amuster"], " "))
            continue
        end
        evaluateocc!(record) # Kommt eines der Ämter vor?!
        evaluateplace!(record, places)
        evaluateip!(record)

        matchkey = record["muster"] = join(record["amuster"], " ")
        if isvalid(matchkey)
            push!(records, QRecord(record))
        else
            matchkey != "" && push!(sreject, matchkey)
        end
    end
    if length(sreject) > 0
        with_logger(filelog) do
            @info "verworfen:" idep, unique(sreject)
        end
    end
    return records
end

function isperson(record)
    reduce(|, (map(val -> occursin("Person", val), record["type"])))
end

"""
    queryGND!(row, qcols, toldateofdeath, tolendofoccupation)

Verwende `toldod` für die Bewertung des Wertes in `:Sterbedatum` und
`toleoo` für die Bewertung des Wertes in `:Amtsende`.

Es ist möglich, dem Datensatz auch Felder für die Suche hinzuzufügen
und sie immer mit dem gleichen Wert zu Belegen, z.B. "Bischof".
"""
function queryGND!(row, qcols, toldateofdeath, tolendofoccupation)
    records = Any[]
    mkeys = String[]
    searchtext = ""
    nvalid = 0
    mkey = ""
    record = nothing
    for colset in qcols
        all(isequal(""), row[colset]) && continue  # Suche nicht nach leeren Feldern
        searchtext = join(row[colset], " ")
        # println(searchtext)
        gndres = getGND(searchtext)
        record, mkey, nvalid = evaluate(gndres, row, toldateofdeath, tolendofoccupation)
        # Brich ab, sobald es gültige Treffer gibt. Dazu dürfen die Anforderungen an einen
        # gültigen Treffer nicht zu gering sein! siehe minscore
        (nvalid > 0) && break
    end
    if nvalid > 0
        writerow!(row, record, mkey, nvalid)
    end
end

"""
Bewerte die Datensätze in `gndres` anhand der Werte in `row`
"""
function evaluate(gndres, row, toldod, toleoo)
    acriteria = Pair{Matchkey, Int}[]
    nvalid = 0
    record = Dict{String, Any}()
    arec = typeof(record)[]
    amk = String[]
    mkout = ""
    mk = ""
    sdsc = 0 # Bewertung der Abweichung für das Sterbedatum
    for rcel in gndres["member"]
        amk = String[]
        mk = ""
        # println(rcel["preferredName"], ": mk: ", mk, " scdod: ", scdod)
        mk, scdod = evaluatedod(rcel, row, toldod, toleoo)
        (mk != "") || continue # Ein passendes Datum ist Grundvoraussetzung
        push!(amk, mk)
        push!(amk, evaluategnfn(rcel, row))
        push!(amk, evaluateocc(rcel))
        push!(amk, evaluateplace(rcel, row))

        mk = join(filter(!isequal(""), amk), " ")
        push!(acriteria, Pair(Matchkey(mk), scdod))
        push!(arec, rcel)
    end
    nvalid = length(acriteria)
    # ## TODO 2020-01-29 in aufrufende Funktion verlegen
    if nvalid > 0
        maxval, indmax = findmax(acriteria)
        if maxval > MINSCORE
            record = arec[indmax]
            mkout = maxval[1]
        else
            nvalid = 0
        end
    end
    return (record, mkout, nvalid)
end

const RYEAR = "([0-9]?[0-9]?[0-9]{2})(/[0-9]?[0-9]\\??)?"
const RGXYEAR = Regex(RYEAR)
const RGXDOBAD = Regex("(" * RYEAR * ")"
                     * " *((-|bis)[^1-9]*"
                     * "(" * RYEAR * "))?")

"""

Verwende `toldod` für die Bewertung der Sterbedatum in `dod` und
`toleoo` für die Bewertung der Werte in `deocs` (Amtsendedaten)

"""
function evaluatedod!(rcel,
                      id,
                      dod::Union{AbstractString, Missing},
                      deocs,
                      toldod,
                      toleoo)
    # touch 2020-02-04
    # Für die Bischöfe vor 1198 gibt es oft nur eine Angabe für das
    # Jahrhundert. "[4. Jh.]"

    function matchdod(dod,
                      datecand::Int,
                      matchkey::AbstractString,
                      toleranceleft::Int,
                      toleranceright::Int,)

        if ismissing(dod) || datecand == 0
            return 0, ""
        end

        delta = dod - datecand
        if delta <= toleranceleft && -delta <= toleranceright
            score = delta > 0 ? toleranceleft - delta : toleranceright + delta
            return score, matchkey
        else
            return -1, "dsd"
        end

    end

    # Lies Daten und bestimme maximale Qualität der Abfrage

    datecand = 0
    if haskey(rcel, "dateOfDeath")
        sdatecand = rcel["dateOfDeath"][1]
        rgm = match(RGXYEAR, sdatecand)
        if rgm != nothing
            datecand = parse(Int, rgm[1])
        else
            @warn ("Kein Datum gefunden in '" * sdatecand * "' für " * string(id))
            datecand = 0
        end
    elseif haskey(rcel, "dateOfBirthAndDeath")
        sdatecand = rcel["dateOfBirthAndDeath"][1]
        rgm = match(RGXDOBAD, sdatecand)
        if rgm != nothing
            if rgm[7] != nothing
                datecand = parse(Int, rgm[7])
            elseif rgm[2] != nothing
                datecand = parse(Int, rgm[2])
            end
        else
            datecand = 0
        end
    end

    matchkey = ""
    score = 0

    sdates = String[]
    if !ismissing(dod) && dod != ""
        sdates = [dod]
        toleranceleft = toleranceright = toldod
        matchkey = "sd"
    elseif length(deocs) > 0
        sdates = deocs
        toleranceleft = TOLERANCELEFTSDA
        toleranceright = toleoo;
        matchkey = "sda"
    else # keine sinnvollen Vergleichsdaten
        return matchkey, score
    end

    # Vergleiche; finde das Maximum
    scmks = Tuple{Int, String}[]
    for sdate in sdates
        dod = parsedate(sdate)
        score, matchkey = matchdod(dod, datecand, matchkey, toleranceleft, toleranceright)
        push!(scmks, (score, matchkey))
    end

    if length(scmks) > 0
        score, matchkey = maximum(scmks)
    end

    if matchkey != ""
        push!(rcel["amuster"], matchkey)
        rcel["zeitguete"] = score
    end

    return matchkey, score
end

"""

Suche nach Familienname und oder Vorname
"""
function evaluategnfn(rcel, row)::String
    ffn = false
    fgn = false
    # Falls es keinen Familiennamen gibt, wird von `checkkey` zurückgegeben
    familyname = row[:Familienname]
    givenname = row[:Vorname]
    if haskey(rcel, "preferredNameEntityForThePerson")
        gd = rcel["preferredNameEntityForThePerson"]
        ffn |= (checkkey(gd, "nameAddition", familyname)
                || checkkey(gd, "surname", familyname))
        fgn |= (checkkey(gd, "forname", givenname)
                || checkkey(gd, "personalName", givenname))
        if ffn && fgn
            return "fn vn"
        end
    end

    if haskey(rcel, "variantNameEntityForThePerson")
        for var in rcel["variantNameEntityForThePerson"]
            ffn |= (checkkey(var, "nameAddition", familyname)
                   || checkkey(var, "surname", familyname))
            fgn |= (checkkey(gd, "forname", givenname)
                    || checkkey(var, "personalName", givenname))
            if ffn && fgn
                return "fn vn"
            end
        end
    end

    if haskey(rcel, "preferredName")
        pn = rcel["preferredName"]
        ffn |= (familyname != "" && occursin(familyname, pn))
        fgn |= occursin(givenname, pn)
        if ffn && fgn
            return "fn vn"
        end
    end

    if haskey(rcel, "variantName")
        ffn |= checkkey(rcel, "variantName", familyname)
        fgn |= checkkey(rcel, "variantName", givenname)
        if ffn && fgn
            return "fn vn"
        end
    end

    ffn && return "fn"
    fgn && return "vn"
    return ""
end

"""

Suche nach Familienname und oder Vorname
"""
function evaluategnfn!(rcel, row)::String

    function matchgnfn(rcel, row)
        # Falls es keinen Familiennamen gibt, wird von `checkkey` zurückgegeben
        ffn = false
        fgn = false

        familyname = row[:Familienname]
        givenname = row[:Vorname]
        if haskey(rcel, "preferredNameEntityForThePerson")
            gd = rcel["preferredNameEntityForThePerson"]
            ffn |= (checkkey(gd, "nameAddition", familyname)
                    || checkkey(gd, "surname", familyname))
            fgn |= (checkkey(gd, "forname", givenname)
                    || checkkey(gd, "personalName", givenname))
            if ffn && fgn
                return "fn vn"
            end
        end

        if haskey(rcel, "variantNameEntityForThePerson")
            for var in rcel["variantNameEntityForThePerson"]
                ffn |= (checkkey(var, "nameAddition", familyname)
                        || checkkey(var, "surname", familyname))
                fgn |= (checkkey(gd, "forname", givenname)
                        || checkkey(var, "personalName", givenname))

                if ffn && fgn
                    return "fn vn"
                end
            end
        end

        if haskey(rcel, "preferredName")
            pn = rcel["preferredName"]
            ffn |= (familyname != "" && occursin(familyname, pn))
            fgn |= occursin(givenname, pn)
            if ffn && fgn
                return "fn vn"
            end
        end

        if haskey(rcel, "variantName")
            ffn |= checkkey(rcel, "variantName", familyname)
            fgn |= checkkey(rcel, "variantName", givenname)
            if ffn && fgn
                return "fn vn"
            end
        end
        ffn && return "fn"
        fgn && return "vn"

        return ""
    end
    matchkey = matchgnfn(rcel, row)
    if matchkey != ""
        push!(rcel["amuster"], matchkey)
    end
    return matchkey
end

function evaluateocc(rcel)::String
    focc = false
    if haskey(rcel, "professionOrOccupation")
        ga = rcel["professionOrOccupation"]
        for gd in ga
            gp = gd["label"]
            focc = occursin(r"((b|B)ischof|(E|e)lekt)", gp)
            focc && return "ba"
        end
        return ""
    else
        return ""
    end
end

function evaluateocc!(rcel)::Bool
    # Es soll genügen, wenn überhaupt ein passendes
    # (z.B. bischöfliches) Amt ausgeübt wurde.
    global rgxocc

    focc = false
    if haskey(rcel, "professionOrOccupation")
        ga = rcel["professionOrOccupation"]
        for gd in ga
            gp = gd["label"]
            rgm = match(rgxocc, gp)
            focc = rgm != nothing
            focc && break
        end
    elseif haskey(rcel, "professionOrOccupationAsLiteral")
        for gp in rcel["professionOrOccupationAsLiteral"]
            rgm = match(rgxocc, gp)
            focc = rgm != nothing
            focc && break
        end
    end

    if focc
        push!(rcel["amuster"], "ba")
    end
    focc
end

function evaluateplace(rcel, row)::String
    fplace = false
    diocese = row[:Bistum]
    if diocese != ""  && haskey(rcel, "placeOfActivity")
        ga = rcel["placeOfActivity"]
        for gd in ga
            gp = gd["label"]
            focc = occursin(diocese, gp)
            focc && return "ao"
        end
        return ""
    else
        return ""
    end
end

"""
    Kommt einer der Orte in `places` in einer der Ortsangaben
    in `rcel` vor?
"""
function evaluateplace!(rcel, places)::Bool
    fplace = false

    if haskey(rcel, "placeOfActivity")
        ga = rcel["placeOfActivity"]
        for place in places, gd in ga
            gp = gd["label"]
            fplace = !ismissing(place) && occursin(place, gp)
            fplace && break
        end
    end
    if fplace
        push!(rcel["amuster"], "ao")
    end
    fplace
end

"""
    filterbynamedod(dgnd::Dict{String, Any},
                         givenname::AbstractString,
                         familyname::AbstractString,
                         dateofdeath::AbstractString;
                         tolerance::Int = 10)

Filtere die Einträge in `dgnd` gegen `givenname`, `familyname` und
`dateofdeath`. Das Sterbedatum darf nicht um mehr als `tolerance`
abweichen. Einheit: Jahre.

## Rückgabewert
Liste von Verzeichnissen, jeweils eins für einen Treffer
"""
function filterbynamedod(dgnd::Dict{String, Any},
                         givenname::AbstractString,
                         familyname::AbstractString,
                         dateofdeath::AbstractString;
                         tolerance::Int = 10)

    res = Array{Dict{String, Any}, 1}(undef, 0)

    # Wandle das Sterbedatum in eine Zahl um
    mdod = match(r"[1-9][0-9]+", dateofdeath)
    mdod != nothing || error("Ungültiges Sterbedatum '$(dateofdeath)'")
    idod = parse(Int, mdod.match)

    for mbel in dgnd["member"]
        mid = match(r"/([[:alnum:]-]+)$", mbel["id"])
        if mid == nothing
            @warn string("Keine ID gefunden für '", givenname, ", ", familyname, "'.")
            continue
        else
            id = mid[1]
        end

        if (findnames(mbel, givenname, familyname, id)
            && acceptdod(mbel, idod, tolerance))
            push!(res, mbel)
        end
    end
    res
end

# veraltet
function encodeurl_simple(s)
    s = replace(s, " " => "%20")
    s = replace(s, "ä" => "%C3%A4")
    s = replace(s, "ö" => "%C3%B6")
    s = replace(s, "ü" => "%C3%BC")

    s = replace(s, "á" => "%C3%A1")
    s = replace(s, "é" => "%C3%A9")
    s = replace(s, "ú" => "%C3%BA")

    s = replace(s, "Ä" => "%C3%84")
    s = replace(s, "Ö" => "%C3%96")
    s = replace(s, "Ü" => "%C3%9C")
    s = replace(s, "ß" => "%C3%9F")
end

function encodeurl(s)
    cs = String[]
    for c in s
        cu = codeunits(string(c))
        for b in cu
            push!(cs, "%")
            push!(cs, bytes2hex([b]))
        end
    end
    cs
    join(cs)
end

function acceptdod(mbel,
                   dod::Int,
                   tolerance)

    if !haskey(mbel, "dateOfDeath")
        return false
    else
        gnddod = mbel["dateOfDeath"][1]
        rg = findfirst(r"^[0-9]+", gnddod)
        rg != nothing || (@info("Keine Jahreszahl in $(gnddod)"); return false)
        ignddod = parse(Int, gnddod[rg])

        delta = abs(dod - ignddod)
        return (delta < tolerance)
    end
end

"""
"""
function findnames(mbel,
                   givenname::AbstractString,
                   familyname::AbstractString,
                   id)

    prefn = mbel["preferredName"]
    if (occursin(givenname, prefn)
        && occursin(familyname, prefn))
        return true
    elseif haskey(mbel, "variantNameEntityForThePerson")
        dvn = mbel["variantNameEntityForThePerson"]
        flaggn = false
        flagfn = false
        for vnel in dvn
        flaggn = (checkkey(vnel, "nameAddition", familyname)
                  || checkkey(vnel, "surname", familyname))
            flagfn = (checkkey(vnel, "forename", givenname)
                      || checkkey(vnel, "personalName", givenname))
            if flaggn && flagfn
                return true
            end
        end
        return false
    else
        return false
    end
end

"""
Prüfe ob `svalue` (Name) in einem Array `dn[key]` vorkommt. Mindenstens ein Teil von
`svalue` muss als Wort in einem der Elemente von `dn[key]` enthalten sein.
"""
function checkkey(dn, key, name::AbstractString)
    name == "" && return false

    # nutze aus, dass die Eingangsnamen keine Präfixe mehr enthalten
    srec = get(dn, key, Any[])
    for field in srec
        Util.checkname(name, field) && return true
    end

    # teile den Namen auf und entferne Ordnungszahlen
    # nameelems = splitname(name)
    # if haskey(dn, key) && length(nameelems) > 0
    #     srec = dn[key]
    #     for field in srec
    #         common = intersect(nameelems, splitname(field))
    #         length(common) > 0 && return true
    #     end
    # end
    return false
end

"""
    splitname(name::AbstractString)

Teile `name`auf und entferne leere Felder und Ordnungszahlen
"""
function splitname(name::AbstractString)
    rgx = r"[^\w]+"

    filtercard(a) = filter(p -> !occursin(r"^[IVX]+$", p), a)
    filterempty(a) = filter(p -> p != "", a)
    split(name, rgx) |> filterempty |> filtercard
end

function countmatches(df; nmax = 500, nmsg = 50)
    dfdst = DataFrame([String, String, String, String, String],
                      [:Familienname, :Vorname, :Sterbedatum, :GND_pn, :GND_dod],
                      0)
    histB = Dict{Int, Int}()
    nrec = 1
    for row in eachrow(df)
        nrec % nmsg == 0 && println("Datensatz: ", nrec)
        nrec += 1
        nrec < nmax || break

        sterbedatum = row[:Sterbedatum]
        vorname = row[:Vorname]
        familienname = row[:Familienname]
        if sterbedatum != ""
            rdt = GND.getGND(vorname * " " * familienname)
            rfilter = GND.filterbynamedod(rdt, vorname, familienname, sterbedatum)
            nr = size(rfilter, 1)
            if nr == 1
                gnd_pn = rfilter[1]["preferredName"]
                gnd_dod = rfilter[1]["dateOfDeath"][1]
                push!(dfdst, (familienname, vorname, sterbedatum, gnd_pn, gnd_dod))
            end
            lc = get!(histB, nr, 0)
            histB[nr] = lc + 1
        end
    end
    histB, dfdst
end

"""
    findstructdata(df::AbstractDataFrame, fsearch, skey; nmsg = 40, nmax = 200)

Finde Feldnamen oder Werte im JSON-Baum.

## Beispiel
``` julia
fsearch = GND.uniontypes!
GND.findstructdata(dfgv, fsearch, "type", nmax = 200)
```
"""
function findstructdata(df::AbstractDataFrame, fsearch, skey; nmsg = 40, nmax = 200)
    nf = Set(String[])
    nrec = 1
    for row in eachrow(df)
        nrec % nmsg == 0 && println("Datensatz: ", nrec)
        nrec += 1
        nrec < nmax || break

        vorname = row[:Vorname]
        familienname = row[:Familienname]
        rdt = GND.getGND(vorname * " " * familienname)
        fsearch(nf, rdt, skey)
    end
    nf
end

"""
    unionfields!(nf, rdt, key::String)

Füge `nf` hinzu: Feldbezeichnungen der Elemente im Feld mit dem
Bezeichener `key` aus dem Datensatz `rdt`.
"""
function unionfields!(nf, rdt, key::String)
    # Aufzurufen z.B. mit "variantNameEntityForThePerson"
    for mbel in rdt["member"]
        if haskey(mbel, key)
            avne = mbel[key]
            for vel in avne
                union!(nf, keys(vel))
            end
        end
    end
end

"""
    unionprofessionvalues!(nf, rdt, key::String)

Füge `nf` hinzu: Berufsbezeichungen im Feld mit dem Bezeichner `key`
aus dem Datensatz `rdt`.
"""
function  unionprofessionvalues!(nf, rdt, key::String)
    # Aufzurufen mit "professionOrOccupation"
    for mbel in rdt["member"]
        if haskey(mbel, key)
            avne = mbel[key]
            for dve in avne
                push!(nf, dve["label"])
            end
        end
    end
end

"""
    uniontypes!(nf, rdt, key::String)

Füge `nf` hinzu: Berufsbezeichungen im Feld mit dem Bezeichner `key`
aus dem Datensatz `rdt`.
"""
function  uniontypes!(nf, rdt, key::String)
    # Aufzurufen mit "type"
    for mbel in rdt["member"]
        if haskey(mbel, key)
            avne = mbel[key]
            for ve in avne
                push!(nf, ve)
            end
        end
    end
end

# Ansatz aufgegeben: Wir müssten Strukturen speichern und dann die Suche organisieren.
"""
    saveGND(df, keys, filename::AbstractString)
"""
function saveGND(df::AbstractDataFrame,
                 keys::Array{Symbol},
                 filename::AbstractString;
                 nmsg = 40, nmax = 200)
    open(filename, "w") do iodst
        nrec = 1
        for row in eachrow(df)
            nrec % nmsg == 0 && println("Datensatz: ", nrec)
            nrec += 1
            nrec < nmax || break

            searchtext = join(row[keys], " ")
            if searchtext == " "
                nrec -= 1
                continue
            end
            # println(searchtext)
            rdt = getGND(searchtext)
            println(iodst, rdt)
        end
    end
end

"""
"""
function reconcile!(df::AbstractDataFrame; nmsg = 40, nmax = 200, offset = 1, tolerance = 5)
    ru = min(size(df, 1), nmax)
    dfv = @view df[offset:ru, :]

    irow = 1
    for row in eachrow(dfv)
        irow % nmsg == 0 && println("Datensatz: ", irow)

        givenname = row[:Vorname]
        familyname = row[:Familienname]

        if givenname != ""    # Bischofseintrag
            if familyname != ""
                reconcilegnfn!(row, tolerance, ScN + ScBA)
            else
                reconcilegn!(row, tolerance, ScVN + ScSD)
            end
        end
        irow += 1
    end
end

function reconcilegnfn!(row, tolerance, level)
    givenname = row[:Vorname]
    dateofdeath = row[:Sterbedatum]
    familyname = row[:Familienname]
    searchtext = givenname * " " * familyname

    rdt = getGND(searchtext)
    nitems = rdt["totalItems"]
    if nitems > 0
        rdtms = rdt["member"]
        sctable = Pair{Int, String}[]
        for mbel in rdtms
            push!(sctable, scoremember(mbel,
                                       givenname,
                                       familyname,
                                       dateofdeath,
                                       tolerance))
        end

        maxval, indmax = findmax(sctable)
        # println("Score: ", maxval, " bei ", indmax)
        nmaxval = count(se -> isequal(maxval[1], se[1]), sctable)
        if maxval[1] >= level
            writerow!(row, rdtms[indmax], maxval[2], nmaxval)
        else
            # Datensätze vorhanden, aber kein überzeugender Inhalt
            row[:nTreffer_GND] = string(nitems)
        end
    else
        # Versuche es mit der Suche nach Vorname, weil die Familiennamen manchmal streuen
        # dann muss aber das Sterbedatum passen
        reconcilegn!(row, tolerance, ScVN + ScSD)
    end
end

function reconcilegn!(row, tolerance, level)
    givenname = row[:Vorname]
    searchtext = givenname * " " * "Bischof"
    dateofdeath = row[:Sterbedatum]

    rdt = getGND(searchtext)
    nitems = rdt["totalItems"]
    if nitems > 0
        rdtms = rdt["member"]
        flagn1 = size(rdtms, 1) == 1
        sctable = Pair{Int, String}[]
        for mbel in rdtms
            push!(sctable, scoremember(mbel,
                                       givenname,
                                       dateofdeath,
                                       tolerance))
        end
        maxval, indmax = findmax(sctable)
        # println("Score: ", maxval, " bei ", indmax)
        nmaxval = count(se -> isequal(maxval[1], se[1]), sctable)

        if maxval[1] >= level#  siehe Festlegung von LevelVN
            writerow!(row, rdtms[indmax], maxval[2], nmaxval)
        else
            # Datensätze vorhanden, aber kein überzeugender Inhalt
            row[:nTreffer_GND] = string(nitems)
        end
    end
end

"""

Bewerte einen Eintrag mit `givenname` und `familyname`.
"""
function scoremember(mbel, givenname, familyname, dateofdeath, tolerance)

    # Handelt es sich um eine Person?
    isperson = reduce(|, (map(val -> occursin("Person", val), mbel["type"])))
    if !isperson
         return Pair(0, "")
    end

    tot = 0

    # Name
    mk = matchkeyname(mbel, givenname, familyname)
    if mk == "fn vn"
        tot += ScFNVN
    elseif mk == "n"
        tot += ScN
    else
        # Name aus Vorname und Familienname nicht gefunden
        return Pair(0, "")
    end

    # Sterbedatum
    scdateod = scoredateofdeath(mbel, dateofdeath, tolerance)
    # println("Ergebnis bei Datum: ", scdateod)
    if scdateod > 0
        tot += scdateod
        mk = mk * " " * "sd"
    elseif scdateod <= 0 # Abweichung zu groß
        return Pair(0, mk)
    elseif scdateod == 0 # keine Daten vorhanden
        ## prüfe den Ort der Aktivität
    end

    # Amt
    if valueprofession(mbel) != ""
        tot += ScBA
        mk = mk * " " * "ba"
    end

    # Ein Eintrag für eine "individualisierte Person" wird vorgezogen
    if valuetype(mbel) == "DifferentiatedPerson"
        tot += ScIP
        mk = mk * " " * "ip"
    end

    return Pair(tot, mk)
end

"""

Suche nach `givenname`und `familienname` in `mbel`. Gib "fn vn", "n" oder "" zurück.
"""
function matchkeyname(mbel, givenname, familyname)::String
    if haskey(mbel, "preferredNameEntityForThePerson")
        gd = mbel["preferredNameEntityForThePerson"]
        ffn = (checkkey(gd, "nameAddition", familyname)
               || checkkey(gd, "surname", familyname))
        fgn = (checkkey(gd, "forname", givenname)
               || checkkey(gd, "personalName", givenname))
        if ffn & fgn
            return ("fn vn")
        end
    end

    if haskey(mbel, "variantNameEntityForThePerson")
        for var in mbel["variantNameEntityForThePerson"]
            ffn = (checkkey(var, "nameAddition", familyname)
                   || checkkey(var, "surname", familyname))
            fgn = (checkkey(gd, "forname", givenname)
                   || checkkey(var, "personalName", givenname))
            if ffn & fgn
                return ("fn vn")
            end
        end

    end

    if haskey(mbel, "preferredName")
        pn = mbel["preferredName"]
        if (occursin(givenname, pn) & occursin(familyname, pn))
            return ("n")
        else
            return ("")
        end
    end

    if haskey(mbel, "variantName")
        pn = mbel["variantName"]
        if (occursin(givenname, pn) & occursin(familyname, pn))
            return ("n")
        else
            return ("")
        end
    end

    @warn "Keine Namensfelder gefunden: " * givenname * " " * familyname
    return ("")
end

"""

Suche nach `givenname` in `mbel`. Gib "vn" oder "" zurück.
"""
function matchkeyname(mbel, givenname)::String
    if haskey(mbel, "preferredNameEntityForThePerson")
        gd = mbel["preferredNameEntityForThePerson"]
        fgn = (checkkey(gd, "forname", givenname)
               || checkkey(gd, "personalName", givenname))
        if fgn
            return ("vn")
        end
    end

    if haskey(mbel, "variantNameEntityForThePerson")
        for var in mbel["variantNameEntityForThePerson"]
            fgn = (checkkey(gd, "forname", givenname)
                   || checkkey(gd, "personalName", givenname))
            if fgn
                return ("vn")
            end
        end
    end

    return ("")
end

function scoredateofdeath(mbel, dateofdeath, tolerance)::Int
    if dateofdeath == ""
        return 0
    end

    rm = match(r"[1-9][0-9]+", dateofdeath)
    rm != nothing || (@info "Ungültiges Sterbedatum '$(dateofdeath)'"; return false)
    dateref = parse(Int, rm.match)

    if haskey(mbel, "dateOfDeath")
        rm = match(r"([1-9][0-9]{3})|([1-9][0-9]{2})", mbel["dateOfDeath"][1])
        date = parse(Int, rm.match)
        delta = abs(dateref - date)
        if delta <= tolerance
            return ScSD + tolerance - delta
        else
            return -1
        end
    else
        return 0
    end
    @warn "Dieser Punkt sollte nicht erreicht werden."
    return 0
end

"""

Bewerte einen Eintrag mit `givenname`.
"""
function scoremember(mbel, givenname, dateofdeath, tolerance)

    # Handelt es sich um eine Person?
    isperson = reduce(|, (map(val -> occursin("Person", val), mbel["type"])))
    if !isperson
         return Pair(0, "")
    end

    tot = 0

    # Der Vorname muss in einem Feld für den Vornamen vorkommen
    # Beispiel: Johann Thomas, Brixen, Bischof

    # Name
    mk = matchkeyname(mbel, givenname)
    if mk == "vn"
        tot += ScVN
    else
        # Name aus Vorname und Familienname nicht gefunden
        return Pair(0, "")
    end

    # Sterbedatum
    scdateod = scoredateofdeath(mbel, dateofdeath, tolerance)
    # println("Ergebnis bei Datum: ", scdateod)
    if scdateod > 0
        tot += scdateod
        mk = mk * " " * "sd"
    elseif scdateod < 0
        return Pair(0, mk) # Abweichung zu groß
    end

    # Amt
    if valueprofession(mbel) != ""
        tot += ScBA
        mk = mk * " " * "ba"
    end

    # Indiv. Person
    if valuetype(mbel) == "DifferentiatedPerson"
        tot += ScIP
        mk = mk * " " * "ip"
    end

    return Pair(tot, mk)
end

"""

Befülle eine Ergebniszeile `row` mit Werten aus `mbel`.
"""
function writerow!(row::DataFrameRow, record::QRecord)
    # rgm = match(r"/([[:alnum:]-]+)$", get(record, "id", ""))
    # row[:ID_GND_GND] = rm == nothing ? "" : rgm[1]
    row[:ID_GND_GND] = record["gndIdentifier"]
    row[:pName_GND] = record["preferredName"]
    row[:Amt_GND] = valueprofession(record)
    row[:Sterbedatum_GND] = valuedateofdeath(record)
    row[:Typ_GND] = valuetype(record)
    row[:Qualitaet_GND] = record["muster"]
    row[:QRang_GND] = getrank(record["muster"])
    row[:Abfrage_GND] = record["abfrage"]
    aid = valueids(record)
    row[:ID_Wikidata_GND] = aid["wikidata"]
    row[:ID_VIAF_GND] = aid["viaf"]
    row[:URL_Wikipedia_GND] = valuewikipedia(record)
end

function valueprofession(record::QRecord)::String
    if haskey(record, "professionOrOccupation")
        ga = record["professionOrOccupation"]
        for gd in ga
            gp = gd["label"]
            occursin(r"((b|B)ischof|(E|e)lekt)", gp) && return(gp)
        end
        return ""
    else
        return ""
    end
end

function valuedateofdeath(record::QRecord)::String
    if haskey(record, "dateOfDeath")
        return(record["dateOfDeath"][1])
    else
        return("")
    end
end

function valuetype(record)::String
    if haskey(record, "type")
        ga = record["type"]
        for ttest in ["DifferentiatedPerson", "Person", "UndifferentiatedPerson"]
            ttest in ga && return(ttest)
        end
        return ""
    else
        return ""
    end
end

function evaluateip!(rcel)::Bool
    fip = "DifferentiatedPerson" == valuetype(rcel)
    if fip
        push!(rcel["amuster"], "ip")
    end
    fip
end

function valuewikipedia(record)
    if haskey(record, "wikipedia")
        ga = record["wikipedia"]
        if haskey(ga[1], "id")
            return ga[1]["id"]
        else
            @warn "Im Element Wikipedia 'id' nicht gefunden " * record["preferredName"]
        end
    else
        return ""
    end
end

function valueids(record)
    keys = ["wikidata", "viaf"]
    did = Dict([k => "" for k in keys])
    rgx = r"[^/]+$"

    function filldid(sameAs)
        valid = sameAs["id"]
        for key in keys
            if occursin(key * ".org", valid)
                rm = match(rgx, valid)
                if rm != nothing
                    did[key] = rm.match
                else
                    @warn ("Schlüssel " * key *
                           " im Feld 'id' nicht gefunden für " * record["preferredName"])
                end
            end
        end
    end

    haskeyid(sameAs) = haskey(sameAs, "id")

    if haskey(record, "sameAs")
        foreach(filldid, filter(haskeyid, record["sameAs"]))
    end

    return did
end

function valueidsT(record)
    # Implementierung mit Elementen von Transducres
    # nicht effizient
    keys = ["wikidata", "viaf"]
    did = Dict([k => "" for k in keys])

    function filldid(p) # p[1] key, p[2] Vergleichswert
        rgm = match(r"[^/]+$", p[2])
        if rgm != nothing
            did[p[1]] = rgm.match
        else
            @warn ("Schlüssel " * key *
                   " im Feld 'id' nicht gefunden für " * record["preferredName"])
        end
    end

    function idtoval(gd)
        foreach(filldid,
                Map(key -> (key, gd["id"]))
                |> Filter(matchkey),
                keys)
    end

    matchkey(p) = occursin(p[1] * ".org", p[2])

    if haskey(record, "sameAs")
        foreach(idtoval,
                Filter(gd -> haskey(gd, "id")),
                record["sameAs"])
    end
    return did
end

function makeGNDDataFrame(df)
    # Übernehme alle Daten
    dfgnd = copy(df)

    for col in GNDSTRINGCOLS
        dfgnd[!, col] .= ""
    end
    dfgnd[!, :QRang_GND] .= RANKMAX

    dfgnd
end

"""
Zähle nicht-leere Datensätze in `df`. Wenn nur einzelne Spalten gewünscht sind,
übergebe man eine entsprechende View auf `df`.
"""
function countnotempty(df::AbstractDataFrame)
    ncol = length(names(df))
    dfc = DataFrame(fill(Int, ncol), names(df), 0)
    counts = fill(0, ncol)
    for row in eachrow(df), (i, v) in enumerate(row)
        v != "" && (counts[i] += 1)
    end
    push!(dfc, counts)
    dfc
end

"""
    clearbylevel!(df::AbstractDataFrame, matchkey::AbstractString, level::Int)

Lösche Datensätze, deren Qualität unter `matchkey`, bzw. `rank` liegt. Die Angaben
sind zur Sicherheit redundant.
"""
function clearbylevel!(df::AbstractDataFrame, matchkey::AbstractString, rank::Int)
    if rank != getrank(matchkey)
        error("Muster und Rang stimmen nicht überein")
    end
    colrank = df[!, :QRang_GND]
    ixr = colrank .> rank
    df[ixr, GNDSTRINGCOLS] .= "" # zulässig!
    df[ixr, :QRang_GND] .= RANKMAX
    count(df[!, :QRang_GND] .< RANKMAX) # Zahl der Datensätze, die erhalten blieben
end

"""
    clearbyindex!(df::AbstractDataFrame, ixr)

Lösche alle GND-Daten aus `df` mit Index `ixr`.
"""
function clearbyindex!(df::AbstractDataFrame, ixr)
    if size(df, 1) != length(ixr)
        @error Unterschiedliche Länge der Argumente
    end
    df[ixr, GNDSTRINGCOLS] .= "" # zulässig!
    df[ixr, :QRang_GND] .= RANKMAX
    count(ixr)
end

"""
    infolevel!(df::AbstractDataFrame, matchkey::AbstractString)

"""
function infolevel(df::AbstractDataFrame, matchkey::AbstractString)
    rank = getrank(matchkey)
    println("Das Muster '", matchkey, "' hat die Position: ", rank, ".")
    println("Datensätze insgesamt: ", size(df, 1))
    colrank = df[!, :QRang_GND]
    println("Rang größer als '", matchkey, "': ", count(colrank .> rank))
    rankmax = maximum(colrank)
    nvalid = count(colrank .< rankmax)
    ndel = count((colrank .< rankmax) .& (colrank .> rank))
    println("Davon kleiner maximaler Rang : ", ndel)
    println("Es bleiben erhalten: ", nvalid - ndel)
end

"""
    deltaID(ref, gs)

Vergleiche IDs: `ref` ist z.B. manuell bestätigt.

## Beispiel
```
deltaID(r[[:GND_ID, :ID_GND_GND]]...)
```
"""
function deltarow(vref, vq)
    if vq == ""
        vref == "" ? (return "vv") : (return "xv")
    elseif vref == ""
        return "vx"
    else
        vref == vq ? (return "GLEICH") : (return "UNGLEICH")
    end
end

"""
    getsda(dfgnd::AbstractDataFrame,
           dfocc::AbstractDataFrame)

Berechne die Differenz zwischen Sterbedatum und letztem Amt
"""
function getsda(dfgnd::AbstractDataFrame,
                dfocc::AbstractDataFrame)
    typeid = eltype(dfgnd[!, COLNAMEID])
    df = DataFrame([typeid, Int], [COLNAMEID, :sda])
    rowsda = Dict{Symbol, Union{typeid, Int}}()

    for row in eachrow(dfgnd)
        sdateod = row[:Sterbedatum_GND]
        iep = row[COLNAMEID]

        rowsda[COLNAMEID] = iep

        dateod = parseyear(sdateod)
        dateod == nothing && continue

        csoe = @from i in dfocc begin
            @where i.ID_Bischof == iep
            @select i.Amtsende
            @collect
        end

        length(csoe) == 0 && continue

        # Zähle nur 0 und größer
        csda = Int[]
        for soe in csoe
            dateoe = parseyear(soe)
            dateoe == nothing && continue
            #println("  Amtsende: ", dateoe)
            delta = dateod - dateoe
            delta >= 0 && push!(csda, delta)
        end
        if length(csda) > 0
            rowsda[:sda] = minimum(csda)
            push!(df, rowsda)
        end
    end
    return df
end

function parseyear(s)
    rgm = match(RGXYEAR, s)
    if rgm == nothing
        return nothing
    else
        return parse(Int, rgm[1])
    end
end

"""

Gib `missing` zurück, wenn kein gültiges Datum gefunden werden kann
"""
function parsedate(sdate::Union{AbstractString, Missing})
    rgxyear = r"[0-9]?[0-9]?[0-9]{2}"
    valdate = missing

    ismissing(sdate) && return valdate

    sdate in ("", "(?)", "?") && return valdate

    rgm = match(rgxyear, sdate)
    if rgm == nothing
        @warn ("Ungültiges Datum in: " * sdate)
    else
        valdate = parse(Int, rgm.match)
    end

    return valdate
end

function parsedate(sdate::Int)
    sdate
end

end # modul GND


text:
__precompile__(true)

"""
Implementaion of test functions for the real valued optimizers.
From the [wikipedia article](https://en.wikipedia.org/wiki/Test_functions_for_optimization)
for test functions.
"""
module TestFunctions

export sphere, euclidean_distance, rastrigin, ackley, rosenbrock, eggholder, beale, square_movingpeak, ellipse, stybliski, mccormick,
       himmelblau, griewank, schafferf6

"""
    sphere(x::AbstractVector{<:Number})

Calculate the square root of the squared sums from the vector `x`.

### Minimum

```math
f(0) = 0
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

### Examples

```jldoctest
julia> sphere([3.0, 4.0])
25.0
```
"""
@inline function sphere(x::AbstractVector{<:Number})
    s = zero(eltype(x))^2
    for e in x
        s += e^2
    end
    return s
end

"""
    euclidean_distance(x::AbstractVector{<:Number})

Calculate the square root of the squared sums from the vector `x`.

### Minimum

```math
f(0) = 0
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

### Examples

```jldoctest
julia> euclidean_distance([3.0, 4.0])
5.0
```
"""
@inline function euclidean_distance(x::AbstractVector{<:Number})
    sqrt(sphere(x))
end

"""
    rastrigin(x::AbstractVector{<:Number}; A::Number = 10)

Rastrigin function. See [Wikipedia](https://en.wikipedia.org/wiki/Rastrigin_function).

### Minimum

```math
f(0) = 0
```

### Recommanded searchspace

```math
-5.12 ≦ x ≦ 5.12
```

### Examples

```jldoctest
julia> rastrigin([0,0])
0
```
"""
@inline function rastrigin(x::AbstractVector{<:Number}; A::Number = 10)
    A*length(x) + sum(x.^2 - A.*cos.(2π.*x))
end

"""
    ackley(x::Number, y::Number)

Ackley's fuction.

### Minimum

```math
f(0, ..., 0) = 0
```

### Recommanded searchspace

```math
-32 ≦ x ≦ 32
```

### Examples

```jldoctest
julia> ackley(0.0, 0.0)
0.0
```
"""
@inline function ackley(x::AbstractVector{<:Number})
    # -20.0*ℯ^(-0.2*sqrt((x^2 + y^2)/2.0)) - ℯ^((cos(2π*x) + cos(2π*y))/2.0) + ℯ + 20.0
    -20*ℯ^(-0.2*sqrt(sum(x.^2)/length(x))) - ℯ^(sum(cos.(2π.*x))/length(x)) + ℯ + 20
end
# @inline ackley(x::AbstractVector{<:Number}) = ackley(x[1], x[2])

"""
    rosenbrock(x::AbstractVector{<:Number})

Rosenbrock function. See [Wikipedia](https://en.wikipedia.org/wiki/Rosenbrock_function).

### Minimum

```math
f(1, 1, ..., 1) = 0
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

### Examples

```jldoctest
julia> rosenbrock([1.0,1.0])
0.0
```
"""
function rosenbrock(x::AbstractVector{<:Number})
    s = zero(eltype(x))
    @inbounds for i in 1:(length(x)-1)
        s += 100*(x[i+1] - x[i]*x[i])^2 + (x[i] - 1)^2
    end
    s
end

"""
    eggholder(x::Number, y::Number)

Eggholder function.

### Minimum

```math
f(512, 404.2319) = -959.6407
```

### Recommanded searchspace

```math
-512 ≦ x,y ≦ 512
```

### Examples

```jldoctest
julia> eggholder(512, 404.2319)
-959.6406627106155
```
"""
@inline function eggholder(x::Number, y::Number)
    y += 47
    -y*sin(sqrt(abs(x/2 + y))) - x*sin(sqrt(abs(x - y)))
end
@inline eggholder(x::AbstractVector{<:Number}) = eggholder(x[1], x[2])

"""
    beale(x::Number, y::Number)

Beale's function.

### Minimum

```math
f(3, 0.5) = 0
```

### Recommanded searchspace

```math
-4.5 ≦ x,y ≦ 4.5
```

### Examples

```jldoctest
julia> beale(3, 0.5)
0
```
"""
@inline function beale(x::Number, y::Number)
    (1.5 - x + x*y)^2 + (2.25 - x + x*y*y)^2 + (2.625 - x + x*y*y*y)^3
end
@inline beale(x::AbstractVector{<:Number}) = beale(x[1], x[2])

"""
    moving_peak(x::AbstractVector{<:Number}; optimum::Vector{<:Number}=[0.0, 0.0])

Square function with adjustable global optimum.

### Minimum

```nath
f(optimum) = 0.02
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

# Examples

```jldoctest
julia> moving_peak([0.0, 0.0], optimum=[1.0, 1.0])
1.4142135623730951
```
"""
@inline function square_movingpeak(x::AbstractVector{<:Number}; optimum::Vector{<:Number}=[0, 0])
    sphere(x.-optimum)
end

"""
    ellipse(x::AbstractVector{<:Number}; a::Number=-2, b::Number=1)

Ellipse function recommanded by Thomas Rheinhardt in 2015. Default updatefunction for position and velocity in the PSO
may have problems with this function. A 2011 presented way with a gravity center and a more random selection
seems to be more robust.

### Minimum

```math
f(-0.5, -1.5) = 0.0
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

# Examples

```jldoctest
julia> ellipse(-0.5, -1.5)
0.0
```
"""
@inline function ellipse(x::Number, y::Number; a::Number=-2, b::Number=1)
    (x + y - a)*(x + y - a)/100 + (x - y - b)*(x - y - b)*100
end
@inline ellipse(x::AbstractVector{<:Number}; a::Number=-2, b::Number=1) = ellipse(x[1], x[2], a=a, b=b)

"""
    stybliski(x::AbstractVector{<:Number})

Styblinski function.

### Minimum

```math
f(-2.903534, ..., -2.903534) ≈ -39.16616*length(input)
```

### Recommanded searchspace

```math
-5 ≦ x ≦ 5
```

# Examples

```jldoctest
julia> stybliski([-2.903534, -2.903534])
-78.3323314075428
```
"""
@inline function stybliski(x::AbstractVector{<:Number})
    sum(x.^4 .- 16 .*x.^2 .+ 5 .* x)/2
end

"""
    mccormick(x::Number, y::Number)

McCormick's function.

### Minimum

```math
f(-0.54719, -1.54719) = -1.9133
```

### Recommanded searchspace

```math
-1.5 ≦ x ≦ 4
-3   ≦ y ≦ 4.5
```

### Examples

```jldoctest
julia> mccormick(0, 0)
1.0
```
"""
@inline function mccormick(x::Number, y::Number)
    sin(x + y) + (x - y)^2 - 1.5x + 2.5y + 1
end
@inline mccormick(x::AbstractVector{<:Number}) = mccormick(x[1], x[2])

"""
    himmelblau(x::Number, y::Number)

Himmelblau function.

### Minimum

```math
f(-0.270845, -0.923039) = 181.617 (local maximum)

f(3.0, 2.0) = 0.0
f(-2.805118,  3.131312) = 0.0
f(-3.77931 , -3.283186) = 0.0
f( 3.584428, -1.848126) = 0.0
```

### Recommanded searchspace

```math
-5 ≦ x ≦ 5
```

### Examples

```jldoctest
julia> himmelblau(3, 2)
0
```
"""
@inline function himmelblau(x::Number, y::Number)
    (x^2 + y - 11)^2 + (x + y^2 - 7)^2
end
@inline himmelblau(x::AbstractVector{<:Number}) = himmelblau(x[1], x[2])

"""
    griewank(x::AbstractVector{<:Number})

Griewank function. See [Wikipedia](https://en.wikipedia.org/wiki/Griewank_function).

### Minimum

```math
f(0, 0, ..., 0) = 0
```

### Recommanded searchspace

```math
-∞ ≦ x ≦ ∞
```

### Examples

```jldoctest
julia> griewank([0.0, 0.0])
0.0
```
"""
function griewank(x::AbstractVector{<:Number})
    1 + sphere(x)/4_000 - prod(cos.(x./(1:length(x))))
end

"""
    schafferf6(x::AbstractVector{<:Number}; optimum::Vector{<:Number}=[0.0, 0.0])

Square function with adjustable global optimum.

### Minimum

```nath
f(0, 0, ..., 0) = 0.0
```

### Recommanded searchspace

```math
-100 ≦ x ≦ 100
```

# Examples

```jldoctest
julia> schafferf6([0.0, 0.0])
0.0
```
"""
@inline function schafferf6(x::Number, y::Number)
    0.5 + (sin(sqrt(x^2 + y^2))^2 - 0.5)/((1 + 0.001*(x^2 + y^2))^2)
end
@inline schafferf6(x::AbstractVector{<:Number}) = schafferf6(x[1], x[2])

end

ext: jl


text:
"""
    PossibleEncodings

**fields**
- `encodings`           -- Array of encodings to try
- `iscorrectencoding`   -- callable object
- `lastcorrectencoding` -- index of correct encoding last time open was called
"""
mutable struct PossibleEncodings
  encodings :: Array{StringEncodings.Encodings.Encoding,1}
  iscorrectencoding :: Function
  lastcorrectencoding :: Int
  io :: IO
  PossibleEncodings(enc,ice) = new(enc,ice,0,IOStream(""))
end

function iscorrectencoding_logfile(io)
  firstline = readline(io)
  if firstline[1:nextind(firstline, 1, 7)] == "Circuit:"
    seekstart(io.stream) # good idea?
    return true
  else
    return false
  end
end

function tryopen!(fname::AbstractString, enc::PossibleEncodings, i)
  try_io = open(fname,enc.encodings[i])
  if try_io!=nothing
    if enc.iscorrectencoding(try_io)
      enc.io = try_io
      return true
    else
      close(try_io.stream)  # ???
    end
  end
  return false
end

function Base.open(fname::AbstractString, enc::PossibleEncodings)
  if enc.lastcorrectencoding != 0 &&
     tryopen!(fname,enc,enc.lastcorrectencoding)
    return enc.io
  end
  for i in eachindex(enc.encodings)
    if i!=enc.lastcorrectencoding && tryopen!(fname,enc,i)
      enc.lastcorrectencoding = i
      return enc.io
    end
  end
  throw(ErrorException("no valid encoding found for $fname"))
end


text:
in_channel = 3
out_channel = 5
N = 6

ug = SimpleWeightedGraph(N)
add_edge!(ug, 1, 2, 2); add_edge!(ug, 1, 3, 2); add_edge!(ug, 2, 3, 1)
add_edge!(ug, 3, 4, 5); add_edge!(ug, 2, 5, 2); add_edge!(ug, 3, 6, 2)

dg = SimpleWeightedDiGraph(N)
add_edge!(dg, 1, 3, 2); add_edge!(dg, 2, 3, 2); add_edge!(dg, 1, 6, 1)
add_edge!(dg, 2, 5, -2); add_edge!(dg, 3, 4, -2); add_edge!(dg, 3, 5, -1)

el_ug = Vector{Int64}[[2, 3], [1, 3, 5], [1, 2, 4, 6], [3], [2], [3]]
el_dg = Vector{Int64}[[3, 6], [3, 5], [4, 5], [], [], []]

@testset "weightedgraphs" begin
    @testset "GCNConv" begin
        gc = GCNConv(ug, in_channel=>out_channel)
        @test size(gc.weight) == (out_channel, in_channel)
        @test size(gc.bias) == (out_channel,)
        @test graph(gc.fg) === ug
    end

    @testset "ChebConv" begin
        k = 4
        cc = ChebConv(ug, in_channel=>out_channel, k)
        @test size(cc.weight) == (out_channel, in_channel, k)
        @test size(cc.bias) == (out_channel,)
        @test graph(cc.fg) == ug
        @test cc.k == k
        @test cc.in_channel == in_channel
        @test cc.out_channel == out_channel
    end

    @testset "GraphConv" begin
        gc = GraphConv(ug, in_channel=>out_channel)
        @test graph(gc.fg) == ug
        @test size(gc.weight1) == (out_channel, in_channel)
        @test size(gc.weight2) == (out_channel, in_channel)
        @test size(gc.bias) == (out_channel,)
    end

    @testset "GATConv" begin
        for heads = [1, 5]
            for concat = [true, false]
                gat = GATConv(ug, in_channel=>out_channel, heads=heads, concat=concat)
                @test graph(gat.fg) == ug
                @test size(gat.weight) == (out_channel * heads, in_channel)
                @test size(gat.bias) == (out_channel * heads,)
                @test size(gat.a) == (2*out_channel, heads, 1)
            end
        end
    end

    @testset "GatedGraphConv" begin
        num_layers = 3
        ggc = GatedGraphConv(ug, out_channel, num_layers)
        @test graph(ggc.fg) == ug
        @test size(ggc.weight) == (out_channel, out_channel, num_layers)
    end

    @testset "EdgeConv" begin
        ec = EdgeConv(ug, Dense(2*in_channel, out_channel))
        @test graph(ec.fg) == ug
    end
end


text:
# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md

using JuliaFEM
using JuliaFEM.Postprocess
using JuliaFEM.Testing

@testset "extrapolate stress from gauss points to nodes" begin
    X = Dict{Int, Vector{Float64}}(
        1 => [0.0, 0.0],
        2 => [6.0, 0.0],
        3 => [6.0, 6.0],
        4 => [0.0, 6.0],
        5 => [12.0, 0.0],
        6 => [12.0, 6.0])
    el1 = Element(Quad4, [1, 2, 3, 4])
    el2 = Element(Quad4, [2, 5, 6, 3])
    el1.id = 1
    el2.id = 2
    elements = [el1, el2]
    time = 0.0
    update!(elements, "geometry", X)
    update!(get_integration_points(el1), "stress", time => [1.0, 2.0, 3.0])
    update!(get_integration_points(el2), "stress", time => [2.0, 3.0, 4.0])
    field_name = "stress"
    field_dim = 3
    calc_nodal_values!(elements, field_name, field_dim, time)
    s1 = el1("stress", [0.0, 0.0], time)
    s2 = el2("stress", [0.0, 0.0], time)
    # visually checked, see blog post "Postprocessing stress"
    @test isapprox(s1, [1.125, 2.125, 3.125])
    @test isapprox(s2, [1.875, 2.875, 3.875])
end


text:
# idempotency
convert(::Type{Float65}, x::Float65) = x
# ambiguity reduction: first specify interconversion with Bool
convert(::Type{Float65}, x::Bool) = convert(Float65, convert(Float64,x))
convert(::Type{Bool}, x::Float65) = !iszero(x)
promote_rule(::Type{Float65}, ::Type{Bool}) = Float65
# ambiguity resolution: specify interconversion with Float64
convert(::Type{Float64}, x::Float65) = reflect(x.fp)
convert(::Type{Float65}, x::Float64) = Float65(x)
promote_rule(::Type{Float64}, ::Type{Float65}) = Float65
# special numerical types
convert(::Type{Float65}, x::Irrational) = convert(Float65, convert(Float64, x))
promote_rule(::Type{Float65}, ::Type{Irrational}) = Float65
# delegation with non-leaf types
convert(::Type{Float65}, x::Integer) = convert(Float65, convert(Float64, convert(Int64,x)))
convert(::Type{Float65}, x::AbstractFloat) = convert(Float65, convert(Float64,x))
convert(::Type{Integer}, x::Float65) = convert(Int64, convert(Float64, x))
convert(::Type{AbstractFloat}, x::Float65) = convert(Float64, x)
promote_rule(::Type{Float65}, ::Type{Integer}) = Float65
promote_rule(::Type{Float65}, ::Type{AbstractFloat}) = Float65
# autoprocessable types
for T in (:Int16, :Int32, :Int64, :Float16, :Float32,
          :(Rational{Int32}), :(Rational{Int64}))
    @eval begin
        convert(::Type{Float65}, x::$T) = Float65(convert(Float64,x))
        convert(::Type{$T}, x::Float65) = ($T)( convert(Float64,x) )
        promote_rule(::Type{Float65}, ::Type{$T}) = Float65
    end
end


text:
using StatsDiscretizations
using Test
using LinearAlgebra
using Random

Random.seed!(1)
Zs = rand(1000)

discr = StatsDiscretizations.samplehull_discretizer(Zs)

# these tests are actually applicable more broadly
#@test findfirst(discr, -100) == 1
@test discr[findfirst.(Ref(discr), Zs)] == discr.(Zs)

#
f = DiscretizedFunction(discr, mean)

#@test f(discr[1]) == -Inf
#@test f(-100) == -Inf
#@test f.(Zs) == mean.(discr.(Zs))

using JuMP
using ECOS

using SplitApplyCombine

model = Model(ECOS.Optimizer)

f = StatsDiscretizations.add_discretized_function!(model, discr)

@objective(model, Min, sum(  (f.(Zs) .-  Zs  ).^2 ))

optimize!(model)

fsolve = JuMP.value(f)

_output_dict = groupsum( z->discr(z), sort(Zs)) ./ groupcount( z->discr(z), sort(Zs))

@test maximum(abs.(collect(_output_dict .- fsolve.dictionary))) ≈ 0.0 atol=1e-5


text:
"""
    struct Parallel.MultipleDijkstraState{T, U}

An [`AbstractPathState`](@ref) designed for Parallel.dijkstra_shortest_paths calculation.
"""
struct MultipleDijkstraState{T <: Real,U <: Integer} <: AbstractPathState
    dists::Matrix{T}
    parents::Matrix{U}
end

"""
    Parallel.dijkstra_shortest_paths(g, sources=vertices(g), distmx=weights(g))

Compute the shortest paths between all pairs of vertices in graph `g` by running
[`dijkstra_shortest_paths`] for every vertex and using an optional list of source vertex `sources` and
an optional distance matrix `distmx`. Return a [`Parallel.MultipleDijkstraState`](@ref) with relevant
traversal information.
"""
function dijkstra_shortest_paths(g::AbstractGraph{U},
    sources::AbstractVector=vertices(g),
    distmx::AbstractMatrix{T}=weights(g)) where T <: Real where U

    n_v = nv(g)
    r_v = length(sources)

    # TODO: remove `Int` once julialang/#23029 / #23032 are resolved
    dists   = SharedMatrix{T}(Int(r_v), Int(n_v))
    parents = SharedMatrix{U}(Int(r_v), Int(n_v))

    @sync @distributed for i in 1:r_v
        state = LightGraphs.dijkstra_shortest_paths(g, sources[i], distmx)
        dists[i, :] = state.dists
        parents[i, :] = state.parents
    end

    result = MultipleDijkstraState(sdata(dists), sdata(parents))
    return result
end


text:
function string_literals()
    l1 = "literals\a"
    l2 = "literals\b"
    l3 = "literals\f"
    l4 = "literals\n"
    l5 = "literals\r"
    l6 = "literals\v"
    l7 = "literals\t"
    l8 = "+"
    @assert(l8 == "+")
    l9 = "+"
    @assert(l9 == "+")
    m = "literal\n           literal"
    s = "literal"
    t = "literal"
end

function integer_literals()
    l1 = 0b10110
    @assert(l1 == 22)
    l2 = 0o333
    @assert(l2 == 219)
    l3 = 50
    @assert(l3 == 50)
    l4 = 0x14c
    @assert(l4 == 332)
end

function floating_point_literals()
    l1 = 25.99999
    l2 = 26.11111
    l3 = 0.001
    l4 = 10.0
    l5 = 10000000000.0
    l6 = 3.14e-10
    l7 = 3.141593
    @assert(l3 == 0.001)
    @assert(l4 == 10)
    @assert(l5 == 10000000000)
    @assert(l6 == 3.14e-10)
    @assert(l7 == 3.141593)
end

function imaginary()
    l1 = 5im
end

function boolean_literals()
    l1 = 1 == true
    l2 = 1 == false
    l3 = true + l1
    l4 = false + l2
    @assert(l1 == 1)
    @assert(l2 == 0)
    @assert(l3 == 2)
    @assert(l4 == 0)
end

if abspath(PROGRAM_FILE) == @__FILE__
    string_literals()
    integer_literals()
    floating_point_literals()
    imaginary()
    boolean_literals()
end


text:
# Autogenerated wrapper script for CUDA_Toolkit_jll for x86_64-w64-mingw32-cuda+10.0
export libcublas, libcudadevrt, libcufft, libcupti, libcurand, libcusolver, libcusparse, libdevice, libnvtoolsext, libnvvm, nvdisasm, nvlink, ptxas

JLLWrappers.@generate_wrapper_header("CUDA_Toolkit")
JLLWrappers.@declare_library_product(libcublas, "cublas64_100.dll")
JLLWrappers.@declare_file_product(libcudadevrt)
JLLWrappers.@declare_library_product(libcufft, "cufft64_100.dll")
JLLWrappers.@declare_library_product(libcupti, "cupti64_100.dll")
JLLWrappers.@declare_library_product(libcurand, "curand64_100.dll")
JLLWrappers.@declare_library_product(libcusolver, "cusolver64_100.dll")
JLLWrappers.@declare_library_product(libcusparse, "cusparse64_100.dll")
JLLWrappers.@declare_file_product(libdevice)
JLLWrappers.@declare_library_product(libnvtoolsext, "nvToolsExt64_1.dll")
JLLWrappers.@declare_library_product(libnvvm, "nvvm64_33_0.dll")
JLLWrappers.@declare_executable_product(nvdisasm)
JLLWrappers.@declare_executable_product(nvlink)
JLLWrappers.@declare_executable_product(ptxas)
function __init__()
    JLLWrappers.@generate_init_header()
    JLLWrappers.@init_library_product(
        libcublas,
        "bin\\cublas64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_file_product(
        libcudadevrt,
        "lib\\cudadevrt.lib",
    )

    JLLWrappers.@init_library_product(
        libcufft,
        "bin\\cufft64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libcupti,
        "bin\\cupti64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libcurand,
        "bin\\curand64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libcusolver,
        "bin\\cusolver64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libcusparse,
        "bin\\cusparse64_100.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_file_product(
        libdevice,
        "share\\libdevice\\libdevice.10.bc",
    )

    JLLWrappers.@init_library_product(
        libnvtoolsext,
        "bin\\nvToolsExt64_1.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libnvvm,
        "bin\\nvvm64_33_0.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_executable_product(
        nvdisasm,
        "bin\\nvdisasm.exe",
    )

    JLLWrappers.@init_executable_product(
        nvlink,
        "bin\\nvlink.exe",
    )

    JLLWrappers.@init_executable_product(
        ptxas,
        "bin\\ptxas.exe",
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
type MetaEvent <: TrackEvent
    dT::Int
    metatype::UInt8
    data::Array{UInt8,1}
end

function ismetaevent(b::UInt8)
    b == 0xFF
end

function readmetaevent(dT::Int64, f::IO)
    # Meta events are 0xFF - type (1 byte) - variable length data length - data bytes
    skip(f, 1) # Skip the 0xff that starts the event
    metatype = read(f, UInt8)
    datalength = readvariablelength(f)
    data = read(f, UInt8, datalength)

    MetaEvent(dT, metatype, data)
end

function writeevent(f::IO, event::MetaEvent)
    writevariablelength(f, event.dT)
    write(f, META)
    write(f, event.metatype)
    writevariablelength(f, convert(Int64, length(event.data)))
    write(f, event.data)
end

export MetaEvent


text:
module MatDeforStVKModule

using FinEtools.FTypesModule: FInt, FFlt, FCplxFlt, FFltVec, FIntVec, FFltMat, FIntMat, FMat, FVec, FDataDict
import FinEtoolsDeforLinear.DeforModelRedModule: AbstractDeforModelRed, DeforModelRed3D, DeforModelRed2DStrain, DeforModelRed2DStress, DeforModelRed2DAxisymm, DeforModelRed1D, nstressstrain, nthermstrain
import FinEtoolsDeforLinear.MatDeforModule: AbstractMatDefor, stressvtot!, stressttov!, strainttov!
import ..MatDeforNonlinearModule: AbstractMatDeforNonlinear, totlag2currsymm!
import LinearAlgebra: Transpose, Diagonal, mul!
At_mul_B!(C, A, B) = mul!(C, Transpose(A), B)
A_mul_B!(C, A, B) = mul!(C, A, B)
import LinearAlgebra: eigen, eigvals, norm, cholesky, cross, dot, log, diagm, det

"""
	MatDeforStVK{MR<:AbstractDeforModelRed, MTAN<:Function, MUPD<:Function} <: AbstractMatDeforNonlinear

Type for triaxial St Venant-Kirchhoff hyperelastic material.

"""
struct  MatDeforStVK{MR<:AbstractDeforModelRed, MTAN<:Function, MUPD<:Function} <: AbstractMatDeforNonlinear
	mr::Type{MR} # model reduction type
	mass_density::FFlt # mass density
	E1::FFlt
	E2::FFlt
	E3::FFlt
	G12::FFlt
	G13::FFlt
	G23::FFlt
	nu12::FFlt
	nu13::FFlt
	nu23::FFlt
	tangentmoduli!::MTAN # Function to return the tangent moduli matrix
	update!::MUPD # Function to update the material state
	_D::FFltMat # cache the constant stiffness matrix of the material
end

################################################################################
# 3-D solid model
################################################################################

"""
    MatDeforStVK(mr::Type{DeforModelRed3D}, mass_density::FFlt, E1::FFlt, E2::FFlt, E3::FFlt, G12::FFlt, G13::FFlt, G23::FFlt, nu12::FFlt, nu13::FFlt, nu23::FFlt)

Create triaxial St Venant-Kirchhoff hyperelastic material.

In general, the material is assumed to be orthotropic. There is a
specialized constructor for an isotropic version.
"""
function MatDeforStVK(mr::Type{DeforModelRed3D}, mass_density::FFlt, E1::FFlt, E2::FFlt, E3::FFlt, G12::FFlt, G13::FFlt, G23::FFlt, nu12::FFlt, nu13::FFlt, nu23::FFlt)
	compliance =[1/E1      -nu12/E1    -nu13/E1  0   0   0;
				-nu12/E1     1/E2      -nu23/E2  0   0   0;
				-nu13/E1   -nu23/E2       1/E3   0   0   0;
				0           0           0      1/G12 0   0;
				0           0           0        0 1/G13 0;
				0           0           0        0   0 1/G23];
	_D = inv(compliance);
	_I3 = [1.0 0 0; 0 1.0 0; 0 0 1.0]
	function tangentmoduli3d!(self::MatDeforStVK, D::FFltMat, statev::FFltVec, Fn1::FFltMat, Fn::FFltMat, tn::FFlt, dtn::FFlt, loc::FFltMat, label::FInt)
		return totlag2currsymm!(D, self._D, Fn1)
	end
	function update3d!(self::MatDeforStVK, statev::FFltVec, cauchy::FFltVec, output::FFltVec, Fn1::FFltMat, Fn::FFltMat, tn::FFlt, dtn::FFlt, loc::FFltMat=zeros(3,1), label::FInt=0, quantity=:nothing)
		@assert length(cauchy) == nstressstrain(self.mr)
		# Note: the code below allocates. This may not be a big deal when the
		# code gets called only a few times, but in explicit dynamics
		# situations  this would be very expensive. The material type needs to
		# provide these temporary arrays as buffers.
		C = Fn1'*Fn1;# Cauchy-Green deformation tensor
		E = 1/2*(C-_I3)# Green-Lagrange strain
		Egl = fill(0.0, nstressstrain(self.mr))
		strainttov!(mr, Egl, E);
		S = self._D * Egl;
		St = fill(0.0, 3, 3)
		stressvtot!(mr, St, S)
		J = det(Fn1);
		cauchyt = Fn1*(St/J)*Fn1'; # Cauchy stress
		stressttov!(mr, cauchy, cauchyt)
		if quantity == :nothing
			#Nothing to be copied to the output array
		elseif quantity == :cauchy || quantity == :Cauchy
			(length(output) >= 6) || (output = zeros(6)) # make sure we can store it
			copyto!(output, cauchy);
		elseif quantity == :pressure || quantity == :Pressure
			output[1]  =  -sum(cauchy[1:3])/3.
		elseif quantity == :princCauchy || quantity == :princcauchy
			ep = eigen(cauchyt);
			(length(output) >= 3) || (output = zeros(3)) # make sure we can store it
			copyto!(output,  sort(ep.values, rev=true));
		elseif quantity==:vonMises || quantity==:vonmises || quantity==:von_mises || quantity==:vm
			s1=cauchy[1]; s2=cauchy[2]; s3=cauchy[3];
			s4=cauchy[4]; s5=cauchy[5]; s6=cauchy[6];
			(length(output) >= 1) || (output = zeros(1)) # make sure we can store it
			output[1] = sqrt(1.0/2*((s1-s2)^2+(s1-s3)^2+(s2-s3)^2+6*(s4^2+s5^2+s6^2)))
		end
		return output
	end
	return MatDeforStVK(mr, mass_density, E1::FFlt, E2::FFlt, E3::FFlt, G12::FFlt, G13::FFlt, G23::FFlt, nu12::FFlt, nu13::FFlt, nu23::FFlt,
		tangentmoduli3d!, update3d!, _D)
end

"""
    MatDeforStVK(mr::Type{MR}, E::FFlt, nu::FFlt) where {MR}

Create St Venant-Kirchhoff isotropic elastic material.

The mass density is the default value.
"""
function MatDeforStVK(mr::Type{MR}, E::FFlt, nu::FFlt) where {MR}
	mass_density = 1.0
	G = E / (2 * (1 + nu))
	return MatDeforStVK(mr, mass_density, E, E, E, G, G, G, nu, nu, nu)
end

"""
    MatDeforStVK(mr::Type{MR}, E::FFlt, nu::FFlt) where {MR}

Create St Venant-Kirchhoff isotropic elastic material.

The mass density is the default value.
"""
function MatDeforStVK(mr::Type{MR}, E1::FFlt, E2::FFlt, E3::FFlt, G12::FFlt, G13::FFlt, G23::FFlt, nu12::FFlt, nu13::FFlt, nu23::FFlt) where {MR}
	mass_density = 1.0
	return MatDeforStVK(mr, mass_density, E1::FFlt, E2::FFlt, E3::FFlt, G12::FFlt, G13::FFlt, G23::FFlt, nu12::FFlt, nu13::FFlt, nu23::FFlt)
end

end


text:
# Higher-order scalar stochastic diffential equations

# Scalar SDE type ————————————————————————

abstract type ScalarSde{T} <: AbstractSde{T} end

"""Scalar stochastic differential equation (SDE)

The differential equation is given by:
    a[1]\\*y(t) + a[2]⋅(dy/dt)(t) + ⋯  + a[p]\\*(dᵖ⁻¹y/dᵖ⁻¹t) + (dᵖy/dᵖt)(t) = ϵ(t)
"""
struct Sde{T} <: ScalarSde{T}
  "Parameter vector"
  a::Vector{T}
  "Driving noise standard deviation"
  σϵ::T
  function Sde(a::Vector{T},σϵ::T) where {T}
    (isreal(σϵ) && real(σϵ)>0) || error("σϵ must be positive")
    new{T}(a,σϵ)
  end
end

function Sde(a::Vector{T},σϵ::U) where {T,U}
  V = promote_type(T,U)
  return Sde(convert(Vector{V},a),convert(V,σϵ))
end

Sde(θ::NamedTuple,σϵ) = Sde(θ.a,σϵ)

Sde(eq::Sde) = eq

function Sde(a;σy=one(eltype(a)))
  g = std(Sde(a,one(eltype(a))))
  σϵ = σy/g
  return Sde(a,σϵ)
end

function Base.convert(::Type{AbstractSde{T}},x::Sde) where {T}
  return Sde(convert(Vector{T},x.a),convert(T,x.σϵ))
end

function Base.show(io::IO,eq::Sde)
  p = order(eq)
  print(io,"SDE($p) ")
  cc = map(enumerate(eq.a)) do (i,a)
    ystr = i==1 ? "y" : @sprintf "dₜ%sy" expchar(i-1)
    @sprintf "%0.3g⋅%s" a ystr
  end
  cf = @sprintf "dₜ%sy" expchar(p)
  c = [cc;cf]
  lhs = join(c," + ")
  σϵstr = @sprintf "σϵ = %0.3g" eq.σϵ
  print(io,"$lhs = ϵ | $σϵstr")
end

function Base.show(io::IO,::MIME"text/plain",eq::Sde)
  print(io,"Scalar Stochastic Differential Equation\n",eq)
end

label(m::T) where T<:TimeSeriesModel = T.name

label(eq::ScalarSde) = "SDE($(order(eq)))"

order(eq::Sde) = length(eq.a)

statedim(eq::ScalarSde) = order(eq)

measdim(eq::ScalarSde) = 1

# Zygote-compatible code for a unit matrix
eye(T,n) = Matrix{T}(I,n,n)
Zygote.@nograd eye

function Base.convert(::Type{VectorSde},eq::Sde{T}) where {T}
  p = order(eq)
  # A = vcat([zeros(T,p-1,1) I],-eq.a') # Fails in Zygote v0.4.20
  A = vcat(hcat(zeros(T,p-1,1),eye(T,p-1)),-eq.a') # works with Zygote
  C = [one(T) zeros(T,1,p-1)]
  # Q = vcat(zeros(T,p-1,p),hcat(zeros(T,1,p-1),eq.σϵ^2)) # Fails in Zygote v0.4.20
  Q = vcat(zeros(T,p-1,p),hcat(zeros(T,1,p-1),[eq.σϵ^2])) # works in Zygote
  R = zeros(T,1,1)
  return VectorSde(A,C,Q,R)
end

ScalarSdeProcess{T,U} = SdeProcess{T,U} where {T,U<:ScalarSde{T}}

function Base.convert(::Type{VectorSdeProcess},proc::ScalarSdeProcess)
  eq = convert(VectorSde,proc.eq)
  return SdeProcess(eq,proc.state)
end

label(proc::ScalarSdeProcess) = label(proc.eq)

# Method required to fix method ambiguity for `convert`.
Base.convert(::Type{T},proc::T) where T<:ScalarSdeProcess = proc

function Statistics.std(eq::Sde)
  mv = convert(VectorSde,eq)
  Σs = statcovz(mv)
  σy = √Σs[1,1]
  return σy
end

statcovz(eq::Sde) = statcovz(convert(VectorSde,eq))

function Statistics.mean(eqs::Vector{<:Sde})
  a = mean(getproperty.(eqs,:a))
  σϵ = mean(getproperty.(eqs,:σϵ))
  return Sde(a,σϵ)
end

"""
$SIGNATURES

Is `eq` a stable differential equation?
"""
isstable(eq::ScalarSde) = all(real.(roots(eq)).<0)

"""
$SIGNATURES

Check whether `eq` a stable differential equation. Raise error if not,
otherwise return `eq`.
"""
function checkstable(eq::Sde)
  isstable(eq) ||  error("Unstable model $eq")
  return eq
end

function expchar(i)
  expchars = collect("¹²³⁴⁵⁶⁷⁸⁹")
  i≤9 ? string(expchars[i]) : "($i)"
end

describe_model(m::TimeSeriesModel) = println(m)

"
$SIGNATURES

Describe a stochastic differential equation
"
function describe_model(eq::ScalarSde)
  print("$eq | ")
  p = order(eq)
  if isstable(eq)
    σy = std(eq)
    @printf "σy = %0.3g" σy
  else
    printstyled("unstable\n",color=:yellow)
  end
  r = roots(eq)
  println("\nRoots")
  printroots(r)
end


text:
#=
  Euler #18 in Julia.

  Problem 18:
  """
  By starting at the top of the triangle below and moving to adjacent
  numbers on the row below, the maximum total from top to bottom is 23.

  3
  7 4
  2 4 6
  8 5 9 3

  That is, 3 + 7 + 4 + 9 = 23.

  Find the maximum total from top to bottom of the triangle below:

   75
   95 64
   17 47 82
   18 35 87 10
   20 04 82 47 65
   19 01 23 75 03 34
   88 02 77 73 07 63 67
   99 65 04 28 06 16 70 92
   41 41 26 56 83 40 80 70 33
   41 48 72 33 47 32 37 16 94 29
   53 71 44 65 25 43 91 52 97 51 14
   70 11 33 28 77 73 17 78 39 68 17 57
   91 71 52 38 17 14 91 43 58 50 27 29 48
   63 66 04 68 89 53 67 30 73 16 69 87 40 31
   04 62 98 27 23 09 70 98 73 93 38 53 60 04 23

  NOTE: As there are only 16384 routes, it is possible to solve this problem
  by trying every route. However, Problem 67, is the same challenge with a
  triangle containing one-hundred rows; it cannot be solved by brute force,
  and requires a clever method! ;o)
  """

  This Julia program was created by Hakan Kjellerstrand, hakank@gmail.com
  See also my Julia page: http://www.hakank.org/julia/

=#

include("Euler.jl")

# Recursive function for euler18()
function pp(row, col, sum, tri, max_val)
    if sum > max_val
        max_val = sum
    end
    row += 1
    if row <= length(tri)
        for i in 0:1
            max_val = pp(row, col + i, sum + tri[row][col + i], tri, max_val)
        end
    end
    return max_val
end

# 0.00015198s
function euler18a()
    triangle  =
         [[75],
         [95,64],
         [17,47,82],
         [18,35,87,10],
         [20, 4,82,47,65],
         [19, 1,23,75, 3,34],
         [88, 2,77,73, 7,63,67],
         [99,65, 4,28, 6,16,70,92],
         [41,41,26,56,83,40,80,70,33],
         [41,48,72,33,47,32,37,16,94,29],
         [53,71,44,65,25,43,91,52,97,51,14],
         [70,11,33,28,77,73,17,78,39,68,17,57],
         [91,71,52,38,17,14,91,43,58,50,27,29,48],
         [63,66, 4,68,89,53,67,30,73,16,69,87,40,31],
         [ 4,62,98,27,23, 9,70,98,73,93,38,53,60, 4,23]];
    max_val = 0
    max_val = pp(1, 1, triangle[1][1], triangle, max_val)

    return max_val
end

run_euler(euler18a)

max_stars_repo_name: Wikunia/hakank

max_issues_repo_name: Wikunia/hakank

max_forks_repo_name: Wikunia/hakank


text:
using DelayNetPlots
using Test

@testset "DelayNetPlots.jl" begin
    # Write your tests here.
end

max_stars_repo_name: ddahlbom/DelayNetPlots

max_issues_repo_name: ddahlbom/DelayNetPlots

max_forks_repo_name: ddahlbom/DelayNetPlots


text:
# Bogumił Kamiński, 2022

# Codes for chapter 12

# Codes for section 12.1

# Code for listing 12.1

import Downloads
using SHA
git_zip = "git_web_ml.zip"
if !isfile(git_zip)
    Downloads.download("https://snap.stanford.edu/data/" *
                       "git_web_ml.zip",
                       git_zip)
end
isfile(git_zip)
open(sha256, git_zip) == [0x56, 0xc0, 0xc1, 0xc2,
                          0xc4, 0x60, 0xdc, 0x4c,
                          0x7b, 0xf8, 0x93, 0x57,
                          0xb1, 0xfe, 0xc0, 0x20,
                          0xf4, 0x5e, 0x2e, 0xce,
                          0xba, 0xb8, 0x1d, 0x13,
                          0x1d, 0x07, 0x3b, 0x10,
                          0xe2, 0x8e, 0xc0, 0x31]

# Code for opening a zip archive

import ZipFile
git_archive = ZipFile.Reader(git_zip)

# Code for listing 12.2

function ingest_to_df(archive::ZipFile.Reader, filename::AbstractString)
    idx = only(findall(x -> x.name == filename, archive.files))
    return CSV.read(read(archive.files[idx]), DataFrame)
end

# Code for working with zip archive

git_archive.files

git_archive.files[2].name

findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files)
findall(x -> x.name == "", git_archive.files)

only(findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files))
only(findall(x -> x.name == "", git_archive.files))

# Code for listing 12.3

using CSV
using DataFrames
edges_df = ingest_to_df(git_archive, "git_web_ml/musae_git_edges.csv");
classes_df = ingest_to_df(git_archive, "git_web_ml/musae_git_target.csv");
close(git_archive)
summary(edges_df)
describe(edges_df, :min, :max, :mean, :nmissing, :eltype)
summary(classes_df)
describe(classes_df, :min, :max, :mean, :nmissing, :eltype)

# Code for updating data frame columns using broadcasting

edges_df .+= 1
classes_df.id .+= 1

# Code for examples of data frame broadcasting

df = DataFrame(a=1:3, b=[4, missing, 5])
df .^ 2
coalesce.(df, 0)
df .+ [10, 11, 12]

# Code for checking the order of :id column in a data frame

classes_df.id == axes(classes_df, 1)

# Code for the difference between ! and : in broadcasting assignment

df = DataFrame(a=1:3, b=1:3)
df[!, :a] .= "x"
df[:, :b] .= "x"
df

# Code for the difference between ! and : in assignment

df = DataFrame(a=1:3, b=1:3, c=1:3)
df[!, :a] = ["x", "y", "z"]
df[:, :b] = ["x", "y", "z"]
df[:, :c] = [11, 12, 13]
df

# Codes for section 12.2

# Code for listing 12.4

using Graphs
gh = SimpleGraph(nrow(classes_df))
for (src, dst) in eachrow(edges_df)
    add_edge!(gh, src, dst)
end
gh
ne(gh)
nv(gh)

# Code for iterator destruction in iteration specification

mat = [1 2; 3 4; 5 6]
for (x1, x2) in eachrow(mat)
    @show x1, x2
end

# Code for getting degrees of nodes in the graph

degree(gh)

# Code for adding a column to a data frame

classes_df.deg = degree(gh)

# Code for the difference between ! and : when adding a column

df = DataFrame()
x = [1, 2, 3]
df[!, :x1] = x
df[:, :x2] = x
df
df.x1 === x
df.x2 === x
df.x2 == x

# Code for creating a column using broadcasting

df.x3 .= 1
df

# Code for edge iterator of a graph

edges(gh)

e1 = first(edges(gh))
dump(e1)
e1.src
e1.dst

# Code for listing 12.5

function deg_class(gh, class)
    deg_ml = zeros(Int, length(class))
    deg_web = zeros(Int, length(class))
    for edge in edges(gh)
        a, b = edge.src, edge.dst
        if class[b] == 1
            deg_ml[a] += 1
        else
            deg_web[a] += 1
        end
        if class[a] == 1
            deg_ml[b] += 1
        else
            deg_web[b] += 1
        end
    end
    return (deg_ml, deg_web)
end

# Code for computing machine learning and web neighbors for gh graph

classes_df.deg_ml, classes_df.deg_web =
deg_class(gh, classes_df.ml_target)

# Code for checking type stability of deg_class function

@time deg_class(gh, classes_df.ml_target);
@code_warntype deg_class(gh, classes_df.ml_target)

# Code for checking the classes_df summary statistics

describe(classes_df, :min, :max, :mean, :std)

# Code for average degree of node in the graph

2 * ne(gh) / nv(gh)

# Code for checking correctness of computations

classes_df.deg_ml + classes_df.deg_web == classes_df.deg

# Code for showing that DataFrames.jl checks consistency of stored objects

df = DataFrame(a=1, b=11)
push!(df.a, 2)
df

# Codes for section 12.3

# Code for computing groupwise means of columns

using Statistics
for type in [0, 1], col in ["deg_ml", "deg_web"]
    println((type, col, mean(classes_df[classes_df.ml_target .== type, col])))
end

gdf = groupby(classes_df, :ml_target)

combine(gdf,
        :deg_ml => mean => :mean_deg_ml,
        :deg_web => mean => :mean_deg_web)

using DataFramesMeta
@combine(gdf,
         :mean_deg_ml = mean(:deg_ml),
         :mean_deg_web = mean(:deg_web))

# Code for simple plotting of relationship between developer degree and type

using Plots
scatter(classes_df.deg_ml, classes_df.deg_web;
        color=[x == 1 ? "black" : "gray" for x in classes_df.ml_target],
        xlabel="degree ml", ylabel="degree web", labels=false)

# Code for aggregation of degree data

agg_df = combine(groupby(classes_df, [:deg_ml, :deg_web]),
                 :ml_target => (x -> 1 - mean(x)) => :web_mean)

# Code for comparison how Julia parses expressions

:ml_target => (x -> 1 - mean(x)) => :web_mean
:ml_target => x -> 1 - mean(x) => :web_mean

# Code for aggregation using DataFramesMeta.jl

@combine(groupby(classes_df, [:deg_ml, :deg_web]),
         :web_mean = 1 - mean(:ml_target))

# Code for getting summary information about the aggregated data frame

describe(agg_df)

# Code for log1p function

log1p(0)

# Code for listing 12.6

function gen_ticks(maxv)
    max2 = round(Int, log2(maxv))
    tick = [0; 2 .^ (0:max2)]
    return (log1p.(tick), tick)
end

log1pjitter(x) = log1p(x) - 0.05 + rand() / 10

using Random
Random.seed!(1234);
scatter(log1pjitter.(agg_df.deg_ml),
        log1pjitter.(agg_df.deg_web);
        zcolor=agg_df.web_mean,
        xlabel="degree ml", ylabel="degree web",
        markersize=2, markerstrokewidth=0, markeralpha=0.8,
        legend=:topleft, labels="fraction web",
        xticks=gen_ticks(maximum(classes_df.deg_ml)),
        yticks=gen_ticks(maximum(classes_df.deg_web)))

# Code for fitting logistic regression model

using GLM
glm(@formula(ml_target~log1p(deg_ml)+log1p(deg_web)), classes_df, Binomial(), LogitLink())

# Code for inspecting @formula result

@formula(ml_target~log1p(deg_ml)+log1p(deg_web))

# Code for inserting columns to a data frame

df = DataFrame(x=1:2)
insertcols!(df, :y => 4:5)
insertcols!(df, :y => 4:5)
insertcols!(df, :z => 1)

insertcols!(df, 1, :a => 0)
insertcols!(df, :x, :pre_x => 2)
insertcols!(df, :x, :post_x => 3, after=true)

max_stars_repo_name: Mo-Gul/JuliaForDataAnalysis

max_issues_repo_name: Mo-Gul/JuliaForDataAnalysis

max_forks_repo_name: Mo-Gul/JuliaForDataAnalysis


text:
# Use baremodule to shave off a few KB from the serialized `.ji` file
baremodule iODBC_jll
using Base
using Base: UUID
import JLLWrappers

JLLWrappers.@generate_main_file_header("iODBC")
JLLWrappers.@generate_main_file("iODBC", UUID("80337aba-e645-5151-a517-44b13a626b79"))
end  # module iODBC_jll


text:
# =========================================================================================
"""
    readoff{T <: AbstractFloat}(
        stream::IOStream,
              ::Type{T}=Float64
    )

Reads a surface model from the given OFF file.

!!! note
    This file type does not support charge models! Hence, the charge list of the returning
    `Model` object is empty and has to be set separately.

# Specification
<http://www.geomview.org/docs/html/OFF.html>

# Return type
`Model{T}`

# Alias

    readoff{T}(fname::String, ::Type{T}=Float64)

Reads the model using a file name rather than a `IOStream` object.
"""
function readoff(
        stream::IOStream,
              ::Type{T}=Float64
    ) where T <: AbstractFloat
    eof(stream) && return Model{T, Triangle{T}}()
    @assert readline(stream) == "OFF" "Invalid OFF file"

    # read number of nodes and elements
    numnodes, numelem = [parse(Int, s) for s in split(readline(stream))]

    nodes = readoff_nodes(stream, numnodes, T)
    Model(nodes, readoff_elements(stream, numelem, nodes))
end

function readoff(
        fname::String,
             ::Type{T}=Float64
    ) where T
    open(fh -> readoff(fh, T), fname)
end

# =========================================================================================
"""
    readoff_nodes{T <: AbstractFloat}(
        stream::IOStream,
        n     ::Int,
              ::Type{T}=Float64
    )

Reads the first `n` nodes from the given OFF file.

# Return type
`Vector{Vector{T}}`
"""
function readoff_nodes(
        stream::IOStream,
        n     ::Int,
              ::Type{T}=Float64
    ) where T <: AbstractFloat
    Vector{T}[[parse(T, e) for e in split(readline(stream))] for _ in 1:n]
end

# =========================================================================================
"""
    readoff_elements{T <: AbstractFloat}(
        stream::IOStream,
        n     ::Int,
        nodes ::Vector{Vector{T}},
              ::Type{T}=Float64
    )

Reads the first `n` elements from the given OFF file.

# Return type
`Vector{Triangle{T}}`
"""
function readoff_elements(
        stream::IOStream,
        n     ::Int,
        nodes ::Vector{Vector{T}}
    ) where T <: AbstractFloat
    Triangle{T}[
        Triangle(
            [nodes[parse(Int, e) + 1] for e in split(readline(stream))[2:end]]...
        ) for _ in 1:n
    ]
end


text:
const KEY_MAPPING = Dict{Char, Symbol}(
   '\x00' => Symbol("Ctrl@"),
   '\x01' => Symbol("CtrlA"),
   '\x02' => Symbol("CtrlB"),
   '\x03' => Symbol("CtrlC"),
   '\x04' => Symbol("CtrlD"),
   '\x05' => Symbol("CtrlE"),
   '\x06' => Symbol("CtrlF"),
   '\x07' => Symbol("CtrlG"),
   '\x08' => Symbol("CtrlH"),
   '\x09' => Symbol("CtrlI"),
   '\x0A' => Symbol("CtrlJ"),
   '\x0B' => Symbol("CtrlK"),
   '\x0C' => Symbol("CtrlL"),
   '\x0D' => Symbol("CtrlM"),
   '\x0E' => Symbol("CtrlN"),
   '\x0F' => Symbol("CtrlO"),
   '\x10' => Symbol("CtrlP"),
   '\x11' => Symbol("CtrlQ"),
   '\x12' => Symbol("CtrlR"),
   '\x13' => Symbol("CtrlS"),
   '\x14' => Symbol("CtrlT"),
   '\x15' => Symbol("CtrlU"),
   '\x16' => Symbol("CtrlV"),
   '\x17' => Symbol("CtrlW"),
   '\x18' => Symbol("CtrlX"),
   '\x19' => Symbol("CtrlY"),
   '\x1A' => Symbol("CtrlZ"),
   '\x1B' => Symbol("Ctrl["),
   '\x1C' => Symbol("Ctrl\\"),
   '\x1D' => Symbol("Ctrl]"),
   '\x1E' => Symbol("Ctrl^"),
   '\x1F' => Symbol("Ctrl_"),
)


text:
using Test
using Printf
using CSV, DataFrames, Tables
using HypothesisTests

PRS_DATA_PATH = get(ENV, "JULIA_PRS_TEST_DATA_PATH", nothing)
if PRS_DATA_PATH === nothing
    PRS_DATA_PATH = mktempdir()
    cd(PRS_DATA_PATH) do
        isdir("PRScs") || run(`git clone https://github.com/getian107/PRScs`)
        isfile("test.bim") || run(`ln -s PRScs/test_data/test.bim .`)
        isfile("sumstats.txt") || run(`ln -s PRScs/test_data/sumstats.txt .`)
        if !isdir("ldblk_1kg_eur")
            run(`wget -q https://www.dropbox.com/s/mt6var0z96vb6fv/ldblk_1kg_eur.tar.gz`)
            run(`tar xf ldblk_1kg_eur.tar.gz`)
        end
    end
end

#These tests check the following:
#Script runs using phi auto version
#Script runs using specified fi
#Script successfully saves results using out_path and with out_dir

function test_harness(;a=1,b=0.5,phi=1e-02,chr=22,niter=1000,out_header=false)
    a = repr(a)
    b = repr(b)
    phi = @sprintf("%1.0e", phi)
    chr = repr(chr)
    niter = repr(niter)

    cmd_jl = `julia --project -e "using PolygenicRiskScores; PolygenicRiskScores.main()" -- --ref_dir=$PRS_DATA_PATH/ldblk_1kg_eur --bim_prefix=$PRS_DATA_PATH/test --sst_file=$PRS_DATA_PATH/sumstats.txt --n_gwas=200000 --chrom=$chr --phi=$phi --n_iter=$niter --out_dir=$PRS_DATA_PATH/output_jl`
    if out_header
        push!(cmd_jl.exec, "--out_header")
    end
    run(cmd_jl)

    run(`python3 $PRS_DATA_PATH/PRScs/PRScs.py --ref_dir=$PRS_DATA_PATH/ldblk_1kg_eur --bim_prefix=$PRS_DATA_PATH/test --sst_file=$PRS_DATA_PATH/sumstats.txt --n_gwas=200000 --chrom=$chr --phi=$phi --n_iter=$niter --out_dir=$PRS_DATA_PATH/output_py`)

    output_jl = "output_jl_pst_eff_a$(a)_b$(b)_phi$(phi)_chr$(chr).txt"
    output_py = "output_py_pst_eff_a$(a)_b$(b)_phi$(phi)_chr$(chr).txt"

    beta_jl=nothing
    beta_py=nothing

    for (kind, output_path) in ((:jl, output_jl), (:py, output_py))
        output_path = joinpath(PRS_DATA_PATH, output_path)
        @test isfile(output_path)
        @test stat(output_path).size > 30_000
        header = [:CHR, :SNP, :BP, :A1, :A2, :BETA]
        if out_header
            f = CSV.File(output_path)
        else
            f = CSV.File(output_path; header=header)
        end
        for (idx,col) in enumerate(header)
            if out_header && kind == :jl
                @test Tables.columnnames[idx] == col
            end
            @test length(getproperty(f, col)) == 1000
        end
        if kind == :jl
            beta_jl = f.BETA
        else
            beta_py = f.BETA
        end
    end

    # FIXME: Add accuracy/comparison tests
    # Do a t-test to make sure the beta results are not significantly different
    @test pvalue(EqualVarianceTTest(beta_jl,beta_py)) > 0.01

    rm(joinpath(PRS_DATA_PATH, output_jl))
    rm(joinpath(PRS_DATA_PATH, output_py))
end

@testset "Basic Options" begin
    # Default run
    test_harness()
end

# run julia once with seed X
# get hash of output file
# run julia again with seed X
# test that hash of new output file == old output file