text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

# Content in this file is extracted from BinaryProvider.jl, see LICENSE.method

module PlatformEngines
using SHA, Logging, UUIDs, Random
import ...Pkg: Pkg, TOML, pkg_server, depots1

export probe_platform_engines!, parse_7z_list, parse_tar_list, verify,
       download_verify, unpack, package, download_verify_unpack,
       list_tarball_files, list_tarball_symlinks

# To reduce method invalidation, it's best to call a logging method that
# avoids introduction of backedges.
# See https://github.com/JuliaLang/julia/pull/35714
const logging_level = isdefined(Base.CoreLogging, :_invoked_min_enabled_level) ? Base.CoreLogging._invoked_min_enabled_level : Base.CoreLogging.min_enabled_level

# In this file, we setup the `gen_download_cmd()`, `gen_unpack_cmd()` and
# `gen_package_cmd()` functions by providing methods to probe the environment
# and determine the most appropriate platform binaries to call.

"""
    gen_download_cmd(url::AbstractString, out_path::AbstractString, hdrs::AbstractString...)

Return a `Cmd` that will download resource located at `url` and store it at
the location given by `out_path`.

This method is initialized by `probe_platform_engines!()`, which should be
automatically called upon first import of `BinaryProvider`.
"""
gen_download_cmd = (url::AbstractString, out_path::AbstractString, hdrs::AbstractString...) ->
    error("Call `probe_platform_engines!()` before `gen_download_cmd()`")

"""
    gen_unpack_cmd(tarball_path::AbstractString, out_path::AbstractString;
                   excludelist::Union{AbstractString, Nothing} = nothing)

Return a `Cmd` that will unpack the given `tarball_path` into the given
`out_path`.  If `out_path` is not already a directory, it will be created.
excludlist is an optional file which contains a list of files that is not unpacked
This option is mainyl used to exclude symlinks from extraction (see: `copyderef`)

This method is initialized by `probe_platform_engines!()`, which should be
automatically called upon first import of `BinaryProvider`.
"""
gen_unpack_cmd = (tarball_path::AbstractString, out_path::AbstractString,
                  excludelist::Union{AbstractString, Nothing} = nothing) ->
    error("Call `probe_platform_engines!()` before `gen_unpack_cmd()`")

"""
    gen_package_cmd(in_path::AbstractString, tarball_path::AbstractString)

Return a `Cmd` that will package up the given `in_path` directory into a
tarball located at `tarball_path`.

This method is initialized by `probe_platform_engines!()`, which should be
automatically called upon first import of `BinaryProvider`.
"""
gen_package_cmd = (in_path::AbstractString, tarball_path::AbstractString) ->
    error("Call `probe_platform_engines!()` before `gen_package_cmd()`")

"""
    gen_list_tarball_cmd(tarball_path::AbstractString)

Return a `Cmd` that will list the files contained within the tarball located at
`tarball_path`.  The list will not include directories contained within the
tarball.

This method is initialized by `probe_platform_engines!()`.
"""
gen_list_tarball_cmd = (tarball_path::AbstractString) ->
    error("Call `probe_platform_engines!()` before `gen_list_tarball_cmd()`")

"""
    parse_tarball_listing(output::AbstractString)

Parses the result of `gen_list_tarball_cmd()` into something useful.

This method is initialized by `probe_platform_engines!()`.
"""
parse_tarball_listing = (output::AbstractString) ->
    error("Call `probe_platform_engines!()` before `parse_tarball_listing()`")

"""
    parse_symlinks(output::AbstractString)

Returns a regex to parse symlinks from tarball listings.

This method is initialized by `probe_platform_engines!()`.
"""

parse_symlinks = () ->
    error("Call `probe_platform_engines!()` before `parse_symlinks()`")

"""
    probe_cmd(cmd::Cmd; verbose::Bool = false)

Returns `true` if the given command executes successfully, `false` otherwise.
"""
function probe_cmd(cmd::Cmd; verbose::Bool = false)
    if verbose
        @info("Probing $(cmd.exec[1]) as a possibility...")
    end
    try
        success(cmd)
        if verbose
            @info("  Probe successful for $(cmd.exec[1])")
        end
        return true
    catch
        return false
    end
end

already_probed = false

"""
    probe_symlink_creation(dest::AbstractString)

Probes whether we can create a symlink within the given destination directory,
to determine whether a particular filesystem is "symlink-unfriendly".
"""
function probe_symlink_creation(dest::AbstractString)
    while !isdir(dest)
        dest = dirname(dest)
    end

    # Build arbitrary (non-existent) file path name
    link_path = joinpath(dest, "binaryprovider_symlink_test")
    while ispath(link_path)
        link_path *= "1"
    end

    loglevel = logging_level(current_logger())
    try
        disable_logging(Logging.Warn)
        symlink("foo", link_path)
        return true
    catch e
        if isa(e, Base.IOError)
            return false
        end
        rethrow(e)
    finally
        disable_logging(loglevel-1)
        rm(link_path; force=true)
    end
end

"""
    probe_platform_engines!(;verbose::Bool = false)

Searches the environment for various tools needed to download, unpack, and
package up binaries.  Searches for a download engine to be used by
`gen_download_cmd()` and a compression engine to be used by `gen_unpack_cmd()`,
`gen_package_cmd()`, `gen_list_tarball_cmd()` and `parse_tarball_listing()`.
Running this function
will set the global functions to their appropriate implementations given the
environment this package is running on.

This probing function will automatically search for download engines using a
particular ordering; if you wish to override this ordering and use one over all
others, set the `BINARYPROVIDER_DOWNLOAD_ENGINE` environment variable to its
name, and it will be the only engine searched for. For example, put:

    ENV["BINARYPROVIDER_DOWNLOAD_ENGINE"] = "fetch"

within your `~/.juliarc.jl` file to force `fetch` to be used over `curl`.  If
the given override does not match any of the download engines known to this
function, a warning will be printed and the typical ordering will be performed.

Similarly, if you wish to override the compression engine used, set the
`BINARYPROVIDER_COMPRESSION_ENGINE` environment variable to its name (e.g. `7z`
or `tar`) and it will be the only engine searched for.  If the given override
does not match any of the compression engines known to this function, a warning
will be printed and the typical searching will be performed.

If `verbose` is `true`, print out the various engines as they are searched.
"""
function probe_platform_engines!(;verbose::Bool = false)
    global already_probed
    global gen_download_cmd, gen_list_tarball_cmd, gen_package_cmd
    global gen_unpack_cmd, parse_tarball_listing, parse_symlinks

    # Quick-escape for Pkg, since we do this a lot
    if already_probed
        return
    end

    # download_engines is a list of (test_cmd, download_opts_functor)
    # The probulator will check each of them by attempting to run `$test_cmd`,
    # and if that works, will set the global download functions appropriately.
    download_engines = [
        (`curl --help`, (url, path, hdrs...) ->
            `curl -H$hdrs -C - -\# -f -o $path -L $url`),
        (`wget --help`, (url, path, hdrs...) ->
            `wget --tries=5 --header=$hdrs -c -O $path $url`),
        (`fetch --help`, (url, path, hdrs...) -> begin
            isempty(hdrs) || error("`fetch` does not support passing headers")
            `fetch -f $path $url`
        end),
        (`busybox wget --help`, (url, path, hdrs...) ->
            `busybox wget --header=$hdrs -c -O $path $url`),
    ]
    Sys.isapple() && pushfirst!(download_engines,
        (`/usr/bin/curl --help`, (url, path, hdrs...) ->
            `/usr/bin/curl -H$hdrs -C - -\# -f -o $path -L $url`))

    # 7z is rather intensely verbose.  We also want to try running not only
    # `7z` but also a direct path to the `7z.exe` bundled with Julia on
    # windows, so we create generator functions to spit back functors to invoke
    # the correct 7z given the path to the executable:
    unpack_7z = (exe7z) -> begin
        return (tarball_path, out_path, excludelist = nothing) ->
        pipeline(pipeline(`$exe7z x $(tarball_path) -y -so`,
                 `$exe7z x -si -y -ttar -o$(out_path) $(excludelist === nothing ? [] : "-x@$(excludelist)")`);
                 stdout=devnull, stderr=devnull)
    end
    package_7z = (exe7z) -> begin
        return (in_path, tarball_path) ->
            pipeline(pipeline(`$exe7z a -ttar -so a.tar "$(joinpath(".",in_path,"*"))"`,
                     `$exe7z a -si $(tarball_path)`); stdout=devnull, stderr=devnull)
    end
    list_7z = (exe7z) -> begin
        return (path; verbose = false) ->
            pipeline(`$exe7z x $path -so`, `$exe7z l -ttar -y -si $(verbose ? ["-slt"] : [])`)
    end

    # the regex at the last position is meant for parsing the symlinks from verbose 7z-listing
    # "Path = ([^\r\n]+)\r?\n" matches the symlink name which is followed by an optional return and a new line
    # (?:[^\r\n]+\r?\n)+ = a group of non-empty lines (information belonging to one file is written as a block of lines followed by an empty line)
    # more info on regex and a powerful online tester can be found at https://regex101.com
    # Symbolic Link = ([^\r\n]+)"s) matches the source filename
    # Demo 7z listing of tar files:
    # 7-Zip [64] 16.04 : Copyright (c) 1999-2016 Igor Pavlov : 2016-10-04
    #
    #
    # Listing archive:
    # --
    # Path =
    # Type = tar
    # Code Page = UTF-8
    #
    # ----------
    # Path = .
    # Folder = +
    # Size = 0
    # Packed Size = 0
    # Modified = 2018-08-22 11:44:23
    # Mode = 0rwxrwxr-x
    # User = travis
    # Group = travis
    # Symbolic Link =
    # Hard Link =

    # Path = .\lib\libpng.a
    # Folder = -
    # Size = 10
    # Packed Size = 0
    # Modified = 2018-08-22 11:44:51
    # Mode = 0rwxrwxrwx
    # User = travis
    # Group = travis
    # Symbolic Link = libpng16.a
    # Hard Link =
    #
    # Path = .\lib\libpng16.a
    # Folder = -
    # Size = 334498
    # Packed Size = 334848
    # Modified = 2018-08-22 11:44:49
    # Mode = 0rw-r--r--
    # User = travis
    # Group = travis
    # Symbolic Link =
    # Hard Link =
    gen_7z = (p) -> (unpack_7z(p), package_7z(p), list_7z(p), parse_7z_list, r"Path = ([^\r\n]+)\r?\n(?:[^\r\n]+\r?\n)+Symbolic Link = ([^\r\n]+)"s)
    compression_engines = Tuple[]

    (tmpfile, io) = mktemp()
    write(io, "Demo file for tar listing (Pkg.jl)")
    close(io)

    for tar_cmd in [`tar`, `busybox tar`]
        # try to determine the tar list format
        local symlink_parser
        try
            # Windows 10 now has a `tar` but it needs the `-f -` flag to use stdin/stdout
            # The Windows 10 tar does not work on substituted drives (`subst U: C:\Users`)
            # If a drive letter is part of the filename, then tar spits out a warning on stderr:
            # "tar: Removing leading drive letter from member names"
            # Therefore we cd to tmpdir() first
            cd(tempdir()) do
                tarListing = read(pipeline(`$tar_cmd -cf - $(basename(tmpfile))`, `$tar_cmd -tvf -`), String)
            end
            # obtain the text of the line before the filename
            m = match(Regex("((?:\\S+\\s+)+?)$tmpfile"), tarListing)[1]
            # count the number of words before the filename
            nargs = length(split(m, " "; keepempty = false))
            # build a regex for catching the symlink:
            # "^l" = line starting with l
            # "(?:\S+\s+){$nargs} = nargs non-capturing groups of many non-spaces "\S+" and many spaces "\s+"
            # "(.+?)" = a non-greedy sequence of characters: the symlink
            # "(?: -> (.+?))?" = an optional group of " -> " followed by a non-greedy sequence of characters: the source of the link
            # "\r?\$" = matches the end of line with an optional return character for some OSes
            # Demo listings
            # drwxrwxr-x  0 sabae  sabae       0 Sep  5  2018 collapse_the_symlink/
            # lrwxrwxrwx  0 sabae  sabae       0 Sep  5  2018 collapse_the_symlink/foo -> foo.1
            # -rw-rw-r--  0 sabae  sabae       0 Sep  5  2018 collapse_the_symlink/foo.1
            # lrwxrwxrwx  0 sabae  sabae       0 Sep  5  2018 collapse_the_symlink/foo.1.1 -> foo.1
            # lrwxrwxrwx  0 sabae  sabae       0 Sep  5  2018 collapse_the_symlink/broken -> obviously_broken
            #
            # drwxrwxr-x sabae/sabae       0 2018-09-05 18:19 collapse_the_symlink/
            # lrwxrwxrwx sabae/sabae       0 2018-09-05 18:19 collapse_the_symlink/foo -> foo.1
            #
            # lrwxrwxr-x 1000/1000 498007696 2009-11-27 00:14:00 link1 -> source1
            # lrw-rw-r-- 1000/1000 1359020032 2019-06-03 12:02:03 link2 -> sourcedir/source2
            #
            # now a pathological link "2009 link with blanks"
            # this can only be tracked by determining the tar format beforehand:
            # lrw-rw-r-- 0 1000 1000 1359020032 Jul  8 2009 2009 link with blanks -> target with blanks
            symlink_parser = Regex("^l(?:\\S+\\s+){$nargs}(.+?)(?: -> (.+?))?\\r?\$", "m")
        catch
            # generic expression for symlink parsing
            # this will fail, if the symlink contains space characters (which is highly improbable, though)
            # "^l.+?" = a line starting with an "l" followed by a sequence of non-greedy characters
            # \S+? the filename consisting of non-space characters, the rest as above
            symlink_parser = r"^l.+? (\S+?)(?: -> (.+?))?\r?$"m
        end
        # Some tar's aren't smart enough to auto-guess decompression method. :(
        unpack_tar = (tarball_path, out_path, excludelist = nothing) -> begin
            Jjz = "z"
            if endswith(tarball_path, ".xz")
                Jjz = "J"
            elseif endswith(tarball_path, ".bz2")
                Jjz = "j"
            end
            return `$tar_cmd --no-same-owner -mx$(Jjz)f $(tarball_path) -C$(out_path) $(excludelist === nothing ? [] : "-X$(excludelist)")`
        end
        package_tar = (in_path, tarball_path) -> begin
            Jjz = "z"
            if endswith(tarball_path, ".xz")
                Jjz = "J"
            elseif endswith(tarball_path, ".bz2")
                Jjz = "j"
            end
            return `$tar_cmd -c$(Jjz)f $tarball_path -C$(in_path) .`
        end
        list_tar = (in_path; verbose = false) -> begin
            Jjz = "z"
            if endswith(in_path, ".xz")
                Jjz = "J"
            elseif endswith(in_path, ".bz2")
                Jjz = "j"
            end
            return `$tar_cmd $(verbose ? "-t$(Jjz)vf" : "-t$(Jjz)f") $in_path`
        end
        push!(compression_engines, (
            `$tar_cmd --help`,
            unpack_tar,
            package_tar,
            list_tar,
            parse_tar_list,
            symlink_parser
        ))
    end
    rm(tmpfile, force = true)

    # For windows, we need to tweak a few things, as the tools available differ
    @static if Sys.iswindows()
        # For download engines, we will most likely want to use powershell.
        # Let's generate a functor to return the necessary powershell magics
        # to download a file, given a path to the powershell executable
        psh_download = (psh_path) -> begin
            return (url, path, hdrs...) -> begin
                webclient_code = """
                [System.Net.ServicePointManager]::SecurityProtocol =
                    [System.Net.SecurityProtocolType]::Tls12;
                \$webclient = (New-Object System.Net.Webclient);
                \$webclient.UseDefaultCredentials = \$true;
                \$webclient.Proxy.Credentials = \$webclient.Credentials;
                \$webclient.Headers.Add("user-agent", \"Pkg.jl (https://github.com/JuliaLang/Pkg.jl)\");
                """
                for hdr in hdrs
                    key, val = split(hdr, r":\s*", limit=2)
                    webclient_code *= """
                    \$webclient.Headers.Add($(repr(key)), $(repr(val)));
                    """
                end
                webclient_code *= """
                \$webclient.DownloadFile(\"$url\", \"$path\")
                """
                replace(webclient_code, "\n" => " ")
                return `$psh_path -NoProfile -Command "$webclient_code"`
            end
        end

        # We want to search both the `PATH`, and the direct path for powershell
        psh_path = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe")
        prepend!(download_engines, [
            (`$psh_path -Command ""`, psh_download(psh_path))
        ])
        prepend!(download_engines, [
            (`powershell -Command ""`, psh_download(`powershell`))
        ])

        # We greatly prefer `7z` as a compression engine on Windows
        prepend!(compression_engines, [(`7z --help`, gen_7z("7z")...)])

        # For purposes of in-buildtree execution, we look in `bin`
        exe7z = joinpath(Sys.BINDIR, "7z.exe")
        prepend!(compression_engines, [(`$exe7z --help`, gen_7z(exe7z)...)])

        # But most commonly, we'll find `7z` sitting in `libexec`, bundled with Julia
        exe7z = joinpath(Sys.BINDIR, "..", "libexec", "7z.exe")
        prepend!(compression_engines, [(`$exe7z --help`, gen_7z(exe7z)...)])
    end

    # Allow environment override
    if haskey(ENV, "BINARYPROVIDER_DOWNLOAD_ENGINE")
        engine = ENV["BINARYPROVIDER_DOWNLOAD_ENGINE"]
        es = split(engine)
        dl_ngs = filter(e -> e[1].exec[1:length(es)] == es, download_engines)
        if isempty(dl_ngs)
            all_ngs = join([d[1].exec[1] for d in download_engines], ", ")
            warn_msg  = "Ignoring BINARYPROVIDER_DOWNLOAD_ENGINE as its value "
            warn_msg *= "of `$(engine)` doesn't match any known valid engines."
            warn_msg *= " Try one of `$(all_ngs)`."
            @warn(warn_msg)
        else
            # If BINARYPROVIDER_DOWNLOAD_ENGINE matches one of our download engines,
            # then restrict ourselves to looking only at that engine
            download_engines = dl_ngs
        end
    end

    if haskey(ENV, "BINARYPROVIDER_COMPRESSION_ENGINE")
        engine = ENV["BINARYPROVIDER_COMPRESSION_ENGINE"]
        es = split(engine)
        comp_ngs = filter(e -> e[1].exec[1:length(es)] == es, compression_engines)
        if isempty(comp_ngs)
            all_ngs = join([c[1].exec[1] for c in compression_engines], ", ")
            warn_msg  = "Ignoring BINARYPROVIDER_COMPRESSION_ENGINE as its "
            warn_msg *= "value of `$(engine)` doesn't match any known valid "
            warn_msg *= "engines. Try one of `$(all_ngs)`."
            @warn(warn_msg)
        else
            # If BINARYPROVIDER_COMPRESSION_ENGINE matches one of our download
            # engines, then restrict ourselves to looking only at that engine
            compression_engines = comp_ngs
        end
    end

    download_found = false
    compression_found = false

    if verbose
        @info("Probing for download engine...")
    end

    # Search for a download engine
    for (test, dl_func) in download_engines
        if probe_cmd(`$test`; verbose=verbose)
            # Set our download command generator
            gen_download_cmd = (url, out_path, hdrs...) -> begin
                isdefined(Base, :download_url) && (url = Base.download_url(url))
                dl_func(url, out_path, hdrs...)
            end
            download_found = true

            if verbose
                @info("Found download engine $(test.exec[1])")
            end
            break
        end
    end

    if verbose
        @info("Probing for compression engine...")
    end

    # Search for a compression engine
    for (test, unpack, package, list, parse, symlink) in compression_engines
        if probe_cmd(`$test`; verbose=verbose)
            # Set our compression command generators
            gen_unpack_cmd = unpack
            gen_package_cmd = package
            gen_list_tarball_cmd = list
            parse_tarball_listing = parse
            parse_symlinks = () -> symlink

            if verbose
                @info("Found compression engine $(test.exec[1])")
            end

            compression_found = true
            break
        end
    end

    # Build informative error messages in case things go sideways
    errmsg = ""
    if !download_found
        errmsg *= "No download engines found. We looked for: "
        errmsg *= join([d[1].exec[1] for d in download_engines], ", ")
        errmsg *= ". Install one and ensure it  is available on the path.\n"
    end

    if !compression_found
        errmsg *= "No compression engines found. We looked for: "
        errmsg *= join([c[1].exec[1] for c in compression_engines], ", ")
        errmsg *= ". Install one and ensure it is available on the path.\n"
    end

    # Error out if we couldn't find something
    if !download_found || !compression_found
        error(errmsg)
    end
    already_probed = true
end

"""
    parse_7z_list(output::AbstractString)

Given the output of `7z l`, parse out the listed filenames.  This funciton used
by  `list_tarball_files`.
"""
function parse_7z_list(output::AbstractString)
    lines = [chomp(l) for l in split(output, "\n")]

    # If we didn't get anything, complain immediately
    if isempty(lines)
        return []
    end

    # Remove extraneous "\r" for windows platforms
    for idx in 1:length(lines)
        if endswith(lines[idx], '\r')
            lines[idx] = lines[idx][1:end-1]
        end
    end

    # Find index of " Name".  Have to `collect()` as `findfirst()` doesn't work with
    # generators: https://github.com/JuliaLang/julia/issues/16884
    header_row = findfirst(collect(occursin(" Name", l) && occursin(" Attr", l) for l in lines))
    name_idx = findfirst("Name", lines[header_row])[1]
    attr_idx = findfirst("Attr", lines[header_row])[1] - 1

    # Filter out only the names of files, ignoring directories
    lines = [l[name_idx:end] for l in lines if length(l) > name_idx && l[attr_idx] != 'D']
    if isempty(lines)
        return []
    end

    # Extract within the bounding lines of ------------
    bounds = [i for i in 1:length(lines) if all([c for c in lines[i]] .== Ref('-'))]
    lines = lines[bounds[1]+1:bounds[2]-1]

    # Eliminate `./` prefix, if it exists
    for idx in 1:length(lines)
        if startswith(lines[idx], "./") || startswith(lines[idx], ".\\")
            lines[idx] = lines[idx][3:end]
        end
    end

    return lines
end

"""
    parse_7z_list(output::AbstractString)

Given the output of `tar -t`, parse out the listed filenames.  This function
used by `list_tarball_files`.
"""
function parse_tar_list(output::AbstractString)
    lines = [chomp(l) for l in split(output, "\n")]
    for idx in 1:length(lines)
        while endswith(lines[idx], '\r')
            lines[idx] = lines[idx][1:end-1]
        end
    end

    # Drop empty lines and and directories
    lines = [l for l in lines if !isempty(l) && !endswith(l, '/')]

    # Eliminate `./` prefix, if it exists
    for idx in 1:length(lines)
        if startswith(lines[idx], "./") || startswith(lines[idx], ".\\")
            lines[idx] = lines[idx][3:end]
        end
    end

    # make sure paths are always returned in the system's default way
    return Sys.iswindows() ? replace.(lines, ['/' => '\\']) : lines
end

is_secure_url(url::AbstractString) =
    occursin(r"^(https://|\w+://(127\.0\.0\.1|localhost)(:\d+)?($|/))"i, url)

function get_server_dir(url::AbstractString, server=pkg_server())
    server === nothing && return
    url == server || startswith(url, "$server/") || return
    m = match(r"^\w+://([^\\/]+)(?:$|/)", server)
    if m === nothing
        @warn "malformed Pkg server value" server
        return
    end
    joinpath(depots1(), "servers", m.captures[1])
end

const AUTH_ERROR_HANDLERS = []

function handle_auth_error(url, err; verbose::Bool = false)
    handled, should_retry = false, false
    for (scheme, handler) in AUTH_ERROR_HANDLERS
        occursin(scheme, url) || continue
        handled, should_retry = handler(url, pkg_server(), err)
        handled && break
    end
    handled && should_retry && return get_auth_header(url; verbose = verbose)
    return nothing
end

"""
    register_auth_error_handler(urlscheme::Union{AbstractString, Regex}, f)

Registers `f` as the topmost handler for failures in package server authentication.

A handler is only invoked if `occursin(urlscheme, url)` is true (where `url` is the URL Pkg
is currently trying to download.)

`f` must be a function that takes three input arguments `(url, pkgserver, err)`, where `url` is the
URL currently being downloaded, `pkgserver = Pkg.pkg_server()` the current package server, and
`err` is one of `no-auth-file`, `insecure-connection`, `malformed-file`, `no-access-token`,
`no-refresh-key` or `insecure-refresh-url`.

The handler `f` needs to return a tuple of `Bool`s `(handled, should_retry)`. If `handled` is `false`,
the next handler in the stack will be called, otherwise handling terminates; `get_auth_header` is called again if `should_retry`
is `true`.

`register_auth_error_handler` returns a zero-arg function that can be called to deregister the handler.
"""
function register_auth_error_handler(urlscheme::Union{AbstractString, Regex}, f)
    unique!(pushfirst!(AUTH_ERROR_HANDLERS, urlscheme => f))
    return () -> deregister_auth_error_handler(urlscheme, f)
end

"""
    deregister_auth_error_handler(urlscheme::Union{AbstractString, Regex}, f)

Removes `f` from the stack of authentication error handlers.
"""
function deregister_auth_error_handler(urlscheme::Union{AbstractString, Regex}, f)
    filter!(handler -> handler !== (urlscheme => f), AUTH_ERROR_HANDLERS)
    return nothing
end

function get_auth_header(url::AbstractString; verbose::Bool = false)
    server_dir = get_server_dir(url)
    server_dir === nothing && return
    auth_file = joinpath(server_dir, "auth.toml")
    isfile(auth_file) || return handle_auth_error(url, "no-auth-file"; verbose=verbose)
    # TODO: check for insecure auth file permissions
    if !is_secure_url(url)
        @warn "refusing to send auth info over insecure connection" url=url
        return handle_auth_error(url, "insecure-connection"; verbose=verbose)
    end
    # parse the auth file
    auth_info = try
        TOML.parsefile(auth_file)
    catch err
        @error "malformed auth file" file=auth_file err=err
        return handle_auth_error(url, "malformed-file"; verbose=verbose)
    end
    # check for an auth token
    if !haskey(auth_info, "access_token")
        @warn "auth file without access_token field" file=auth_file
        return handle_auth_error(url, "no-access-token"; verbose=verbose)
    end
    auth_header = "Authorization: Bearer $(auth_info["access_token"])"
    # handle token expiration and refresh
    expires_at = Inf
    if haskey(auth_info, "expires_at")
        expires_at = min(expires_at, auth_info["expires_at"]::Integer)
    end
    if haskey(auth_info, "expires_in")
        expires_at = min(expires_at, mtime(auth_file) + auth_info["expires_in"]::Integer)
    end
    # if token is good until ten minutes from now, use it
    time_now = time()
    if expires_at ≥ time_now + 10*60 # ten minutes
        return auth_header
    end
    if !haskey(auth_info, "refresh_url") || !haskey(auth_info, "refresh_token")
        if expires_at ≤ time_now
            @warn "expired auth without refresh keys" file=auth_file
        end
        # try it anyway since we can't refresh
        return something(handle_auth_error(url, "no-refresh-key"; verbose=verbose), auth_header)
    end
    refresh_url = auth_info["refresh_url"]
    if !is_secure_url(refresh_url)
        @warn "ignoring insecure auth refresh URL" url=refresh_url
        return something(handle_auth_error(url, "insecure-refresh-url"; verbose=verbose), auth_header)
    end
    verbose && @info "Refreshing expired auth token..." file=auth_file
    tmp = tempname()
    refresh_auth = "Authorization: Bearer $(auth_info["refresh_token"])"
    try download(refresh_url, tmp, auth_header=refresh_auth, verbose=verbose)
    catch err
        @warn "token refresh failure" file=auth_file url=refresh_url err=err
        rm(tmp, force=true)
        return handle_auth_error(url, "token-refresh-failed"; verbose=verbose)
    end
    auth_info = try TOML.parsefile(tmp)
    catch err
        @warn "discarding malformed auth file" url=refresh_url err=err
        rm(tmp, force=true)
        return something(handle_auth_error(url, "malformed-file"; verbose=verbose), auth_header)
    end
    if !haskey(auth_info, "access_token")
        if haskey(auth_info, "refresh_token")
            auth_info["refresh_token"] = "*"^64
        end
        @warn "discarding auth file without access token" auth=auth_info
        rm(tmp, force=true)
        return something(handle_auth_error(url, "no-access-token"; verbose=verbose), auth_header)
    end
    if haskey(auth_info, "expires_in")
        expires_in = auth_info["expires_in"]
        if expires_in isa Number
            expires_at = floor(Int64, time_now + expires_in)
            # overwrite expires_at (avoids clock skew issues)
            auth_info["expires_at"] = expires_at
        end
    end
    let auth_info = auth_info
        open(tmp, write=true) do io
            TOML.print(io, auth_info, sorted=true)
        end
    end
    mv(tmp, auth_file, force=true)
    return "Authorization: Bearer $(auth_info["access_token"])"
end

function hash_data(strs::AbstractString...)
    ctx = SHA.SHA224_CTX()
    for str in strs
        data = Vector{UInt8}(str)
        len = length(data)
        while true
            push!(data, len % UInt8)
            len == 0 && break
            len >>= 8
        end
        SHA.update!(ctx, data)
    end
    return bytes2hex(@view SHA.digest!(ctx)[1:20])
end

function load_telemetry_file(file::AbstractString)
    if !ispath(file)
        info, changed = TOML.DictType(), true
    else
        info, changed = try
            TOML.parsefile(file), false
        catch err
            @warn "replacing malformed telemetry file" file=file err=err
            TOML.DictType(), true
        end
    end
    # bail early if fully opted out
    get(info, "telemetry", true) == false && return info
    # some validity checking helpers
    is_valid_uuid(x) = false
    is_valid_salt(x) = false
    is_valid_vars(x) = false
    is_valid_uuid(x::Bool) = !x # false is valid, true is not
    is_valid_salt(x::Bool) = !x # false is valid, true is not
    is_valid_vars(x::Bool) = true
    is_valid_uuid(x::AbstractString) = occursin(Pkg.REPLMode.uuid_re, x)
    is_valid_salt(x::AbstractString) = occursin(r"^[0-9a-zA-Z]+$", x)
    is_valid_vars(x::AbstractVector) = all(s isa AbstractString for s in x)
    # generate or fix system-specific info
    if !haskey(info, "client_uuid") || !is_valid_uuid(info["client_uuid"])
        info["client_uuid"] = string(uuid4())
        changed = true
    end
    if !haskey(info, "secret_salt") || !is_valid_salt(info["secret_salt"])
        info["secret_salt"] = randstring(36)
        changed = true
    end
    if haskey(info, "ci_variables") && !is_valid_vars(info["ci_variables"])
        delete!(info, "ci_variables")
        changed = true
    end
    changed || return info
    # write telemetry file atomically (if on same file system)
    mkpath(dirname(file))
    tmp = tempname()
    let info = info
        open(tmp, write=true) do io
            TOML.print(io, info, sorted=true)
        end
    end
    mv(tmp, file, force=true)
    # reparse file in case a different process wrote it first
    return load_telemetry_file(file)
end

# based on information in this post:
# https://github.community/t5/GitHub-Actions/Have-the-CI-environment-variable-set-by-default/m-p/32358/highlight/true#M1097
const CI_VARIABLES = [
    "APPVEYOR",
    "CI",
    "CIRCLECI",
    "CONTINUOUS_INTEGRATION",
    "GITHUB_ACTIONS",
    "GITLAB_CI",
    "JULIA_CI",
    "TF_BUILD",
    "TRAVIS",
]

function get_telemetry_headers(url::AbstractString)
    headers = String[]
    server_dir = get_server_dir(url)
    server_dir === nothing && return headers
    push!(headers, "Julia-Pkg-Protocol: 1.0")
    info = load_telemetry_file(joinpath(server_dir, "telemetry.toml"))
    get(info, "telemetry", true) == false && return headers
    # general system information
    push!(headers, "Julia-Version: $VERSION")
    system = Pkg.BinaryPlatforms.triplet(Pkg.BinaryPlatforms.platform_key_abi())
    push!(headers, "Julia-System: $system")
    # install-specific information
    if info["client_uuid"] != false
        push!(headers, "Julia-Client-UUID: $(info["client_uuid"])")
        if info["secret_salt"] != false
            salt_hash = hash_data("salt", info["client_uuid"], info["secret_salt"])
            project_hash = hash_data("project", Base.active_project(), info["secret_salt"])
            push!(headers, "Julia-Salt-Hash: $salt_hash")
            push!(headers, "Julia-Project-Hash: $project_hash")
        end
    end
    # CI indicator variables
    ci_variables = get(info, "ci_variables", CI_VARIABLES)
    ci_variables == true && (ci_variables = CI_VARIABLES)
    if ci_variables != false
        ci_info = String[]
        for var in CI_VARIABLES ∩ map(uppercase, ci_variables)
            val = get(ENV, var, nothing)
            state = val === nothing ? "n" :
                lowercase(val) in ("true", "t", "1", "yes", "y") ? "t" :
                lowercase(val) in ("false", "f", "0", "no", "n") ? "f" : "o"
            push!(ci_info, "$var=$state")
        end
        if !isempty(ci_info)
            push!(headers, "Julia-CI-Variables: "*join(ci_info, ';'))
        end
    end
    # interactive session?
    push!(headers, "Julia-Interactive: $(isinteractive())")
    return headers
end

"""
    download(
        url::AbstractString,
        dest::AbstractString;
        verbose::Bool = false,
        auth_header::Union{AbstractString, Nothing} = nothing,
    )

Download file located at `url`, store it at `dest`, continuing if `dest`
already exists and the server and download engine support it.
"""
function download(
    url::AbstractString,
    dest::AbstractString;
    verbose::Bool = false,
    auth_header::Union{AbstractString, Nothing} = nothing,
)
    headers = String[]
    if auth_header === nothing
        auth_header = get_auth_header(url, verbose=verbose)
    end
    if auth_header !== nothing
        push!(headers, auth_header)
    end
    for header in get_telemetry_headers(url)
        push!(headers, header)
    end
    download_cmd = gen_download_cmd(url, dest, headers...)
    if verbose
        # @info("Downloading $(url) to $(dest)...")
    end
    try
        run(download_cmd, (devnull, verbose ? stdout : devnull, verbose ? stderr : devnull))
    catch e
        if isa(e, InterruptException)
            rethrow()
        end
        error("Could not download $(url) to $(dest):\n$(e)")
    end
end

"""
    download_verify(
        url::AbstractString,
        hash::Union{AbstractString, Nothing},
        dest::AbstractString;
        verbose::Bool = false,
        force::Bool = false,
        quiet_download::Bool = false,
    )

Download file located at `url`, verify it matches the given `hash`, and throw
an error if anything goes wrong.  If `dest` already exists, just verify it. If
`force` is set to `true`, overwrite the given file if it exists but does not
match the given `hash`.

This method returns `true` if the file was downloaded successfully, `false`
if an existing file was removed due to the use of `force`, and throws an error
if `force` is not set and the already-existent file fails verification, or if
`force` is set, verification fails, and then verification fails again after
redownloading the file.

If `quiet_download` is set to `false`, this method will print to
stdout when downloading a new file.  If it is set to `true` (default, and `verbose` is
set to `false`) the downloading process will be completely silent.  If
`verbose` is set to `true`, messages about integrity verification will be
printed in addition to messages regarding downloading.
"""
function download_verify(
    url::AbstractString,
    hash::Union{AbstractString, Nothing},
    dest::AbstractString;
    verbose::Bool = false,
    force::Bool = false,
    quiet_download::Bool = false,
)
    # Whether the file existed in the first place
    file_existed = false

    if isfile(dest)
        file_existed = true
        if verbose
            @info("Destination file $(dest) already exists, verifying...")
        end

        # verify download, if it passes, return happy.  If it fails, (and
        # `force` is `true`, re-download!)
        if hash !== nothing && verify(dest, hash; verbose=verbose)
            return true
        elseif !force
            error("Verification failed, not overwriting $(dest)")
        end
    end

    # Make sure the containing folder exists
    mkpath(dirname(dest))

    # Download the file, optionally continuing
    download(url, dest; verbose=verbose || !quiet_download)
    if hash !== nothing && !verify(dest, hash; verbose=verbose)
        # If the file already existed, it's possible the initially downloaded chunk
        # was bad.  If verification fails after downloading, auto-delete the file
        # and start over from scratch.
        if file_existed
            if verbose
                @info("Continued download didn't work, restarting from scratch")
            end
            Base.rm(dest; force=true)

            # Download and verify from scratch
            download(url, dest; verbose=verbose || !quiet_download)
            if hash !== nothing && !verify(dest, hash; verbose=verbose)
                error("Verification failed")
            end
        else
            # If it didn't verify properly and we didn't resume, something is
            # very wrong and we must complain mightily.
            error("Verification failed")
        end
    end

    # If the file previously existed, this means we removed it (due to `force`)
    # and redownloaded, so return `false`.  If it didn't exist, then this means
    # that we successfully downloaded it, so return `true`.
    return !file_existed
end

function get_tarball_contents(path::AbstractString; verbose_tar::Bool = false)
    if !isfile(path)
        error("Tarball path $(path) does not exist")
    end

    # Run the listing command, then parse the output
    cmd = gen_list_tarball_cmd(path; verbose=verbose_tar)
    output = try
        out_pipe = Pipe()
        P = run(pipeline(cmd; stdout=out_pipe); wait=false)
        close(out_pipe.in)
        output = @async String(read(out_pipe))
		wait(P)
		fetch(output)
    catch
        error("Could not list contents of tarball $(path)")
    end

    return output
end

"""
    list_tarball_files(path::AbstractString; verbose::Bool = false)

Given a `.tar.gz` filepath, list the compressed contents.
"""
function list_tarball_files(tarball_path::AbstractString)
    return parse_tarball_listing(get_tarball_contents(tarball_path))
end

"""
    list_tarball_symlinks(path::AbstractString; verbose::Bool = false)

Given a `.tar.gz` filepath, return a dictionary of symlinks in the archive
"""
function list_tarball_symlinks(tarball_path::AbstractString)
    output = get_tarball_contents(tarball_path; verbose_tar = true)
    mm = [m.captures for m in eachmatch(parse_symlinks(), output)]
    symlinks = [m[1] => joinpath(dirname(m[1]), m[2]) for m in mm]
    return symlinks
end

"""
    unpack(tarball_path::AbstractString, dest::AbstractString;
           verbose::Bool = false)

Unpack tarball located at file `tarball_path` into directory `dest`.
"""
function unpack(tarball_path::AbstractString, dest::AbstractString;
                verbose::Bool = false)
    # unpack into dest
    mkpath(dest)

    # The user can force usage of our dereferencing workarounds for filesystems
    # that don't support symlinks, but it is also autodetected.
    copyderef = (get(ENV, "BINARYPROVIDER_COPYDEREF", "") == "true") || !probe_symlink_creation(dest)

    # If we should "copyderef" what we do is to unpack everything except symlinks
    # then copy the sources of the symlinks to the destination of the symlink instead.
    # This is to work around filesystems that are mounted (such as SMBFS filesystems)
    # that do not support symlinks.
    excludelist = nothing

    if copyderef
        symlinks = list_tarball_symlinks(tarball_path)
        if length(symlinks) > 0
            (excludelist, io) = mktemp()
            write(io, join([s[1] for s in symlinks], "\n"))
            close(io)
        end
    end

    cmd = gen_unpack_cmd(tarball_path, dest, excludelist)
    try
        run(cmd)
    catch e
        if isa(e, InterruptException)
            rethrow()
        end
        error("Could not unpack $(tarball_path) into $(dest)")
    end

    if copyderef && length(symlinks) > 0
        @info("Replacing symlinks in tarball by their source files ...\n" * join(string.(symlinks),"\n"))
        for s in symlinks
            sourcefile = normpath(joinpath(dest, s[2]))
            destfile   = normpath(joinpath(dest, s[1]))

            if isfile(sourcefile)
                cp(sourcefile, destfile, force = true)
            else
                @warn("Symlink source '$sourcefile' does not exist!")
            end
        end
        rm(excludelist; force = true)
    end
end

"""
    package(src_dir::AbstractString, tarball_path::AbstractString)

Compress `src_dir` into a tarball located at `tarball_path`.
"""
function package(src_dir::AbstractString, tarball_path::AbstractString)
    # For now, use environment variables to set the gzip compression factor to
    # level 9, eventually there will be new enough versions of tar everywhere
    # to use -I 'gzip -9', or even to switch over to .xz files.
    withenv("GZIP" => "-9") do
        cmd = gen_package_cmd(src_dir, tarball_path)
        try
            run(cmd, (devnull, devnull, devnull))
        catch e
            if isa(e, InterruptException)
                rethrow()
            end
            error("Could not package $(src_dir) into $(tarball_path)")
        end
    end
end

"""
    download_verify_unpack(
        url::AbstractString,
        hash::Union{AbstractString, Nothing},
        dest::AbstractString;
        tarball_path = nothing,
        ignore_existence::Bool = false,
        force::Bool = false,
        verbose::Bool = false,
        quiet_download::Bool = false,
    )

Helper method to download tarball located at `url`, verify it matches the
given `hash`, then unpack it into folder `dest`.  In general, the method
`install()` should be used to download and install tarballs into a `Prefix`;
this method should only be used if the extra functionality of `install()` is
undesired.

If `tarball_path` is specified, the given `url` will be downloaded to
`tarball_path`, and it will not be removed after downloading and verification
is complete.  If it is not specified, the tarball will be downloaded to a
temporary location, and removed after verification is complete.

If `force` is specified, a verification failure will cause `tarball_path` to be
deleted (if it exists), the `dest` folder to be removed (if it exists) and the
tarball to be redownloaded and reverified.  If the verification check is failed
a second time, an exception is raised.  If `force` is not specified, a
verification failure will result in an immediate raised exception.

If `ignore_existence` is set, the tarball is unpacked even if the destination
directory already exists.

Returns `true` if a tarball was actually unpacked, `false` if nothing was
changed in the destination prefix.
"""
function download_verify_unpack(
    url::AbstractString,
    hash::Union{AbstractString, Nothing},
    dest::AbstractString;
    tarball_path = nothing,
    ignore_existence::Bool = false,
    force::Bool = false,
    verbose::Bool = false,
    quiet_download::Bool = false,
)
    # First, determine whether we should keep this tarball around
    remove_tarball = false
    if tarball_path === nothing
        remove_tarball = true

        function url_ext(url)
            url = basename(url)

            # Chop off urlparams
            qidx = findfirst(isequal('?'), url)
            if qidx !== nothing
                url = url[1:qidx]
            end

            # Try to detect extension
            dot_idx = findlast(isequal('.'), url)
            if dot_idx === nothing
                return nothing
            end

            return url[dot_idx+1:end]
        end

        # If extension of url contains a recognized extension, use it, otherwise use ".gz"
        ext = url_ext(url)
        if !(ext in ["tar", "gz", "tgz", "bz2", "xz"])
            ext = "gz"
        end

        # Work around windows limitations regarding tempname()
        tarball_path = "$(tempname())-download.$(ext)"
        tries = 0
        while isfile(tarball_path) && tries < 100
            tarball_path = "$(tempname())-download.$(ext)"
            tries += 1
        end
        if tries >= 100
            error("Unable to generate unused tempname! Clean up your temporary folder $(dirname(tempname())) and try again.")
        end
    end

    # Download the tarball; if it already existed and we needed to remove it
    # then we should remove the unpacked path as well
    should_delete = !download_verify(url, hash, tarball_path;
                                     force=force, verbose=verbose, quiet_download=quiet_download)
    if should_delete
        if verbose
            @info("Removing dest directory $(dest) as source tarball changed")
        end
        Base.rm(dest; recursive=true, force=true)
    end

    # If the destination path already exists, don't bother to unpack
    if !ignore_existence && isdir(dest)
        if verbose
            @info("Destination directory $(dest) already exists, returning")
        end

        # Signify that we didn't do any unpacking
        return false
    end

    try
        if verbose
            @info("Unpacking $(tarball_path) into $(dest)...")
        end
        unpack(tarball_path, dest; verbose=verbose)
    finally
        if remove_tarball
            Base.rm(tarball_path)
        end
    end

    # Signify that we did some unpacking!
    return true
end

"""
    verify(path::AbstractString, hash::AbstractString;
           verbose::Bool = false, report_cache_status::Bool = false)

Given a file `path` and a `hash`, calculate the SHA256 of the file and compare
it to `hash`.  This method caches verification results in a `"\$(path).sha256"`
file to accelerate reverification of files that have been previously verified.
If no `".sha256"` file exists, a full verification will be done and the file
will be created, with the calculated hash being stored within the `".sha256"`
file.  If a `".sha256"` file does exist, its contents are checked to ensure
that the hash contained within matches the given `hash` parameter, and its
modification time shows that the file located at `path` has not been modified
since the last verification.

If `report_cache_status` is set to `true`, then the return value will be a
`Symbol` giving a granular status report on the state of the hash cache, in
addition to the `true`/`false` signifying whether verification completed
successfully.
"""
function verify(path::AbstractString, hash::AbstractString; verbose::Bool = false,
                report_cache_status::Bool = false, hash_path::AbstractString="$(path).sha256")

    # Check hash string format
    if !occursin(r"^[0-9a-f]{64}$"i, hash)
        msg = "Hash value must be 64 hexadecimal characters (256 bits), "
        if !isascii(hash)
            msg *= "given hash value is non-ASCII"
        elseif occursin(r"^[0-9a-f]*$"i, hash)
            msg *= "given hash value has the wrong length ($(length(hash)))"
        else
            msg *= "given hash value contains non-hexadecimal characters"
        end
        msg *= ": $(repr(hash))"
        error(msg)
    end
    hash = lowercase(hash)

    # Check to see if the hash cache is consistent
    status = :hash_consistent

    # First, it must exist
    if isfile(hash_path)
        # Next, it must contain the same hash as what we're verifying against
        if read(hash_path, String) == hash
            # Next, it must be no older than the actual path
            if stat(hash_path).mtime >= stat(path).mtime
                # If all of that is true, then we're good!
                if verbose
                    @info("Hash cache is consistent, returning true")
                end
                status = :hash_cache_consistent

                # If we're reporting our status, then report it!
                if report_cache_status
                    return true, status
                else
                    return true
                end
            else
                if verbose
                    @info("File has been modified, hash cache invalidated")
                end
                status = :file_modified
            end
        else
            if verbose
                @info("Verification hash mismatch, hash cache invalidated")
            end
            status = :hash_cache_mismatch
        end
    else
        if verbose
            @info("No hash cache found")
        end
        status = :hash_cache_missing
    end

    calc_hash = open(path) do file
        bytes2hex(sha256(file))
    end
    @assert occursin(r"^[0-9a-f]{64}$", calc_hash)

    if verbose
        @info("Calculated hash $calc_hash for file $path")
    end

    if calc_hash != hash
        msg  = "Hash Mismatch!\n"
        msg *= "  Expected sha256:   $hash\n"
        msg *= "  Calculated sha256: $calc_hash"
        @error(msg)
        if report_cache_status
            return false, :hash_mismatch
        else
            return false
        end
    end

    # Try to save a hash cache if everything worked out fine
    try
        open(hash_path, "w") do file
            write(file, hash)
        end
    catch e
        if isa(e, InterruptException)
            rethrow(e)
        end

        if verbose
            @warn("Unable to create hash cache file $(hash_path)")
        end
    end

    if report_cache_status
        return true, status
    else
        return true
    end
end

end # module PlatformEngines


text:
function add_kepler_contribution(obj::FieldOrbit, arg0::PositionAngle, arg1::RealFieldElement, arg2::Vector{RealFieldElement})
    return jcall(obj, "addKeplerContribution", void, (PositionAngle, RealFieldElement, Vector{RealFieldElement}), arg0, arg1, arg2)
end

function get_a(obj::FieldOrbit)
    return jcall(obj, "getA", RealFieldElement, ())
end

function get_a_dot(obj::FieldOrbit)
    return jcall(obj, "getADot", RealFieldElement, ())
end

function get_e(obj::FieldOrbit)
    return jcall(obj, "getE", RealFieldElement, ())
end

function get_e_dot(obj::FieldOrbit)
    return jcall(obj, "getEDot", RealFieldElement, ())
end

function get_equinoctial_ex(obj::FieldOrbit)
    return jcall(obj, "getEquinoctialEx", RealFieldElement, ())
end

function get_equinoctial_ex_dot(obj::FieldOrbit)
    return jcall(obj, "getEquinoctialExDot", RealFieldElement, ())
end

function get_equinoctial_ey(obj::FieldOrbit)
    return jcall(obj, "getEquinoctialEy", RealFieldElement, ())
end

function get_equinoctial_ey_dot(obj::FieldOrbit)
    return jcall(obj, "getEquinoctialEyDot", RealFieldElement, ())
end

function get_hx(obj::FieldOrbit)
    return jcall(obj, "getHx", RealFieldElement, ())
end

function get_hx_dot(obj::FieldOrbit)
    return jcall(obj, "getHxDot", RealFieldElement, ())
end

function get_hy(obj::FieldOrbit)
    return jcall(obj, "getHy", RealFieldElement, ())
end

function get_hy_dot(obj::FieldOrbit)
    return jcall(obj, "getHyDot", RealFieldElement, ())
end

function get_i(obj::FieldOrbit)
    return jcall(obj, "getI", RealFieldElement, ())
end

function get_i_dot(obj::FieldOrbit)
    return jcall(obj, "getIDot", RealFieldElement, ())
end

function get_le(obj::FieldOrbit)
    return jcall(obj, "getLE", RealFieldElement, ())
end

function get_le_dot(obj::FieldOrbit)
    return jcall(obj, "getLEDot", RealFieldElement, ())
end

function get_lm(obj::FieldOrbit)
    return jcall(obj, "getLM", RealFieldElement, ())
end

function get_lm_dot(obj::FieldOrbit)
    return jcall(obj, "getLMDot", RealFieldElement, ())
end

function get_lv(obj::FieldOrbit)
    return jcall(obj, "getLv", RealFieldElement, ())
end

function get_lv_dot(obj::FieldOrbit)
    return jcall(obj, "getLvDot", RealFieldElement, ())
end

function get_type(obj::FieldOrbit)
    return jcall(obj, "getType", OrbitType, ())
end

function has_derivatives(obj::FieldOrbit)
    return jcall(obj, "hasDerivatives", jboolean, ())
end

function interpolate(obj::FieldTimeInterpolable, arg0::FieldAbsoluteDate, arg1::Stream)
    return jcall(obj, "interpolate", FieldTimeInterpolable, (FieldAbsoluteDate, Stream), arg0, arg1)
end

function shifted_by(obj::FieldOrbit, arg0::RealFieldElement)
    return jcall(obj, "shiftedBy", FieldOrbit, (RealFieldElement,), arg0)
end

function shifted_by(obj::FieldTimeShiftable, arg0::jdouble)
    return jcall(obj, "shiftedBy", FieldTimeInterpolable, (jdouble,), arg0)
end

function to_orbit(obj::FieldOrbit)
    return jcall(obj, "toOrbit", Orbit, ())
end

ext: jl


text:
#
# asian-option.jl
#
function run-asian(N = 100000, PutCall = 'C';)
# European Asian option.
# Uses geometric or arithmetic average.
# Euler and Milstein discretization for Black-Scholes.
# Option features.
  println("Setting option parameters");
  S0  = 100;      # Spot price
  K   = 100;      # Strike price
  r   = 0.05;     # Risk free rate
  q   = 0.0;      # Dividend yield
  v   = 0.2;      # Volatility
  tma = 0.25;     # Time to maturity

  Averaging = 'A';  # 'A'rithmetic or 'G'eometric
  OptType = (PutCall == 'C' ? "CALL" : "PUT");
  println("Option type is $OptType");
# Simulation settings.
  println("Setting simulation parameters");
  T = 100;         # Number of time steps
  dt = tma/T;      # Time increment

# Initialize the terminal stock price matrices
# for the Euler and Milstein discretization schemes.
 S = zeros(Float64,N,T);
  for n=1:N
    	S[n,1] = S0;
  end

# Simulate the stock price under the Euler and Milstein schemes.
# Take average of terminal stock price.
  println("Looping $N times.");
  A = zeros(Float64,N);
  for n=1:N
    	for t=2:T
      		dW = (randn(1)[1])*sqrt(dt);
     		 z0 = (r - q - 0.5*v*v)*S[n,t-1]*dt;
     		 z1 = v*S[n,t-1]*dW;
      		z2 = 0.5*v*v*S[n,t-1]*dW*dW;
      		S[n,t] = S[n,t-1] + z0 + z1 + z2;
   	 end
   	 if cmp(Averaging,'A') == 0
      		A[n] = mean(S[n,:]);
   	 elseif cmp(Averaging,'G') == 0
      		A[n] = exp(mean(log(S[n,:])));
   	 end
  end

# Define the payoff
  P = zeros(Float64,N);
  if cmp(PutCall,'C') == 0
    	for n = 1:N
      		P[n] = max(A[n] - K, 0);
  	  end
  elseif cmp(PutCall,'P') == 0
    	for n = 1:N
     		 P[n] = max(K - A[n], 0);
   	 end
  end
# Calculate the price of the Asian option
 AsianPrice = exp(-r*tma)*mean(P);
 @printf "Price: %10.4f\n" AsianPrice;
end

run-asian()

max_stars_repo_path: Module 3/Chapter01/asian-option.jl

max_issues_repo_path: Module 3/Chapter01/asian-option.jl

max_forks_repo_path: Module 3/Chapter01/asian-option.jl


text:
## Dubins car
# TODO test and inplace dynamics

function dubins_dynamics!(xdot,x,u)
    xdot[1] = u[1]*cos(x[3])
    xdot[2] = u[1]*sin(x[3])
    xdot[3] = u[2]
    xdot
end
n,m = 3,2

model = Model(dubins_dynamics!,n,m)

##########################
## Constrained Examples ##
##########################

# Unconstrained Parallel Park
# initial and goal states
x0 = [0.;0.;0.]
xf = [0.;1.;0.]

# costs
Q = (1e-2)*Diagonal(I,n)
Qf = 100.0*Diagonal(I,n)
R = (1e-2)*Diagonal(I,m)

# simulation
tf = 3.0
dt = 0.01

obj_uncon = LQRObjective(Q, R, Qf, tf, x0, xf)

dubinscar = [model, obj_uncon]

## Box Parallel Park
x0 = [0.0;0.0;0.]
xf = [0.0;1.0;0.]
tf =  3.
Qf = 100.0*Diagonal(I,n)
Q = (1e-3)*Diagonal(I,n)
R = (1e-2)*Diagonal(I,m)

obj = LQRObjective(Q, R, Qf, tf, x0, xf)

x_min = [-0.25; -0.001; -Inf]
x_max = [0.25; 1.001; Inf]

obj_con_box = TrajectoryOptimization.ConstrainedObjective(obj,x_min=x_min,x_max=x_max)

dubinscar_parallelpark = [model, obj_con_box]

## Obstacle Avoid (x3)
x0 = [0.0;0.0;0.]
xf = [10.0;10.0;0.]
tf =  3.0
Qf = 100.0*Diagonal(I,n)
Q = (1e-3)*Diagonal(I,n)
R = (1e-2)*Diagonal(I,m)

obj = LQRObjective(Q, R, Qf, tf, x0, xf)

u_min = [0.; -100.]
u_max = [5.75; 100.]
x_min = [-Inf; -Inf; -Inf]
x_max = [Inf; Inf; Inf]

n_circles_3obs = 3
r_3obs = 1.0
circles_3obs = ((2.5,2.5,r_3obs),(5.,5.,r_3obs),(7.5,7.5,r_3obs))

function cI_3obs(c,x,u)
    for i = 1:n_circles_3obs
        c[i] = TrajectoryOptimization.circle_constraint(x,circles_3obs[i][1],circles_3obs[i][2],circles_3obs[i][3])
    end
    c
end

obj_con_obstacles = TrajectoryOptimization.ConstrainedObjective(obj,x_min=x_min,x_max=x_max,cI=cI_3obs)
obj_con_obstacles_control = TrajectoryOptimization.ConstrainedObjective(obj,u_min=u_min,u_max=u_max,x_min=x_min,x_max=x_max,cI=cI_3obs)

dubinscar_obstacles = [model, obj_con_obstacles,circles_3obs]
dubinscar_obstacles_control_limits = [model, obj_con_obstacles_control,circles_3obs]

## Escape
x0 = [2.5;2.5;0.]
xf = [7.5;2.5;0.]
tf =  3.0
Qf = 100.0*Diagonal(I,n)
Q = (1e-3)*Diagonal(I,n)
R = (1e-2)*Diagonal(I,m)
u_min = [-5.; -5.]
u_max = [5.; 5.]

obj = LQRObjective(Q, R, Qf, tf, x0, xf)

r = 0.5
s1 = 30; s2 = 50; s3 = 15
circles_escape = []
for i in range(0,stop=5,length=s1)
    push!(circles_escape,(0.,i,r))
end
for i in range(0,stop=5,length=s1)
    push!(circles_escape,(5.,i,r))
end
for i in range(0,stop=5,length=s1)
    push!(circles_escape,(10.,i,r))
end
for i in range(0,stop=10,length=s2)
    push!(circles_escape,(i,0.,r))
end
for i in range(0,stop=3,length=s3)
    push!(circles_escape,(i,5.,r))
end
for i in range(5,stop=8,length=s3)
    push!(circles_escape,(i,5.,r))
end

n_circles_escape = 3*s1 + s2 + 2*s3

function cI_escape(c,x,u)
    for i = 1:n_circles_escape
        c[i] = TrajectoryOptimization.circle_constraint(x,circles_escape[i][1],circles_escape[i][2],circles_escape[i][3])
    end
    c
end

obj_escape = TrajectoryOptimization.ConstrainedObjective(obj,cI=cI_escape,u_min=u_min,u_max=u_max)

dubinscar_escape = [model,obj_escape,circles_escape]


text:
@testset "environment" begin
    using Pkg

    try
        Pkg.activate(atomjldir)

        @static VERSION ≥ v"1.4" && begin

        # test current project info
        prj_info = Atom.project_info()
        @test prj_info.name == "Atom"
        @test prj_info.path == joinpath(atomjldir, "Project.toml")

        # test all active project listing: at least environment for this package is active
        prjs = Atom.allprojects()
        @test Atom.project_info() in prjs.projects
        @test prjs.active == "Atom"

        end  # if VERSION ≥ v"1.4"
    finally
        Pkg.activate()
    end
end


text:
using Clang

const INCLUDE = "Csdp-6.1.1/include"

idx = Clang.cindex.parse_header("$INCLUDE/declarations.h";
                                diagnostics=true,
                                includes=[INCLUDE],
                                args=["-DNOSHORTS"])
cursor = idx.data[1]


text:
import Random
import Test

Test.@testset "examples" begin
    Test.@testset "linear regression" begin
        Random.seed!(123)
        include(joinpath(examples_directory, "linear_regression.jl"))
        include(joinpath(test_directory, "examples", "linear_regression.jl"))

    end

    Test.@testset "logistic regression" begin
        Random.seed!(123)
        include(joinpath(examples_directory, "logistic_regression.jl"))
        include(joinpath(test_directory, "examples", "logistic_regression.jl"))
    end

    Test.@testset "multinomial logistic regression" begin
        Random.seed!(123)
        @info("This next test may go five to fifteen minutes without producing any output")
        include(joinpath(examples_directory, "multinomial_logistic_regression.jl"))
        include(joinpath(test_directory, "examples", "multinomial_logistic_regression.jl"))
    end
end


text:
#Benchmarking of RBC variants
using DifferentiableStateSpaceModels, BenchmarkTools
using DifferentiableStateSpaceModels.Examples
const RBC = BenchmarkGroup()
const RBC["rbc_observables"] = BenchmarkGroup()

# Prepare the model for the RBC observables
const m_rbc_observables = @include_example_module(Examples.rbc_observables)
# Basic Steady State
const p_f_rbc_observables = (ρ = 0.2, δ = 0.02, σ = 0.01, Ω_1 = 0.01)
const p_d_rbc_observables = (α = 0.5, β = 0.95)
const c_rbc_observables_1 = SolverCache(m_rbc_observables, Val(1), p_d_rbc_observables)
const c_rbc_observables_2 = SolverCache(m_rbc_observables, Val(2), p_d_rbc_observables)

# The derivatives caches need to be already filled in case they are done out of order
const c_rbc_observables_1_p = SolverCache(m_rbc_observables, Val(1), p_d_rbc_observables)
const c_rbc_observables_2_p = SolverCache(m_rbc_observables, Val(2), p_d_rbc_observables)
generate_perturbation(m_rbc_observables, p_d_rbc_observables, p_f_rbc_observables, Val(1);
                      cache = c_rbc_observables_1_p)
generate_perturbation(m_rbc_observables, p_d_rbc_observables, p_f_rbc_observables, Val(2);
                      cache = c_rbc_observables_2_p)

# Can't hurt to  call the derivatives to precompile as well, eventhough they will be written over
generate_perturbation_derivatives!(m_rbc_observables, p_d_rbc_observables,
                                   p_f_rbc_observables, c_rbc_observables_1_p)
generate_perturbation_derivatives!(m_rbc_observables, p_d_rbc_observables,
                                   p_f_rbc_observables, c_rbc_observables_2_p)

# rbc_observables
const RBC["rbc_observables"]["first_order"] = @benchmarkable generate_perturbation($m_rbc_observables,
                                                                                $p_d_rbc_observables,
                                                                                $p_f_rbc_observables,
                                                                                Val(1);
                                                                                cache = $c_rbc_observables_1)
const RBC["rbc_observables"]["first_order_p"] = @benchmarkable generate_perturbation_derivatives!($m_rbc_observables,
                                                                                $p_d_rbc_observables,
                                                                                $p_f_rbc_observables,
                                                                                $c_rbc_observables_1_p)
const RBC["rbc_observables"]["second_order"] = @benchmarkable generate_perturbation($m_rbc_observables,
                                                                                $p_d_rbc_observables,
                                                                                $p_f_rbc_observables,
                                                                                Val(2);
                                                                                cache = $c_rbc_observables_2)
const RBC["rbc_observables"]["second_order_p"] = @benchmarkable generate_perturbation_derivatives!($m_rbc_observables,
                                                                                $p_d_rbc_observables,
                                                                                $p_f_rbc_observables,
                                                                                $c_rbc_observables_2_p)
const RBC["rbc_observables"]["second_order_no_cache"] = @benchmarkable generate_perturbation($m_rbc_observables,
                                                                                $p_d_rbc_observables,
                                                                                $p_f_rbc_observables,
                                                                                Val(2)) # just to see how long the cache takes
const RBC["rbc_observables"]["first_order_solver_cache"] = @benchmarkable SolverCache($m_rbc_observables,
                                                                                Val(1),
                                                                                $p_d_rbc_observables)
const RBC["rbc_observables"]["second_order_solver_cache"] = @benchmarkable SolverCache($m_rbc_observables,
                                                                                Val(2),
                                                                                $p_d_rbc_observables)

# return for the suite
RBC

# Uncomment to run on its own, otherwise nests into full benchmarking
# results = run(RBC; verbose = true)
# results["rbc_observables"] # etc.


text:
using Primes

sum = 2
currentprime = 2
for i in 2:100000
    currentprime = nextprime(currentprime + 1)
    sum += currentprime
end
println("The sum of the first 100,000 primes is $sum")

curprime = 1
arr = zeros(Int, 20)
for i in 1:20
    curprime = nextprime(curprime + 1)
    arr[i] = curprime
end
println("The first 20 primes are ", arr)

println("the primes between 100 and 150 are ", primes(100,150))
println("The number of primes between 7,700 and 8,000 is ", length(primes(7700, 8000)))
println("The 10,000th prime is ", prime(10000))

max_stars_repo_name: ethansaxenian/RosettaDecode

max_issues_repo_name: ethansaxenian/RosettaDecode

max_forks_repo_name: ethansaxenian/RosettaDecode