text:
"""
Main module for `Documenter.jl` -- a documentation generation package for Julia.

Two functions are exported from this module for public use:

- [`makedocs`](@ref). Generates documentation from docstrings and templated markdown files.
- [`deploydocs`](@ref). Deploys generated documentation from *Travis-CI* to *GitHub Pages*.

# Exports

$(EXPORTS)

"""
module Documenter

using Test: @testset, @test
using DocStringExtensions
import Base64: base64decode

"""
    abstract type Plugin end

Any plugin that needs to either solicit user input or store information in a
[`Documents.Document`](@ref) should create a subtype of `Plugin`. The
subtype, `T <: Documenter.Plugin`, must have an empty constructor `T()` that
initialized `T` with the appropriate default values.

To retrieve the values stored in `T`, the plugin can call [`Documents.getplugin`](@ref).
If `T` was passed to [`makedocs`](@ref), the passed type will be returned. Otherwise,
a new `T` object will be created.
"""
abstract type Plugin end

abstract type Writer end

# Submodules
# ----------

include("Utilities/Utilities.jl")
include("DocMeta.jl")
include("DocSystem.jl")
include("Anchors.jl")
include("Documents.jl")
include("Expanders.jl")
include("DocTests.jl")
include("Builder.jl")
include("CrossReferences.jl")
include("DocChecks.jl")
include("Writers/Writers.jl")
include("Deps.jl")

import .Utilities: Selectors
import .Writers.HTMLWriter: HTML, asset
import .Writers.HTMLWriter.RD: KaTeX, MathJax, MathJax2, MathJax3
import .Writers.LaTeXWriter: LaTeX

# User Interface.
# ---------------
export Deps, makedocs, deploydocs, hide, doctest, DocMeta, asset,
    KaTeX, MathJax, MathJax2, MathJax3

"""
    makedocs(
        root    = "<current-directory>",
        source  = "src",
        build   = "build",
        clean   = true,
        doctest = true,
        modules = Module[],
        repo    = "",
        highlightsig = true,
        sitename = "",
        expandfirst = [],
    )

Combines markdown files and inline docstrings into an interlinked document.
In most cases [`makedocs`](@ref) should be run from a `make.jl` file:

```julia
using Documenter
makedocs(
    # keywords...
)
```

which is then run from the command line with:

```sh
\$ julia make.jl
```

The folder structure that [`makedocs`](@ref) expects looks like:

    docs/
        build/
        src/
        make.jl

# Keywords

**`root`** is the directory from which `makedocs` should run. When run from a `make.jl` file
this keyword does not need to be set. It is, for the most part, needed when repeatedly
running `makedocs` from the Julia REPL like so:

    julia> makedocs(root = joinpath(dirname(pathof(MyModule)), "..", "docs"))

**`source`** is the directory, relative to `root`, where the markdown source files are read
from. By convention this folder is called `src`. Note that any non-markdown files stored
in `source` are copied over to the build directory when [`makedocs`](@ref) is run.

**`build`** is the directory, relative to `root`, into which generated files and folders are
written when [`makedocs`](@ref) is run. The name of the build directory is, by convention,
called `build`, though, like with `source`, users are free to change this to anything else
to better suit their project needs.

**`clean`** tells [`makedocs`](@ref) whether to remove all the content from the `build`
folder prior to generating new content from `source`. By default this is set to `true`.

**`doctest`** instructs [`makedocs`](@ref) on whether to try to test Julia code blocks
that are encountered in the generated document. By default this keyword is set to `true`.
Doctesting should only ever be disabled when initially setting up a newly developed package
where the developer is just trying to get their package and documentation structure correct.
After that, it's encouraged to always make sure that documentation examples are runnable and
produce the expected results. See the [Doctests](@ref) manual section for details about
running doctests.

Setting `doctest` to `:only` allows for doctesting without a full build. In this mode, most
build stages are skipped and the `strict` keyword is ignore (a doctesting error will always
make `makedocs` throw an error).

**`modules`** specifies a vector of modules that should be documented in `source`. If any
inline docstrings from those modules are seen to be missing from the generated content then
a warning will be printed during execution of [`makedocs`](@ref). By default no modules are
passed to `modules` and so no warnings will appear. This setting can be used as an indicator
of the "coverage" of the generated documentation.
For example Documenter's `make.jl` file contains:

```julia
makedocs(
    modules = [Documenter],
    # ...
)
```

and so any docstring from the module `Documenter` that is not spliced into the generated
documentation in `build` will raise a warning.

**`repo`** specifies a template for the "link to source" feature. If you are
using GitHub, this is automatically generated from the remote. If you are using
a different host, you can use this option to tell Documenter how URLs should be
generated. The following placeholders will be replaced with the respective
value of the generated link:

  - `{commit}` Git branch or tag name, or commit hash
  - `{path}` Path to the file in the repository
  - `{line}` Line (or range of lines) in the source file

BitBucket, GitLab and Azure DevOps are supported along with GitHub, for example:

```julia
makedocs(repo = \"https://gitlab.com/user/project/blob/{commit}{path}#{line}\") # GitLab
makedocs(repo = \"https://dev.azure.com/org/project/_git/repo?path={path}&version={commit}{line}&lineStartColumn=1&lineEndColumn=1\") # Azure DevOps
makedocs(repo = \"https://bitbucket.org/user/project/src/{commit}/{path}#lines-{line}\") # BitBucket
```

**`highlightsig`** enables or disables automatic syntax highlighting of leading, unlabeled
code blocks in docstrings (as Julia code). For example, if your docstring begins with an
indented code block containing the function signature, then that block would be highlighted
as if it were a labeled Julia code block. No other code blocks are affected. This feature
is enabled by default.

**`sitename`** is displayed in the title bar and/or the navigation menu when applicable.

**`expandfirst`** allows some of the pages to be _expanded_ (i.e. at-blocks evaluated etc.)
before the others. Documenter normally evaluates the files in the alphabetic order of their
file paths relative to `src`, but `expandfirst` allows some pages to be prioritized.

For example, if you have `foo.md` and `bar.md`, `bar.md` would normally be evaluated before
`foo.md`. But with `expandfirst = ["foo.md"]`, you can force `foo.md` to be evaluated first.

Evaluation order among the `expandfirst` pages is according to the order they appear in the
argument.

# Experimental keywords

In addition to standard arguments there is a set of non-finalized experimental keyword
arguments. The behaviour of these may change or they may be removed without deprecation
when a minor version changes (i.e. except in patch releases).

**`checkdocs`** instructs [`makedocs`](@ref) to check whether all names within the modules
defined in the `modules` keyword that have a docstring attached have the docstring also
listed in the manual (e.g. there's a `@docs` block with that docstring). Possible values
are `:all` (check all names; the default), `:exports` (check only exported names) and
`:none` (no checks are performed). If `strict` is also enabled then the build will fail if
any missing docstrings are encountered.

**`linkcheck`** -- if set to `true` [`makedocs`](@ref) uses `curl` to check the status codes
of external-pointing links, to make sure that they are up-to-date. The links and their
status codes are printed to the standard output. If `strict` is also enabled then the build
will fail if there are any broken (400+ status code) links. Default: `false`.

**`linkcheck_ignore`** allows certain URLs to be ignored in `linkcheck`. The values should
be a list of strings (which get matched exactly) or `Regex` objects. By default nothing is
ignored.

**`linkcheck_timeout`** configures how long `curl` waits (in seconds) for a link request to
return a response before giving up. The default is 10 seconds.

**`strict`** -- [`makedocs`](@ref) fails the build right before rendering if it encountered
any errors with the document in the previous build phases.

**`workdir`** determines the working directory where `@example` and `@repl` code blocks are
executed. It can be either a path or the special value `:build` (default).

If the `workdir` is set to a path, the working directory is reset to that path for each code
block being evaluated. Relative paths are taken to be relative to `root`, but using absolute
paths is recommended (e.g. `workdir = joinpath(@__DIR__, "..")` for executing in the package
root for the usual `docs/make.jl` setup).

With the default `:build` option, the working directory is set to a subdirectory of `build`,
determined from the source file path. E.g. for `src/foo.md` it is set to `build/`, for
`src/foo/bar.md` it is set to `build/foo` etc.

Note that `workdir` does not affect doctests.

## Output formats

**`format`** allows the output format to be specified. The default format is
[`Documenter.HTML`](@ref) which creates a set of HTML files, but Documenter also provides
PDF output via the [`Documenter.LaTeX`](@ref) writer.

Other formats can be enabled by using other addon-packages. For example, the
[DocumenterMarkdown](https://github.com/JuliaDocs/DocumenterMarkdown.jl) package provides
the original Markdown -> Markdown output. See the [Other Output Formats](@ref) for more
information.

# See Also

A guide detailing how to document a package using Documenter's [`makedocs`](@ref) is provided
in the [setup guide in the manual](@ref Package-Guide).
"""
function makedocs(components...; debug = false, format = HTML(), kwargs...)
    document = Documents.Document(components; format=format, kwargs...)
    # Before starting the build pipeline, we empty out the subtype cache used by
    # Selectors.dispatch. This is to make sure that we pick up any new selector stages that
    # may have been added to the selector pipelines between makedocs calls.
    empty!(Selectors.selector_subtypes)
    cd(document.user.root) do
        Selectors.dispatch(Builder.DocumentPipeline, document)
    end
    debug ? document : nothing
end

"""
$(SIGNATURES)

Allows a page to be hidden in the navigation menu. It will only show up if it happens to be
the current page. The hidden page will still be present in the linear page list that can be
accessed via the previous and next page links. The title of the hidden page can be overridden
using the `=>` operator as usual.

# Usage

```julia
makedocs(
    ...,
    pages = [
        ...,
        hide("page1.md"),
        hide("Title" => "page2.md")
    ]
)
```
"""
hide(page::Pair) = (false, page.first, page.second, [])
hide(page::AbstractString) = (false, nothing, page, [])

"""
$(SIGNATURES)

Allows a subsection of pages to be hidden from the navigation menu. `root` will be linked
to in the navigation menu, with the title determined as usual. `children` should be a list
of pages (note that it **can not** be hierarchical).

# Usage

```julia
makedocs(
    ...,
    pages = [
        ...,
        hide("Hidden section" => "hidden_index.md", [
            "hidden1.md",
            "Hidden 2" => "hidden2.md"
        ]),
        hide("hidden_index.md", [...])
    ]
)
```
"""
hide(root::Pair, children) = (true, root.first, root.second, map(hide, children))
hide(root::AbstractString, children) = (true, nothing, root, map(hide, children))

include("deployconfig.jl")

"""
    deploydocs(
        root   = "<current-directory>",
        target = "build",
        repo   = "<required>",
        branch = "gh-pages",
        deps   = nothing | <Function>,
        make   = nothing | <Function>,
        devbranch = "master",
        devurl = "dev",
        versions = ["stable" => "v^", "v#.#", devurl => devurl],
        push_preview    = false,
        repo_previews   = repo,
        branch_previews = branch,
    )

Converts markdown files generated by [`makedocs`](@ref) to HTML and pushes them to `repo`.
This function should be called from within a package's `docs/make.jl` file after the call to
[`makedocs`](@ref), like so

```julia
using Documenter, PACKAGE_NAME
makedocs(
    # options...
)
deploydocs(
    repo = "github.com/..."
)
```

When building the docs for a tag (i.e. a release) the documentation is deployed to
a directory with the tag name (i.e. `vX.Y.Z`) and to the `stable` directory.
Otherwise the docs are deployed to the directory determined by the `devurl` argument.

# Required keyword arguments

**`repo`** is the remote repository where generated HTML content should be pushed to. Do not
specify any protocol - "https://" or "git@" should not be present. This keyword *must*
be set and will throw an error when left undefined. For example this package uses the
following `repo` value:

```julia
repo = "github.com/JuliaDocs/Documenter.jl.git"
```

# Optional keyword arguments

**`deploy_config`** determines configuration for the deployment.
If this is not specified Documenter will try to autodetect from the
currently running environment. See the manual section about
[Deployment systems](@ref).

**`root`** has the same purpose as the `root` keyword for [`makedocs`](@ref).

**`target`** is the directory, relative to `root`, where generated content that should be
deployed to `gh-pages` is written to. It should generally be the same as
[`makedocs`](@ref)'s `build` and defaults to `"build"`.

**`branch`** is the branch where the generated documentation is pushed. If the branch does
not exist, a new orphaned branch is created automatically. It defaults to `"gh-pages"`.

**`deps`** is the function used to install any additional dependencies needed to build the
documentation. By default nothing is installed.

It can be used e.g. for a Markdown build. The following example installed the `pygments` and
`mkdocs` Python packages using the [`Deps.pip`](@ref) function:

```julia
deps = Deps.pip("pygments", "mkdocs")
```

**`make`** is the function used to specify an additional build phase. By default, nothing gets
executed.

**`devbranch`** is the branch that "tracks" the in-development version of the generated
documentation. By default this value is set to `"master"`.

**`devurl`** the folder that in-development version of the docs will be deployed.
Defaults to `"dev"`.

**`forcepush`** a boolean that specifies the behavior of the git-deployment.
The default (`forcepush = false`) is to push a new commit, but when
`forcepush = true` the changes will be combined with the previous commit and
force pushed, erasing the Git history on the deployment branch.

**`versions`** determines content and order of the resulting version selector in
the generated html. The following entries are valid in the `versions` vector:
 - `"v#"`: includes links to the latest documentation for each major release cycle
   (i.e. `v2.0`, `v1.1`).
 - `"v#.#"`: includes links to the latest documentation for each minor release cycle
   (i.e. `v2.0`, `v1.1`, `v1.0`, `v0.1`).
 - `"v#.#.#"`: includes links to all released versions.
 - `"v^"`: includes a link to the docs for the maximum version
   (i.e. a link `vX.Y` pointing to `vX.Y.Z` for highest `X`, `Y`, `Z`, respectively).
 - A pair, e.g. `"first" => "second"`, which will put `"first"` in the selector,
   and generate a url from which `"second"` can be accessed.
   The second argument can be `"v^"`, to point to the maximum version docs
   (as in e.g. `"stable" => "v^"`).

**`push_preview`** a boolean that specifies if preview documentation should be
deployed from pull requests or not. If your published documentation is hosted
at `"https://USER.github.io/PACKAGE.jl/stable`, by default the preview will be
hosted at `"https://USER.github.io/PACKAGE.jl/previews/PR##"`. This feature
works for pull requests with head branch in the same repository, i.e. not from
forks.

**`branch_previews`** is the branch to which pull request previews are deployed.
It defaults to the value of `branch`.

**`repo_previews`** is the remote repository to which pull request previews are
deployed. It defaults to the value of `repo`.

!!! note
    Pull requests made from forks will not have previews.
    Hosting previews requires access to the deploy key.
    Therefore, previews are available only for pull requests that were
    submitted directly from the main repository.

# Releases vs development branches

[`deploydocs`](@ref) will automatically figure out whether it is deploying the documentation
for a tagged release or just a development branch (usually, based on the environment
variables set by the CI system).

With versioned tags, [`deploydocs`](@ref) discards the build metadata (i.e. `+` and
everything that follows it) from the version number when determining the name of the
directory into which the documentation gets deployed. Pre-release identifiers are preserved.

# See Also

The [Hosting Documentation](@ref) section of the manual provides a step-by-step guide to
using the [`deploydocs`](@ref) function to automatically generate docs and push them to
GitHub.
"""
function deploydocs(;
        root   = Utilities.currentdir(),
        target = "build",
        dirname = "",

        repo   = error("no 'repo' keyword provided."),
        branch = "gh-pages",

        repo_previews   = repo,
        branch_previews = branch,

        deps   = nothing,
        make   = nothing,

        devbranch = "master",
        devurl = "dev",
        versions = ["stable" => "v^", "v#.#", devurl => devurl],
        forcepush::Bool = false,
        deploy_config = auto_detect_deploy_system(),
        push_preview::Bool = false,
    )

    deploy_decision = deploy_folder(deploy_config;
                                    branch=branch,
                                    branch_previews=branch_previews,
                                    devbranch=devbranch,
                                    devurl=devurl,
                                    push_preview=push_preview,
                                    repo=repo,
                                    repo_previews=repo_previews)
    if deploy_decision.all_ok
        deploy_branch = deploy_decision.branch
        deploy_repo = deploy_decision.repo
        deploy_subfolder = deploy_decision.subfolder
        deploy_is_preview = deploy_decision.is_preview

        # Add local bin path if needed.
        Deps.updatepath!()
        # Install dependencies when applicable.
        if deps !== nothing
            @debug "installing dependencies."
            deps()
        end
        # Change to the root directory and try to deploy the docs.
        cd(root) do
            # Find the commit sha.
            # We'll make sure we run the git commands in the source directory (root), in case
            # the working directory has been changed (e.g. if the makedocs' build argument is
            # outside root).
            sha = try
                readchomp(`git rev-parse --short HEAD`)
            catch
                # git rev-parse will throw an error and return code 128 if it is not being
                # run in a git repository, which will make run/readchomp throw an exception.
                # We'll assume that if readchomp fails it is due to this and set the sha
                # variable accordingly.
                "(not-git-repo)"
            end

            @debug "setting up target directory."
            isdir(target) || mkpath(target)
            # Run extra build steps defined in `make` if required.
            if make !== nothing
                @debug "running extra build steps."
                make()
            end
            @debug "pushing new documentation to remote: '$deploy_repo:$deploy_branch'."
            mktempdir() do temp
                git_push(
                    root, temp, deploy_repo;
                    branch=deploy_branch, dirname=dirname, target=target,
                    sha=sha, deploy_config=deploy_config, subfolder=deploy_subfolder,
                    devurl=devurl, versions=versions, forcepush=forcepush,
                    is_preview=deploy_is_preview,
                )
            end
        end
    end
end

"""
    git_push(
        root, tmp, repo;
        branch="gh-pages", dirname="", target="site", sha="", devurl="dev",
        deploy_config, folder,
    )

Handles pushing changes to the remote documentation branch.
The documentation are placed in the folder specified by `subfolder`.
"""
function git_push(
        root, temp, repo;
        branch="gh-pages", dirname="", target="site", sha="", devurl="dev",
        versions, forcepush=false, deploy_config, subfolder,
        is_preview::Bool = false,
    )
    dirname = isempty(dirname) ? temp : joinpath(temp, dirname)
    isdir(dirname) || mkpath(dirname)

    target_dir = abspath(target)

    # Generate a closure with common commands for ssh and https
    function git_commands(sshconfig=nothing)
        # Setup git.
        run(`git init`)
        run(`git config user.name "Documenter.jl"`)
        run(`git config user.email "documenter@juliadocs.github.io"`)
        if sshconfig !== nothing
            run(`git config core.sshCommand "ssh -F $(sshconfig)"`)
        end

        # Fetch from remote and checkout the branch.
        run(`git remote add upstream $upstream`)
        try
            run(`git fetch upstream`)
        catch e
            @error """
            Git failed to fetch $upstream
            This can be caused by a DOCUMENTER_KEY variable that is not correctly set up.
            Make sure that the environment variable is properly set up as a Base64-encoded string
            of the SSH private key. You may need to re-generate the keys with DocumenterTools.
            """
            rethrow(e)
        end

        try
            run(`git checkout -b $branch upstream/$branch`)
        catch e
            @info """
            Checking out $branch failed, creating a new orphaned branch.
            This usually happens when deploying to a repository for the first time and
            the $branch branch does not exist yet. The fatal error above is expected output
            from Git in this situation.
            """
            @debug "checking out $branch failed with error: $e"
            run(`git checkout --orphan $branch`)
            run(`git commit --allow-empty -m "Initial empty commit for docs"`)
        end

        # Copy docs to `subfolder` directory.
        deploy_dir = joinpath(dirname, subfolder)
        gitrm_copy(target_dir, deploy_dir)
        Writers.HTMLWriter.generate_siteinfo_file(deploy_dir, subfolder)

        # Expand the users `versions` vector
        entries, symlinks = Writers.HTMLWriter.expand_versions(dirname, versions)

        # Create the versions.js file containing a list of `entries`.
        # This must always happen after the folder copying.
        Writers.HTMLWriter.generate_version_file(joinpath(dirname, "versions.js"), entries)

        # generate the symlinks, make sure we don't overwrite devurl
        cd(dirname) do
            for kv in symlinks
                i = findfirst(x -> x.first == devurl, symlinks)
                if i === nothing
                    rm_and_add_symlink(kv.second, kv.first)
                else
                    throw(ArgumentError(string("link `$(kv)` cannot overwrite ",
                        "`devurl = $(devurl)` with the same name.")))
                end
            end
        end

        # Add, commit, and push the docs to the remote.
        run(`git add -A .`)
        if !success(`git diff --cached --exit-code`)
            if forcepush
                run(`git commit --amend --date=now -m "build based on $sha"`)
                run(`git push -fq upstream HEAD:$branch`)
            else
                run(`git commit -m "build based on $sha"`)
                run(`git push -q upstream HEAD:$branch`)
            end
        else
            @debug "new docs identical to the old -- not committing nor pushing."
        end
    end

    if authentication_method(deploy_config) === SSH
        # Get the parts of the repo path and create upstream repo path
        user, host, upstream = user_host_upstream(repo)

        keyfile = abspath(joinpath(root, ".documenter"))
        try
            if is_preview
                keycontent = documenter_key_previews(deploy_config)
            else
                keycontent = documenter_key(deploy_config)
            end
            write(keyfile, base64decode(keycontent))
        catch e
            @error """
            Documenter failed to decode the DOCUMENTER_KEY environment variable.
            Make sure that the environment variable is properly set up as a Base64-encoded string
            of the SSH private key. You may need to re-generate the keys with DocumenterTools.
            """
            rm(keyfile; force=true)
            rethrow(e)
        end
        chmod(keyfile, 0o600)

        try
            mktemp() do sshconfig, io
                print(io,
                """
                Host $host
                    StrictHostKeyChecking no
                    User $user
                    HostName $host
                    IdentityFile "$keyfile"
                    IdentitiesOnly yes
                    BatchMode yes
                """)
                close(io)
                chmod(sshconfig, 0o600)
                # git config core.sshCommand requires git 2.10.0, but
                # GIT_SSH_COMMAND works from 2.3.0 so define both.
                withenv("GIT_SSH_COMMAND" => "ssh -F $(sshconfig)") do
                    cd(() -> git_commands(sshconfig), temp)
                end
            end
            post_status(deploy_config; repo=repo, type="success", subfolder=subfolder)
        catch e
            @error "Failed to push:" exception=(e, catch_backtrace())
            post_status(deploy_config; repo=repo, type="error")
        finally
            # Remove the unencrypted private key.
            isfile(keyfile) && rm(keyfile)
        end
    else # authentication_method(deploy_config) === HTTPS
        # The upstream URL to which we push new content authenticated with token
        upstream = authenticated_repo_url(deploy_config)
        try
            cd(git_commands, temp)
            post_status(deploy_config; repo=repo, type="success", subfolder=subfolder)
        catch e
            @error "Failed to push:" exception=(e, catch_backtrace())
            post_status(deploy_config; repo=repo, type="error")
        end
    end
end

function rm_and_add_symlink(target, link)
    if ispath(link)
        @warn "removing `$(link)` and linking `$(link)` to `$(target)`."
        rm(link; force = true, recursive = true)
    end
    symlink(target, link)
end

"""
    user_host_upstream(repo)

Disassemble repo address into user, host, and path to repo. If no user is given, default to
"git". Reassemble user, host and path into an upstream to `git push` to.
"""
function user_host_upstream(repo)
    #= the regex has three parts:
    (?:([^@]*)@)?  matches any number of characters up to the first "@", if present,
        capturing only the characters before the "@" - this captures the username
    (?:([^\/:]*)[\/:]){1}  matches exactly one instance of any number of characters
        other than "/" or ":" up to the first "/" or ":" - this captures the hostname
    [\/]?(.*)  matches the rest of the repo, except an initial "/" if present (e.g. if
        repo is of the form usr@host:/path/to/repo) - this captures the path on the host
    =#
    m = match(r"(?:([^@]*)@)?(?:([^\/:]*)[\/:]){1}[\/]?(.*)", repo)
    (m === nothing) && error("Invalid repo path $repo")
    user, host, pth = m.captures
    user = (user === nothing) ? "git" : user
    upstream = "$user@$host:$pth"
    return user, host, upstream
end

"""
    gitrm_copy(src, dst)

Uses `git rm -r` to remove `dst` and then copies `src` to `dst`. Assumes that the working
directory is within the git repository of `dst` is when the function is called.

This is to get around [#507](https://github.com/JuliaDocs/Documenter.jl/issues/507) on
filesystems that are case-insensitive (e.g. on OS X, Windows). Without doing a `git rm`
first, `git add -A` will not detect case changes in filenames.
"""
function gitrm_copy(src, dst)
    # --ignore-unmatch so that we wouldn't get errors if dst does not exist
    run(`git rm -rf --ignore-unmatch $(dst)`)
    # git rm also removed parent directories
    # if they are empty so need to mkpath after
    mkpath(dst)
    cp(src, dst; force=true)
end

function getenv(regex::Regex)
    for (key, value) in ENV
        occursin(regex, key) && return value
    end
    error("could not find key/iv pair.")
end

"""
    doctest(package::Module; kwargs...)

Convenience method that runs and checks all the doctests for a given Julia package.
`package` must be the `Module` object corresponding to the top-level module of the package.
Behaves like an `@testset` call, returning a testset if all the doctests are successful or
throwing a `TestSetException` if there are any failures. Can be included in other testsets.

# Keywords

**`manual`** controls how manual pages are handled. By default (`manual = true`), `doctest`
assumes that manual pages are located under `docs/src`. If that is not the case, the
`manual` keyword argument can be passed to specify the directory. Setting `manual = false`
will skip doctesting of manual pages altogether.

Additional keywords are passed on to the main [`doctest`](@ref) method.
"""
function doctest(package::Module; manual=true, testset=nothing, kwargs...)
    if pathof(package) === nothing
        throw(ArgumentError("$(package) is not a top-level package module."))
    end
    source = nothing
    if manual === true
         source = normpath(joinpath(dirname(pathof(package)), "..", "docs", "src"))
         isdir(source) || throw(ArgumentError("""
         Package $(package) does not have a documentation source directory at standard location.
         Searched at: $(source)
         If ...
         """))
    end
    testset = (testset === nothing) ? "Doctests: $(package)" : testset
    doctest(source, [package]; testset=testset, kwargs...)
end

"""
    doctest(source, modules; kwargs...)

Runs all the doctests in the given modules and on manual pages under the `source` directory.
Behaves like an `@testset` call, returning a testset if all the doctests are successful or
throwing a `TestSetException` if there are any failures. Can be included in other testsets.

The manual pages are searched recursively in subdirectories of `source` too. Doctesting of
manual pages can be disabled if `source` is set to `nothing`.

# Keywords

**`testset`** specifies the name of test testset (default `Doctests`).

**`doctestfilters`** vector of regex to filter tests (see the manual on [Filtering Doctests](@ref))

**`fix`**, if set to `true`, updates all the doctests that fail with the correct output
(default `false`).

!!! warning
    When running `doctest(...; fix=true)`, Documenter will modify the Markdown and Julia
    source files. It is strongly recommended that you only run it on packages in Pkg's
    develop mode and commit any staged changes. You should also review all the changes made
    by `doctest` before committing them, as there may be edge cases when the automatic
    fixing fails.
"""
function doctest(
        source::Union{AbstractString,Nothing},
        modules::AbstractVector{Module};
        fix = false,
        testset = "Doctests",
        doctestfilters = Regex[],
    )
    function all_doctests()
        dir = mktempdir()
        try
            @debug "Doctesting in temporary directory: $(dir)" modules
            if source === nothing
                source = joinpath(dir, "src")
                mkdir(source)
            end
            makedocs(
                root = dir,
                source = source,
                sitename = "",
                doctest = fix ? :fix : :only,
                modules = modules,
                doctestfilters = doctestfilters,
            )
            true
        catch err
            @error "Doctesting failed" exception=(err, catch_backtrace())
            false
        finally
            try
                rm(dir; recursive=true)
            catch e
                @warn "Documenter was unable to clean up the temporary directory $(dir)" exception = e
            end
        end
    end
    @testset "$testset" begin
        @test all_doctests()
    end
end

end # module


text:
export dnnObjFctn, evalObjFctn

"""
objective function for deep neural networks

J(theta,C) = loss(h(W*Y(theta)), C) + Rtheta(theta) + R(W)

"""
mutable struct dnnObjFctn
     net    :: AbstractMeganetElement              # network param (including data)
     pLoss             # loss function
     pRegTheta         # regularizer for network parameters
     pRegW             # regularizer for classifier
     dnnObjFctn(net,pLoss,pRegTheta,pRegW) =
               new(net,pLoss,pRegTheta,pRegW)
 end

splitWeights(this::dnnObjFctn,x) = (return x[1:nTheta(this.net)], x[nTheta(this.net)+1:end])

function getMisfit(this::dnnObjFctn,thetaW::Vector{T},Y::Array{T},C::Array{T},tmp::Array,doDerivative=true) where {T<:Number}
    theta,W = splitWeights(this,thetaW)
    return getMisfit(this,theta,W,Y,C,tmp,doDerivative)
end

function getMisfit(this::dnnObjFctn,theta::Array{T},W::Array{T},Y::Array{T},C::Array{T},tmp::Array,doDerivative=true) where {T<:Number}

    YN,dummy,tmp = apply(this.net,theta,Y,tmp,doDerivative)

    Fc,hisF,dWF,d2WF,dYF,d2YF = getMisfit(this.pLoss,W,YN,C,doDerivative,doDerivative)

    if doDerivative
        dYF = JthetaTmv(this.net,dYF,zeros(T,0),theta,Y,tmp)
    end
    return Fc,hisF,vec(dYF),vec(dWF),tmp
end

function evalObjFctn(this::dnnObjFctn,thetaW::Array{T},Y::Array{T},C::Array{T},tmp::Array,doDerivative=true) where {T<:Number}
    theta,W = splitWeights(this,thetaW)

    # compute misfit
    Fc,hisF,dFth,dFW,tmp = getMisfit(this,theta,W,Y,C,tmp,doDerivative)

    # regularizer for weights
    Rth,dRth, = regularizer(this.pRegTheta,theta)

    # regularizer for classifier
    RW,dRW, = regularizer(this.pRegW,W)

    Jc = Fc + Rth + RW
    dJ = [dFth+dRth; dFW+dRW]

    return convert(T,Jc),hisF,convert(Array{T},dJ),tmp
end

ext: jl


text:
module FRS_Household_Getter

import CSVFiles
import DataFrames: DataFrame
using Definitions
import Model_Household: Household

include( "hhld_from_frame.jl" )

export  initialise, get_household

MODEL_HOUSEHOLDS = Vector{Union{Missing,Household}}(missing,0)

"""
return (number of households available, num people loaded inc. kids, num hhls in dataset (should always = item[1]))
"""
function initialise(
        ;
        household_name :: AbstractString = "model_households",
        people_name :: AbstractString = "model_people",
        start_year = -1 ) :: Tuple

    global MODEL_HOUSEHOLDS
    hh_dataset =CSVFiles.load( File(format"CSV", "$(MODEL_DATA_DIR)/$(household_name).tab" ),delim='\t' ) |> DataFrame
    # FIXME HORRIBLE HACK - correct (???) weights for num years in dataset
    # we need to generate our own weights here
    nyears = 2018 - start_year
    hh_dataset[!,:weight] ./= nyears
    people_dataset = CSVFiles.load( File(format"CSV", "$(MODEL_DATA_DIR)/$(people_name).tab" ),delim='\t' ) |> DataFrame
    npeople = size( people_dataset)[1]
    nhhlds = size( hh_dataset )[1]
    MODEL_HOUSEHOLDS = Vector{Union{Missing,Household}}(missing,nhhlds)
    for hseq in 1:nhhlds
        MODEL_HOUSEHOLDS[hseq] = load_hhld_from_frame( hseq, hh_dataset[hseq,:], people_dataset )
        uprate!( MODEL_HOUSEHOLDS[hseq] )
    end
    (size(MODEL_HOUSEHOLDS)[1],npeople,nhhlds)
end

function get_household( pos :: Integer ) :: Household
    global MODEL_HOUSEHOLDS
    MODEL_HOUSEHOLDS[pos]
end

end


text:
using OmegaGrad
using OmegaCore
using Zygote

function test_zygote_1()
  x = 1 ~ Normal(0, 1)
  ω = defω()
  ω = x(ω)
  grad(x, ω, ZygoteGrad)

  # Complete this test
  # @test ...
end

function test_zygote_2()
  x = 1 ~ Normal(0, 1)
  ω = defω()
  ω = x(ω)
  grad(x, ω, ZygoteGrad)

  # Complete this test
  # @test ...
end

@testset "ZygoteGrad" begin
  test_zygote_1()
  test_zygote_2()
end

max_stars_repo_name: zenna/expect

max_issues_repo_name: zenna/expect

max_forks_repo_name: zenna/expect


text:
export AR1, AR1TV, ARH

function AR1(θ)
    μ = fill(θ[1],1)
    Σ0 = fill(θ[2].^2,1,1)
    F = fill(θ[3],1,1)
    G = zeros(1,1)
    Q = fill(θ[4].^2,1,1)
    H = [1.0]'
    R = fill(θ[5].^2,1,1)
    Γ = zeros(1,1)
    Θ = fill(1.0,1,1)
    LinearStateSpaceModel(F,Q,G,H,R,Γ,Θ,μ,Σ0)
end

function AR1TV(θ,N)
    μ = fill(θ[1],1)
    Σ0 = fill(θ[2].^2,1,1)
    F = fill(fill(θ[3],1,1),N)
    G = fill(zeros(1,1),N)
    Q = fill(fill(θ[4].^2,1,1),N)
    H = fill([1.0]',N)
    R = fill(fill(θ[5].^2,1,1),N)
    Γ = fill(zeros(1,1),N)
    Θ = fill(fill(1.0,1,1),N)
    TimeVariantLinearStateSpaceModel(F,Q,G,H,R,Γ,Θ,μ,Σ0)
end

"""
An autoregressive model with specified regression coefficients

    - θ = [δ;atanh(φ);σ]
    - β = [α1;α2;σε]

x_{t} = δ + φ*x_{t-1} + σ η_t
y_{t} = α1+ α2*x_t + σε ε_t

"""
function ARH(θ,β)
    δ = θ[1]
    φ = tanh(θ[2])
    σ = θ[3]
    σε = θ[4]
    α1= β[1]
    α2= β[2]

    μ = fill(δ/(1-φ),1)
    Σ0= fill(σ^2/(1-φ^2),1,1)
    F = fill(φ,1,1)
    Q = fill(σ^2,1,1)
    G = fill(δ,1,1)
    H = fill(α2,1,1)
    R = fill(σε^2,1,1)
    Γ = fill(α1,1,1)
    Θ = fill(1.0,1,1)

    LinearStateSpaceModel(F, Q, G, H, R, Γ, Θ, μ, Σ0)
end

function ARH(β)
    μ(θ) = fill(θ[1]/(1-tanh(θ[2])),1)
    Σ0(θ)= fill(θ[3]^2/(1-tanh(θ[2])^2),1,1)
    F(θ) = fill(tanh(θ[2]),1,1)
    Q(θ) = fill(θ[3]^2,1,1)
    G(θ) = fill(θ[1],1,1)
    H(θ) = fill(β[2],1,1)
    R(θ) = fill(θ[4]^2,1,1)
    Γ(θ) = fill(β[1],1,1)
    Θ(θ) = fill(1.0,1,1)

    LinearStateSpaceModel(F, Q, G, H, R, Γ, Θ, μ, Σ0)
end


text:
using JuliaDB
using Base.Test

@testset "setindex!" begin
    x = ndsparse(([1,2],[2,3]), [4,5], chunks=2)
    x[2,3] = 6
    @test x == ndsparse(([1,2], [2,3]), [4,6])
    x[2,4] = 7
    @test x == ndsparse(([1,2,2], [2,3,4]), [4,6,7])

    x = ndsparse(@NT(x=[1,2],y=[2,3]), [4,5], chunks=2)
    x[2,3] = 6
    @test x == ndsparse(@NT(x=[1,2], y=[2,3]), [4,6])
    x[2,4] = 7
    @test x == ndsparse(@NT(x=[1,2,2], y=[2,3,4]), [4,6,7])
end

@testset "extractarray" begin

    t = NDSparse(Columns(a=[1,1,1,2,2], b=[1,2,3,1,2]),
                     Columns(c=[1,2,3,4,5], d=[5,4,3,2,1]))
    for i=[2, 3, 5]
        d = compute(distribute(t, i))
        dist = map(get, map(JuliaDB.nrows, d.domains))
        @test map(length, Dagger.domainchunks(keys(d, 1))) == dist
        @test collect(keys(d, 2)) == t.index.columns[2]
        @test collect( values(d, 2)) == t.data.columns[2]
    end
end

@testset  "printing" begin
    x = distribute(NDSparse([1], [1]), 1)
    @test repr(x) == """
    1-d Distributed NDSparse with 1 values (Int64) in 1 chunks:
    1 │
    ──┼──
    1 │ 1"""
end

import JuliaDB: chunks, index_spaces, has_overlaps
@testset "has_overlaps" begin
    t = NDSparse(Columns([1,1,2,2,2,3], [1,2,1,1,2,1]), [1,2,3,4,5,6])
    d = distribute(t, [2,3,1])
    i = d.domains
    @test !has_overlaps(i, closed=false)
    @test !has_overlaps(i)

    d = distribute(t, [2,2,2])
    i = d.domains
    @test !has_overlaps(i, closed=false)
    @test !has_overlaps(i)

    d = distribute(t, [2,1,3])
    i = d.domains
    @test !has_overlaps(i, closed=false)
    @test has_overlaps(i)
end

import JuliaDB: with_overlaps, delayed
@testset "with_overlaps" begin
    t = NDSparse([1,1,2,2,2,3], [1,2,3,4,5,6])
    d = distribute(t, [2,2,2])
    group_count = 0
    t1 = with_overlaps(d) do cs
        if length(cs) > 1
            group_count += 1
            @test length(cs) == 2
            @test collect(cs[1]) == NDSparse([2,2], [3,4])
            @test collect(cs[2]) == NDSparse([2,3], [5,6])
            return delayed((x,y) -> merge(x,y,agg=nothing))(cs...)
        else
            cs[1]
        end
    end
    @test collect(groupreduce(+, t1)) == groupreduce(+, t)
    @test group_count == 1
end

import JuliaDB: subtable
@testset "subtable" begin
    t = NDSparse([1,2,3,4], [5,6,7,8])
    d = distribute(t, 3)

    for i=1:4
        @test collect(subtable(d, 1:i)) == subtable(t, 1:i)
        if i>=2
            @test collect(subtable(d, 2:i)) == subtable(t, 2:i)
        end
    end
end

@testset "Iterators.partition" begin
    t = compute(distribute(NDSparse([1:7;],[8:14;]), [3,2,2]))
    parts = map(NDSparse, Iterators.partition([1:7;], 2) |> collect,
                              Iterators.partition([8:14;], 2) |> collect)
    @test [p for p in Iterators.partition(t, 2)] == parts
end


text:
using Images

@inline function hsv2rgb(h, s, v)
    const c = v * s
    const x = c * (1 - abs(((h/60) % 2) - 1))
    const m = v - c

    const r,g,b =
        if h < 60
            (c, x, 0)
        elseif h < 120
            (x, c, 0)
        elseif h < 180
            (0, c, x)
        elseif h < 240
            (0, x, c)
        elseif h < 300
            (x, 0, c)
        else
            (c, 0, x)
        end

    (r + m), (b + m), (g + m)
end

function mandelbrot()

    const w, h = 1000, 1000

    const zoom  = 0.5
    const moveX = 0
    const moveY = 0

    const img = Array{RGB{Float64}}(h, w)
    const maxIter = 30

    for x in 1:w
        for y in 1:h
            i = maxIter
            const c = Complex(
                (2*x - w) / (w * zoom) + moveX,
                (2*y - h) / (h * zoom) + moveY
            )
            z = c
            while abs(z) < 2 && (i -= 1) > 0
                z = z^2 + c
            end
            const r,g,b = hsv2rgb(i / maxIter * 360, 1, i / maxIter)
            img[y,x] = RGB{Float64}(r, g, b)
        end
    end

    save("mandelbrot_set.png", img)
end

mandelbrot()

max_stars_repo_name: ethansaxenian/RosettaDecode

max_issues_repo_name: ethansaxenian/RosettaDecode

max_forks_repo_name: ethansaxenian/RosettaDecode


text:
using Calculus
using DataFrames
using StatsModels
using LibGEOS
using GaussianProcesses
using GaussianProcesses: update_mll_and_dmll!, update_mll!, get_params, set_params!
using Distributions
using Random
using Optim
using LinearAlgebra

@testset "GPRealisations gradients" begin
    Random.seed!(1) # for replicability
    τ = 0.3 # constant treatment effect
    nobs = 100
    geordd_df, border_X1, border_X2, border_geo = sim_data(nobs, τ)

    fmla = @formula(outcome ~ GP(X1, X2) | region + covarA + covarB + categ)
    geordd = GeoRDD.regions_from_dataframe(fmla, geordd_df)
    # choose a Gaussian process kernel from the GaussianProcesses.jl package
    k_se = SEIso(log(0.5), log(1.0)) # Squared Exponential spatial kernel
    k_m = Const(log(20.0))           # constant kernel for the mean parameters
    βkern = LinIso(log(1.0))         # linear kernel for the other covariates
    logNoise = 1.0
    # create the Gaussian process object from the regional data:
    mgpcv = GeoRDD.MultiGPCovars(geordd, k_se+k_m, βkern, logNoise)
    param_kwargs = Dict(:domean=>false, :kern=>true, :noise=>true, :beta=>true)
    buf = Matrix{Float64}(undef, nobs, nobs)
    update_mll_and_dmll!(mgpcv, buf; param_kwargs...)
    grad = copy(mgpcv.dmll)
    init_x = get_params(mgpcv; param_kwargs...)

    numer_grad = Calculus.gradient(init_x) do params
        set_params!(mgpcv, params; param_kwargs...)
        update_mll!(mgpcv)
        t = mgpcv.mll
        t
    end
    @test grad ≈ numer_grad atol=1e-3
end


text:
using BinaryProvider, Libdl

# Parse some basic command-line arguments
const verbose = "--verbose" in ARGS
const prefix = Prefix(get([a for a in ARGS if a != "--verbose"], 1, joinpath(@__DIR__, "usr")))
products = [
    LibraryProduct(prefix, String["libmbedcrypto"], :libmbedcrypto),
    LibraryProduct(prefix, String["libmbedtls"], :libmbedtls),
    LibraryProduct(prefix, String["libmbedx509"], :libmbedx509),
]

const juliaprefix = joinpath(Sys.BINDIR, "..")

juliaproducts = Product[
    LibraryProduct(juliaprefix, "libmbedtls", :libmbedtls)
    LibraryProduct(juliaprefix, "libmbedcrypto", :libmbedcrypto)
    LibraryProduct(juliaprefix, "libmbedx509", :libmbedx509)
]

# Download binaries from hosted location
bin_prefix = "https://github.com/JuliaWeb/MbedTLSBuilder/releases/download/v0.16.0"

# Listing of files generated by BinaryBuilder:
download_info = Dict(
    Linux(:aarch64, libc=:glibc) => ("$bin_prefix/MbedTLS.v2.13.1.aarch64-linux-gnu.tar.gz", "051b7b40911154196d0378bd41a2cc0182e4c5e3aa06506702adca36a875018f"),
    Linux(:aarch64, libc=:musl) => ("$bin_prefix/MbedTLS.v2.13.1.aarch64-linux-musl.tar.gz", "d031de3dd7723b647aa8dcb5fbe9777e7db0927e04a8cbf779a4a9da1f767061"),
    Linux(:armv7l, libc=:glibc, call_abi=:eabihf) => ("$bin_prefix/MbedTLS.v2.13.1.arm-linux-gnueabihf.tar.gz", "77adab909961fc750579c0fec87152b74048e466d8b9ca80df6d4f286fc92101"),
    Linux(:armv7l, libc=:musl, call_abi=:eabihf) => ("$bin_prefix/MbedTLS.v2.13.1.arm-linux-musleabihf.tar.gz", "a47e3d6bb09bb7564fea16dbe099e7b772f18128b99a5f92627a97dca3266f5d"),
    Linux(:i686, libc=:glibc) => ("$bin_prefix/MbedTLS.v2.13.1.i686-linux-gnu.tar.gz", "d35401469c134a151ab139e0fd400729f72c223ef8c0a736a76aae63c48dfa95"),
    Linux(:i686, libc=:musl) => ("$bin_prefix/MbedTLS.v2.13.1.i686-linux-musl.tar.gz", "2fa1b8674a3898eb08b3d28bd45fbfe022137a259edf407c564b5a817526647d"),
    Windows(:i686) => ("$bin_prefix/MbedTLS.v2.13.1.i686-w64-mingw32.tar.gz", "5fc0d5749fe644eb60cc5dda361831c0c64e6847559d1c6b719b185979e867ae"),
    Linux(:powerpc64le, libc=:glibc) => ("$bin_prefix/MbedTLS.v2.13.1.powerpc64le-linux-gnu.tar.gz", "8d36c2085f6cec55077fb7af485399a2b8d4a55033962c2a352d764fc153590f"),
    MacOS(:x86_64) => ("$bin_prefix/MbedTLS.v2.13.1.x86_64-apple-darwin14.tar.gz", "907df87621d4a63e1621c585cf351cf4d34a2e880c124f04cb4b404c690cc224"),
    Linux(:x86_64, libc=:glibc) => ("$bin_prefix/MbedTLS.v2.13.1.x86_64-linux-gnu.tar.gz", "46d14792f88781f19c30ec0e9fd1815d61906d9f002499d3e277e206da484a99"),
    Linux(:x86_64, libc=:musl) => ("$bin_prefix/MbedTLS.v2.13.1.x86_64-linux-musl.tar.gz", "fe43ca4323d295f2a05ff1ffde8c724ea43fad6a38d5b902dc8aee47a9160ad4"),
    FreeBSD(:x86_64) => ("$bin_prefix/MbedTLS.v2.13.1.x86_64-unknown-freebsd11.1.tar.gz", "537ab0cdc5fcfb0acc9bb6d013746f80007332b297f88423db1dc1306769f9b5"),
    Windows(:x86_64) => ("$bin_prefix/MbedTLS.v2.13.1.x86_64-w64-mingw32.tar.gz", "99d7f51190b841731139a5a561b7ee54fe99c1688c27ca3efd75a047da88eacf"),
)

# First, check to see if we're all satisfied
gpl = haskey(ENV, "USE_GPL_MBEDTLS")
forcebuild = parse(Bool, get(ENV, "FORCE_BUILD", "false")) || gpl
done = false
if any(!satisfied(p; verbose=verbose) for p in products) || forcebuild
    if haskey(download_info, platform_key()) && !forcebuild
        # Download and install binaries
        url, tarball_hash = download_info[platform_key()]
        install(url, tarball_hash; prefix=prefix, force=true, verbose=verbose)
        done = all(satisfied(p; verbose=verbose) for p in products)
        done && @info "using prebuilt binaries"
    end
    if !done && all(satisfied(p; verbose=verbose) for p in juliaproducts) && !forcebuild
        @info "using julia-shippied binaries"
        products = juliaproducts
    elseif !done || forcebuild
        @info "attempting source build"
        VERSION = "2.13.0"
        url, hash = haskey(ENV, "USE_GPL_MBEDTLS") ?
            ("https://tls.mbed.org/download/mbedtls-$VERSION-gpl.tgz", "a08ddf08aae55fc4f48fbc6281fcb08bc5c53ed53ffd15355ee0d75ec32b53ae") :
            ("https://tls.mbed.org/download/mbedtls-$VERSION-apache.tgz", "593b4e4d2e1629fc407ab4750d69fa309a0ddb66565dc3deb5b60eddbdeb06da")
        download_verify(url, hash, joinpath(@__DIR__, "mbedtls.tgz"), force=true, verbose=true)
        unpack(joinpath(@__DIR__, "mbedtls.tgz"), @__DIR__; verbose=true)
        withenv("VERSION"=>VERSION) do
            run(Cmd(`./build.sh`, dir=@__DIR__))
        end
        if any(!satisfied(p; verbose=verbose) for p in products)
            error("attempted to build mbedtls shared libraries, but they couldn't be located (deps/usr/lib)")
        end
    end
end

depsname = joinpath(@__DIR__, "deps.jl")
write_deps_file(depsname, products, verbose=verbose)
mv(depsname, depsname*".orig", force = true)
open(depsname, "w") do io
    write(io, """
        # using Libdl
        # @show ENV["LD_LIBRARY_PATH"]
        # @show length(Libdl.find_library("libmbedtls.so")) == 0
        # if length(Libdl.find_library("libmbedtls.so")) == 0
            const libmbedcrypto = "libmbedcrypto.so"
            const libmbedtls = "libmbedtls.so"
            const libmbedx509 = "libmbedx509.so"
        # else
            # include("deps.jl.orig")
        # end
    """)
end


text:
"""
Widgets are renderable elements which capture window events.
They are immutable, and should be re-created in a real-time loop.

A widget subtype should always extend the following methods:
- `GeometryExperiments.MeshVertexEncoding`
- `Base.in`
- `zindex`

If `Base.in` is not extended, then either of `GeometryExperiments.boundingelement` and `GeometryExperiments.PointSet` must be extended.

To allow the specification of callbacks, it may extend `callbacks`.
"""
abstract type Widget end

not_implemented_for(x) = error("Not implemented for $(typeof(x))")

GeometryExperiments.MeshVertexEncoding(w::Widget) = not_implemented_for(w)

"""
Test whether the point `p` is inside the widget `w`.
"""
Base.in(p::Point, w::Widget) = p in boundingelement(w)

GeometryExperiments.boundingelement(w::Widget) = boundingelement(PointSet(w))

GeometryExperiments.PointSet(w::Widget) = PointSet(boundingelement(w))

"""
Z-index of the widget. Used to determine whether a widget should be rendered and capture events.
Defaults to zero.
"""
zindex(w::Widget) = 0.


text:
macro temp_grad_min(c1, x1, y1, c2, x2, y2)
    sym_grad11 = Symbol(String(x1)*"_grad")
    sym_grad12 = Symbol(String(y1)*"_grad")
    sym_grad21 = Symbol(String(x2)*"_grad")
    sym_grad22 = Symbol(String(y2)*"_grad")
    esc(quote
        grad_temp = $c1*$x1 < $c1*$y1 ? $c1*$sym_grad11 : $c1*$sym_grad12
        grad_temp += $c2*$x2 < $c2*$y2 ? $c2*$sym_grad21 : $c2*$sym_grad22
        end)
end

macro temp_grad_max(c1, x1, y1, c2, x2, y2)
    sym_grad11 = Symbol(String(x1)*"_grad")
    sym_grad12 = Symbol(String(y1)*"_grad")
    sym_grad21 = Symbol(String(x2)*"_grad")
    sym_grad22 = Symbol(String(y2)*"_grad")
    esc(quote
        grad_temp = $c1*$x1 > $c1*$y1 ? $c1*$sym_grad11 : $c1*$sym_grad12
        grad_temp += $c2*$x2 > $c2*$y2 ? $c2*$sym_grad21 : $c2*$sym_grad22
        end)
end

min4(args...) = findmin(args)
max4(args...) = findmax(args)

const APRIORI_DELTA = 1E-6

function mult_apriori_kernel(x1::MC{N,T}, x2::MC{N,T}, z::Interval{Float64},
                             u1::Float64, u2::Float64, a1::Float64, a2::Float64,
                             u1grad::SVector{N,Float64}, u2grad::SVector{N,Float64}) where {N, T<:Union{NS, MV}}

    x1L = lo(x1);  x1U = hi(x1);  x2L = lo(x2);  x2U = hi(x2)
    x1Li = Interval(lo(x1));  x1Ui = Interval(hi(x1));  x2Li = Interval(lo(x2));  x2Ui = Interval(hi(x2))
    u1i = Interval(u1); u2i = Interval(u2)
    a1i = Interval(a1); a2i = Interval(a2)
    x1cv = x1.cv
    x1cc = x1.cc
    x2cv = x2.cv
    x2cc = x2.cc
    x1i = Interval(x1.cv, x1.cc)
    x2i = Interval(x2.cv, x2.cc)
    x1cv_grad = x1.cv_grad;  x1cc_grad = x1.cc_grad;
    x2cv_grad = x2.cv_grad;  x2cc_grad = x2.cc_grad

    a2x1i = a2i*x1i
    a1x2i = a1i*x2i
    x1Ux2i = x1Ui*x2i
    x2Ux1i = x2Ui*x1i
    x1Lx2i = x1Li*x2i
    x2Lx1i = x2Li*x1i

    # additional constraints from underestimators (convex)
    w1cvi = (x2Ui - a2i)*u1i + (x1Ui - a1i)*u2i + lo(a2x1i) + lo(a1x2i) + a1i*a2i - a1i*x2Ui - a2i*x1Ui - APRIORI_DELTA
    w2cvi = (x2Ui - x2Li)*u1i + lo(x2Lx1i) + lo(a1x2i) - a1i*x2Ui - APRIORI_DELTA
    w3cvi = (x1Ui - x1Li)*u2i + lo(a2x1i) + lo(x1Lx2i) - x1Ui*a2i - APRIORI_DELTA
    w4cvi = (a2i - x2Li)*u1i + (a1i - x1Li)*u2i + lo(x2Lx1i) + lo(x1Lx2i) - a1i*a2i - APRIORI_DELTA

    w1cv = lo(w1cvi)
    w2cv = lo(w2cvi)
    w3cv = lo(w3cvi)
    w4cv = lo(w4cvi)

    # additional constraints from underestimators (concave)
    w1cci = (x2Li - a2i)*u1i + (a1i - x1Ui)*u2i + hi(a2x1i) + hi(x1Ux2i) - a1i*x2Li + APRIORI_DELTA
    w2cci = (x2Li - x2Ui)*u1i + hi(a1x2i) + hi(x2Ux1i) - x2Li*a1i + APRIORI_DELTA
    w3cci = (x1Li - x1Ui)*u2i + hi(a2x1i) + hi(x1Ux2i) - x1Li*a2i + APRIORI_DELTA
    w4cci = (a2i - x2Ui)*u1i + (x1Li - a1i)*u2i + hi(x2Ux1i) + hi(a1x2i) - x1Li*a2i + APRIORI_DELTA

    w1cc = hi(w1cci)
    w2cc = hi(w2cci)
    w3cc = hi(w3cci)
    w4cc = hi(w4cci)

    cv, cvind = max4(w1cv, w2cv, w3cv, w4cv)
    if cvind == 1
        @temp_grad_min(a2, x1cv, x1cc, a1, x2cv, x2cc)
        cv_grad = (x2U - a2)*u1grad + (x1U - a1)*u2grad + grad_temp
    elseif cvind == 2
        @temp_grad_min(x2L, x1cv, x1cc, a1, x2cv, x2cc)
        cv_grad = (x2U - x2L)*u2grad + grad_temp
    elseif cvind == 3
        @temp_grad_min(a2, x1cv, x1cc, x1L, x2cv, x2cc)
        cv_grad = (x1U - x1L)*u2grad + grad_temp
    elseif cvind == 4
        @temp_grad_min(x2L, x1cv, x1cc, x1L, x2cv, x2cc)
        cv_grad = (a2 - x2L)*u1grad + (a1 - x1L)*u2grad + grad_temp
    end

    cc, ccind = min4(w1cc, w2cc, w3cc, w4cc)
    if ccind == 1
        @temp_grad_max(a2, x1cv, x1cc, x1U, x2cv, x2cc)
        cc_grad = (x2L - a2)*u1grad + (a1 - x1U)*u2grad + grad_temp
    elseif ccind == 2
        @temp_grad_max(a1, x1cv, x1cc, x2U, x1cv, x1cc)
        cc_grad = (x2L - x2U)*u1grad + grad_temp
    elseif ccind == 3
        @temp_grad_max(a2, x1cv, x1cc, x1U, x2cv, x2cc)
        cc_grad = (x1L - x1U)*u2grad + grad_temp
    elseif ccind == 4
        @temp_grad_max(x2U, x1cv, x1cc, a1, x2cv, x2cc)
        cc_grad = (a2 - x2U)*u1grad + (x1L - a1)*u2grad + grad_temp
    end

    out = MC{N,T}(cv, cc, z, cv_grad, cc_grad, x1.cnst && x2.cnst)
    return out
end

max_stars_repo_name: PSORLab/RSActivationFunctions

max_issues_repo_name: PSORLab/RSActivationFunctions

max_forks_repo_name: PSORLab/RSActivationFunctions


text:
# This file is auto-generated by AWSMetadata.jl
using AWS
using AWS.AWSServices: snowball
using AWS.Compat
using AWS.UUIDs

"""
    cancel_cluster(cluster_id)
    cancel_cluster(cluster_id, params::Dict{String,<:Any})

Cancels a cluster job. You can only cancel a cluster job while it's in the AwaitingQuorum
status. You'll have at least an hour after creating a cluster job to cancel it.

# Arguments
- `cluster_id`: The 39-character ID for the cluster that you want to cancel, for example
  CID123e4567-e89b-12d3-a456-426655440000.

"""
cancel_cluster(ClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CancelCluster", Dict{String, Any}("ClusterId"=>ClusterId); aws_config=aws_config)
cancel_cluster(ClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CancelCluster", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ClusterId"=>ClusterId), params)); aws_config=aws_config)

"""
    cancel_job(job_id)
    cancel_job(job_id, params::Dict{String,<:Any})

Cancels the specified job. You can only cancel a job before its JobState value changes to
PreparingAppliance. Requesting the ListJobs or DescribeJob action returns a job's JobState
as part of the response element data returned.

# Arguments
- `job_id`: The 39-character job ID for the job that you want to cancel, for example
  JID123e4567-e89b-12d3-a456-426655440000.

"""
cancel_job(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CancelJob", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
cancel_job(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CancelJob", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    create_address(address)
    create_address(address, params::Dict{String,<:Any})

Creates an address for a Snow device to be shipped to. In most regions, addresses are
validated at the time of creation. The address you provide must be located within the
serviceable area of your region. If the address is invalid or unsupported, then an
exception is thrown.

# Arguments
- `address`: The address that you want the Snow device shipped to.

"""
create_address(Address; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateAddress", Dict{String, Any}("Address"=>Address); aws_config=aws_config)
create_address(Address, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateAddress", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("Address"=>Address), params)); aws_config=aws_config)

"""
    create_cluster(address_id, job_type, resources, role_arn, shipping_option)
    create_cluster(address_id, job_type, resources, role_arn, shipping_option, params::Dict{String,<:Any})

Creates an empty cluster. Each cluster supports five nodes. You use the CreateJob action
separately to create the jobs for each of these nodes. The cluster does not ship until
these five node jobs have been created.

# Arguments
- `address_id`: The ID for the address that you want the cluster shipped to.
- `job_type`: The type of job for this cluster. Currently, the only job type supported for
  clusters is LOCAL_USE.
- `resources`: The resources associated with the cluster job. These resources include
  Amazon S3 buckets and optional AWS Lambda functions written in the Python language.
- `role_arn`: The RoleARN that you want to associate with this cluster. RoleArn values are
  created by using the CreateRole API action in AWS Identity and Access Management (IAM).
- `shipping_option`: The shipping speed for each node in this cluster. This speed doesn't
  dictate how soon you'll get each Snowball Edge device, rather it represents how quickly
  each device moves to its destination while in transit. Regional shipping speeds are as
  follows:    In Australia, you have access to express shipping. Typically, Snow devices
  shipped express are delivered in about a day.   In the European Union (EU), you have access
  to express shipping. Typically, Snow devices shipped express are delivered in about a day.
  In addition, most countries in the EU have access to standard shipping, which typically
  takes less than a week, one way.   In India, Snow device are delivered in one to seven
  days.   In the United States of America (US), you have access to one-day shipping and
  two-day shipping.     In Australia, you have access to express shipping. Typically, devices
  shipped express are delivered in about a day.   In the European Union (EU), you have access
  to express shipping. Typically, Snow devices shipped express are delivered in about a day.
  In addition, most countries in the EU have access to standard shipping, which typically
  takes less than a week, one way.   In India, Snow device are delivered in one to seven
  days.   In the US, you have access to one-day shipping and two-day shipping.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"Description"`: An optional description of this specific cluster, for example
  Environmental Data Cluster-01.
- `"ForwardingAddressId"`: The forwarding address ID for a cluster. This field is not
  supported in most regions.
- `"KmsKeyARN"`: The KmsKeyARN value that you want to associate with this cluster.
  KmsKeyARN values are created by using the CreateKey API action in AWS Key Management
  Service (AWS KMS).
- `"Notification"`: The Amazon Simple Notification Service (Amazon SNS) notification
  settings for this cluster.
- `"SnowballType"`: The type of AWS Snow Family device to use for this cluster.   For
  cluster jobs, AWS Snow Family currently supports only the EDGE device type.
- `"TaxDocuments"`: The tax documents required in your AWS Region.
"""
create_cluster(AddressId, JobType, Resources, RoleARN, ShippingOption; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateCluster", Dict{String, Any}("AddressId"=>AddressId, "JobType"=>JobType, "Resources"=>Resources, "RoleARN"=>RoleARN, "ShippingOption"=>ShippingOption); aws_config=aws_config)
create_cluster(AddressId, JobType, Resources, RoleARN, ShippingOption, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateCluster", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("AddressId"=>AddressId, "JobType"=>JobType, "Resources"=>Resources, "RoleARN"=>RoleARN, "ShippingOption"=>ShippingOption), params)); aws_config=aws_config)

"""
    create_job()
    create_job(params::Dict{String,<:Any})

Creates a job to import or export data between Amazon S3 and your on-premises data center.
Your AWS account must have the right trust policies and permissions in place to create a
job for a Snow device. If you're creating a job for a node in a cluster, you only need to
provide the clusterId value; the other job attributes are inherited from the cluster.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AddressId"`: The ID for the address that you want the Snow device shipped to.
- `"ClusterId"`: The ID of a cluster. If you're creating a job for a node in a cluster, you
  need to provide only this clusterId value. The other job attributes are inherited from the
  cluster.
- `"Description"`: Defines an optional description of this specific job, for example
  Important Photos 2016-08-11.
- `"DeviceConfiguration"`: Defines the device configuration for an AWS Snowcone job.
- `"ForwardingAddressId"`: The forwarding address ID for a job. This field is not supported
  in most regions.
- `"JobType"`: Defines the type of job that you're creating.
- `"KmsKeyARN"`: The KmsKeyARN that you want to associate with this job. KmsKeyARNs are
  created using the CreateKey AWS Key Management Service (KMS) API action.
- `"Notification"`: Defines the Amazon Simple Notification Service (Amazon SNS)
  notification settings for this job.
- `"Resources"`: Defines the Amazon S3 buckets associated with this job. With IMPORT jobs,
  you specify the bucket or buckets that your transferred data will be imported into. With
  EXPORT jobs, you specify the bucket or buckets that your transferred data will be exported
  from. Optionally, you can also specify a KeyRange value. If you choose to export a range,
  you define the length of the range by providing either an inclusive BeginMarker value, an
  inclusive EndMarker value, or both. Ranges are UTF-8 binary sorted.
- `"RoleARN"`: The RoleARN that you want to associate with this job. RoleArns are created
  using the CreateRole AWS Identity and Access Management (IAM) API action.
- `"ShippingOption"`: The shipping speed for this job. This speed doesn't dictate how soon
  you'll get the Snow device, rather it represents how quickly the Snow device moves to its
  destination while in transit. Regional shipping speeds are as follows:   In Australia, you
  have access to express shipping. Typically, Snow devices shipped express are delivered in
  about a day.   In the European Union (EU), you have access to express shipping. Typically,
  Snow devices shipped express are delivered in about a day. In addition, most countries in
  the EU have access to standard shipping, which typically takes less than a week, one way.
  In India, Snow devices are delivered in one to seven days.   In the US, you have access to
  one-day shipping and two-day shipping.
- `"SnowballCapacityPreference"`: If your job is being created in one of the US regions,
  you have the option of specifying what size Snow device you'd like for this job. In all
  other regions, Snowballs come with 80 TB in storage capacity.
- `"SnowballType"`: The type of AWS Snow Family device to use for this job.   For cluster
  jobs, AWS Snow Family currently supports only the EDGE device type.  The type of AWS Snow
  device to use for this job. Currently, the only supported device type for cluster jobs is
  EDGE. For more information, see Snowball Edge Device Options in the Snowball Edge Developer
  Guide.
- `"TaxDocuments"`: The tax documents required in your AWS Region.
"""
create_job(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateJob"; aws_config=aws_config)
create_job(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateJob", params; aws_config=aws_config)

"""
    create_return_shipping_label(job_id)
    create_return_shipping_label(job_id, params::Dict{String,<:Any})

Creates a shipping label that will be used to return the Snow device to AWS.

# Arguments
- `job_id`: The ID for a job that you want to create the return shipping label for. For
  example JID123e4567-e89b-12d3-a456-426655440000.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"ShippingOption"`: The shipping speed for a particular job. This speed doesn't dictate
  how soon the device is returned to AWS. This speed represents how quickly it moves to its
  destination while in transit. Regional shipping speeds are as follows:
"""
create_return_shipping_label(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateReturnShippingLabel", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
create_return_shipping_label(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("CreateReturnShippingLabel", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    describe_address(address_id)
    describe_address(address_id, params::Dict{String,<:Any})

Takes an AddressId and returns specific details about that address in the form of an
Address object.

# Arguments
- `address_id`: The automatically generated ID for a specific address.

"""
describe_address(AddressId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeAddress", Dict{String, Any}("AddressId"=>AddressId); aws_config=aws_config)
describe_address(AddressId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeAddress", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("AddressId"=>AddressId), params)); aws_config=aws_config)

"""
    describe_addresses()
    describe_addresses(params::Dict{String,<:Any})

Returns a specified number of ADDRESS objects. Calling this API in one of the US regions
will return addresses from the list of all addresses associated with this account in all US
regions.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The number of ADDRESS objects to return.
- `"NextToken"`: HTTP requests are stateless. To identify what object comes \"next\" in the
  list of ADDRESS objects, you have the option of specifying a value for NextToken as the
  starting point for your list of returned addresses.
"""
describe_addresses(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeAddresses"; aws_config=aws_config)
describe_addresses(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeAddresses", params; aws_config=aws_config)

"""
    describe_cluster(cluster_id)
    describe_cluster(cluster_id, params::Dict{String,<:Any})

Returns information about a specific cluster including shipping information, cluster
status, and other important metadata.

# Arguments
- `cluster_id`: The automatically generated ID for a cluster.

"""
describe_cluster(ClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeCluster", Dict{String, Any}("ClusterId"=>ClusterId); aws_config=aws_config)
describe_cluster(ClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeCluster", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ClusterId"=>ClusterId), params)); aws_config=aws_config)

"""
    describe_job(job_id)
    describe_job(job_id, params::Dict{String,<:Any})

Returns information about a specific job including shipping information, job status, and
other important metadata.

# Arguments
- `job_id`: The automatically generated ID for a job, for example
  JID123e4567-e89b-12d3-a456-426655440000.

"""
describe_job(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeJob", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
describe_job(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeJob", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    describe_return_shipping_label()
    describe_return_shipping_label(params::Dict{String,<:Any})

Information on the shipping label of a Snow device that is being returned to AWS.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"JobId"`: The automatically generated ID for a job, for example
  JID123e4567-e89b-12d3-a456-426655440000.
"""
describe_return_shipping_label(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeReturnShippingLabel"; aws_config=aws_config)
describe_return_shipping_label(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("DescribeReturnShippingLabel", params; aws_config=aws_config)

"""
    get_job_manifest(job_id)
    get_job_manifest(job_id, params::Dict{String,<:Any})

Returns a link to an Amazon S3 presigned URL for the manifest file associated with the
specified JobId value. You can access the manifest file for up to 60 minutes after this
request has been made. To access the manifest file after 60 minutes have passed, you'll
have to make another call to the GetJobManifest action. The manifest is an encrypted file
that you can download after your job enters the WithCustomer status. The manifest is
decrypted by using the UnlockCode code value, when you pass both values to the Snow device
through the Snowball client when the client is started for the first time. As a best
practice, we recommend that you don't save a copy of an UnlockCode value in the same
location as the manifest file for that job. Saving these separately helps prevent
unauthorized parties from gaining access to the Snow device associated with that job. The
credentials of a given job, including its manifest file and unlock code, expire 90 days
after the job is created.

# Arguments
- `job_id`: The ID for a job that you want to get the manifest file for, for example
  JID123e4567-e89b-12d3-a456-426655440000.

"""
get_job_manifest(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetJobManifest", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
get_job_manifest(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetJobManifest", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    get_job_unlock_code(job_id)
    get_job_unlock_code(job_id, params::Dict{String,<:Any})

Returns the UnlockCode code value for the specified job. A particular UnlockCode value can
be accessed for up to 90 days after the associated job has been created. The UnlockCode
value is a 29-character code with 25 alphanumeric characters and 4 hyphens. This code is
used to decrypt the manifest file when it is passed along with the manifest to the Snow
device through the Snowball client when the client is started for the first time. As a best
practice, we recommend that you don't save a copy of the UnlockCode in the same location as
the manifest file for that job. Saving these separately helps prevent unauthorized parties
from gaining access to the Snow device associated with that job.

# Arguments
- `job_id`: The ID for the job that you want to get the UnlockCode value for, for example
  JID123e4567-e89b-12d3-a456-426655440000.

"""
get_job_unlock_code(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetJobUnlockCode", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
get_job_unlock_code(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetJobUnlockCode", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    get_snowball_usage()
    get_snowball_usage(params::Dict{String,<:Any})

Returns information about the Snow Family service limit for your account, and also the
number of Snow devices your account has in use. The default service limit for the number of
Snow devices that you can have at one time is 1. If you want to increase your service
limit, contact AWS Support.

"""
get_snowball_usage(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetSnowballUsage"; aws_config=aws_config)
get_snowball_usage(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetSnowballUsage", params; aws_config=aws_config)

"""
    get_software_updates(job_id)
    get_software_updates(job_id, params::Dict{String,<:Any})

Returns an Amazon S3 presigned URL for an update file associated with a specified JobId.

# Arguments
- `job_id`: The ID for a job that you want to get the software update file for, for example
  JID123e4567-e89b-12d3-a456-426655440000.

"""
get_software_updates(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetSoftwareUpdates", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
get_software_updates(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("GetSoftwareUpdates", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    list_cluster_jobs(cluster_id)
    list_cluster_jobs(cluster_id, params::Dict{String,<:Any})

Returns an array of JobListEntry objects of the specified length. Each JobListEntry object
is for a job in the specified cluster and contains a job's state, a job's ID, and other
information.

# Arguments
- `cluster_id`: The 39-character ID for the cluster that you want to list, for example
  CID123e4567-e89b-12d3-a456-426655440000.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The number of JobListEntry objects to return.
- `"NextToken"`: HTTP requests are stateless. To identify what object comes \"next\" in the
  list of JobListEntry objects, you have the option of specifying NextToken as the starting
  point for your returned list.
"""
list_cluster_jobs(ClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListClusterJobs", Dict{String, Any}("ClusterId"=>ClusterId); aws_config=aws_config)
list_cluster_jobs(ClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListClusterJobs", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ClusterId"=>ClusterId), params)); aws_config=aws_config)

"""
    list_clusters()
    list_clusters(params::Dict{String,<:Any})

Returns an array of ClusterListEntry objects of the specified length. Each ClusterListEntry
object contains a cluster's state, a cluster's ID, and other important status information.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The number of ClusterListEntry objects to return.
- `"NextToken"`: HTTP requests are stateless. To identify what object comes \"next\" in the
  list of ClusterListEntry objects, you have the option of specifying NextToken as the
  starting point for your returned list.
"""
list_clusters(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListClusters"; aws_config=aws_config)
list_clusters(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListClusters", params; aws_config=aws_config)

"""
    list_compatible_images()
    list_compatible_images(params::Dict{String,<:Any})

This action returns a list of the different Amazon EC2 Amazon Machine Images (AMIs) that
are owned by your AWS account that would be supported for use on a Snow device. Currently,
supported AMIs are based on the CentOS 7 (x86_64) - with Updates HVM, Ubuntu Server 14.04
LTS (HVM), and Ubuntu 16.04 LTS - Xenial (HVM) images, available on the AWS Marketplace.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The maximum number of results for the list of compatible images.
  Currently, a Snowball Edge device can store 10 AMIs.
- `"NextToken"`: HTTP requests are stateless. To identify what object comes \"next\" in the
  list of compatible images, you can specify a value for NextToken as the starting point for
  your list of returned images.
"""
list_compatible_images(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListCompatibleImages"; aws_config=aws_config)
list_compatible_images(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListCompatibleImages", params; aws_config=aws_config)

"""
    list_jobs()
    list_jobs(params::Dict{String,<:Any})

Returns an array of JobListEntry objects of the specified length. Each JobListEntry object
contains a job's state, a job's ID, and a value that indicates whether the job is a job
part, in the case of export jobs. Calling this API action in one of the US regions will
return jobs from the list of all jobs associated with this account in all US regions.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"MaxResults"`: The number of JobListEntry objects to return.
- `"NextToken"`: HTTP requests are stateless. To identify what object comes \"next\" in the
  list of JobListEntry objects, you have the option of specifying NextToken as the starting
  point for your returned list.
"""
list_jobs(; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListJobs"; aws_config=aws_config)
list_jobs(params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("ListJobs", params; aws_config=aws_config)

"""
    update_cluster(cluster_id)
    update_cluster(cluster_id, params::Dict{String,<:Any})

While a cluster's ClusterState value is in the AwaitingQuorum state, you can update some of
the information associated with a cluster. Once the cluster changes to a different job
state, usually 60 minutes after the cluster being created, this action is no longer
available.

# Arguments
- `cluster_id`: The cluster ID of the cluster that you want to update, for example
  CID123e4567-e89b-12d3-a456-426655440000.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AddressId"`: The ID of the updated Address object.
- `"Description"`: The updated description of this cluster.
- `"ForwardingAddressId"`: The updated ID for the forwarding address for a cluster. This
  field is not supported in most regions.
- `"Notification"`: The new or updated Notification object.
- `"Resources"`: The updated arrays of JobResource objects that can include updated
  S3Resource objects or LambdaResource objects.
- `"RoleARN"`: The new role Amazon Resource Name (ARN) that you want to associate with this
  cluster. To create a role ARN, use the CreateRole API action in AWS Identity and Access
  Management (IAM).
- `"ShippingOption"`: The updated shipping option value of this cluster's ShippingDetails
  object.
"""
update_cluster(ClusterId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateCluster", Dict{String, Any}("ClusterId"=>ClusterId); aws_config=aws_config)
update_cluster(ClusterId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateCluster", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("ClusterId"=>ClusterId), params)); aws_config=aws_config)

"""
    update_job(job_id)
    update_job(job_id, params::Dict{String,<:Any})

While a job's JobState value is New, you can update some of the information associated with
a job. Once the job changes to a different job state, usually within 60 minutes of the job
being created, this action is no longer available.

# Arguments
- `job_id`: The job ID of the job that you want to update, for example
  JID123e4567-e89b-12d3-a456-426655440000.

# Optional Parameters
Optional parameters can be passed as a `params::Dict{String,<:Any}`. Valid keys are:
- `"AddressId"`: The ID of the updated Address object.
- `"Description"`: The updated description of this job's JobMetadata object.
- `"ForwardingAddressId"`: The updated ID for the forwarding address for a job. This field
  is not supported in most regions.
- `"Notification"`: The new or updated Notification object.
- `"Resources"`: The updated JobResource object, or the updated JobResource object.
- `"RoleARN"`: The new role Amazon Resource Name (ARN) that you want to associate with this
  job. To create a role ARN, use the CreateRoleAWS Identity and Access Management (IAM) API
  action.
- `"ShippingOption"`: The updated shipping option value of this job's ShippingDetails
  object.
- `"SnowballCapacityPreference"`: The updated SnowballCapacityPreference of this job's
  JobMetadata object. The 50 TB Snowballs are only available in the US regions.
"""
update_job(JobId; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateJob", Dict{String, Any}("JobId"=>JobId); aws_config=aws_config)
update_job(JobId, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateJob", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId), params)); aws_config=aws_config)

"""
    update_job_shipment_state(job_id, shipment_state)
    update_job_shipment_state(job_id, shipment_state, params::Dict{String,<:Any})

Updates the state when a the shipment states changes to a different state.

# Arguments
- `job_id`: The job ID of the job whose shipment date you want to update, for example
  JID123e4567-e89b-12d3-a456-426655440000.
- `shipment_state`: The state of a device when it is being shipped.  Set to RECEIVED when
  the device arrives at your location. Set to RETURNED when you have returned the device to
  AWS.

"""
update_job_shipment_state(JobId, ShipmentState; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateJobShipmentState", Dict{String, Any}("JobId"=>JobId, "ShipmentState"=>ShipmentState); aws_config=aws_config)
update_job_shipment_state(JobId, ShipmentState, params::AbstractDict{String, <:Any}; aws_config::AbstractAWSConfig=global_aws_config()) = snowball("UpdateJobShipmentState", Dict{String, Any}(mergewith(_merge, Dict{String, Any}("JobId"=>JobId, "ShipmentState"=>ShipmentState), params)); aws_config=aws_config)


text:
Base.show(io::IO, keyctx::KeyContext) = print(io, "num_lock=", keyctx.numlock, ", caps=", keyctx.caps)

KeyCombination(key::KeySymbol, modifiers::KeyModifierState) = KeyCombination(key.symbol, modifiers)

function KeyCombination(key_combination::AbstractString)
    els = split(lowercase(key_combination), "+")
    char_part = last(els)
    fkey = match(r"f\d+$", char_part)
    if !isnothing(fkey)
        char_part = fkey.match
    else
        @assert length(char_part) == 1 "Character part $char_part of $key_combination must be a single character"
    end
    modifiers = els[1:end-1]
    KeyCombination(char_part, KeyModifierState(; map(x -> (Symbol(x), true), modifiers)...))
end

const fkeys = Dict(
    '\uffbe' => "f1",
    '\uffbf' => "f2",
    '\uffc0' => "f3",
    '\uffc1' => "f4",
    '\uffc2' => "f5",
    '\uffc3' => "f6",
    '\uffc4' => "f7",
    '\uffc5' => "f8",
    '\uffc6' => "f9",
    '\uffc7' => "f10",
    '\uffc8' => "f11",
    '\uffc9' => "f12",
)

function Base.show(io::IO, key::KeyCombination)
    states = String[]
    for field ∈ reverse(fieldnames(KeyModifierState))
        getproperty(key.state, field) ? push!(states, string(field)) : nothing
    end
    push!(states, key.key)
    print(io, join(states, '+'))
end

function Base.show(io::IO, button::MouseState)
    states = String[]
    for field ∈ fieldnames(MouseState)
        push!(states, string(field, '=', getproperty(button, field)))
    end
    print(io, join(states, ", "))
end

"""
Construct a KeyCombination instance from a string as `"[<state_1>+[...<state_n>+]]<key>"`.
The string must be a list of elements separated by '+' characters, with only one non-state character or one fkey as string. For example, `"k"`, `"alt+k"` and `"ctrl+alt+shift+k"` are valid strings, but `"k+a"` is not. Casing is ignored. Fkeys are simply f<1-12> keys, for example `key"alt+f4"` is valid.
"""
macro key_str(expr) esc(:(KeyCombination($(Meta.parse("\"$(escape_string(expr))\""))))) end

Base.Dict(event::MouseState) = Dict{Symbol, Bool}((k => getproperty(event, k)) for k ∈ fieldnames(MouseState))

"""
Retrieve a list of pressed button symbols for the left, middle and right buttons from a [`MouseState`](@ref).
"""
function pressed_buttons(input::MouseState)
    inputs = filter(x -> x.second && x.first ∈ (:left, :middle, :right), Dict(input))
    pressed_buttons = Set(keys(inputs))
end

const no_input = '\0'


text:
@testset "Test Simulation Models" begin
    models = SimulationModels(
        [
            DecisionModel(
                MockOperationProblem;
                horizon=48,
                interval=Hour(24),
                steps=2,
                name="DAUC",
            ),
            DecisionModel(
                MockOperationProblem;
                horizon=24,
                interval=Hour(1),
                steps=2 * 24,
                name="HAUC",
            ),
            DecisionModel(
                MockOperationProblem;
                horizon=12,
                interval=Minute(5),
                steps=2 * 24 * 12,
                name="ED",
            ),
        ],
        EmulationModel(MockEmulationProblem; resolution=Minute(1), name="AGC"),
    )

    @test length(PSI.get_decision_models(models)) == 3
    @test PSI.get_emulation_model(models) !== nothing

    @test_throws ErrorException SimulationModels(
        [
            DecisionModel(
                MockOperationProblem;
                horizon=48,
                interval=Hour(24),
                steps=2,
                name="DAUC",
            ),
            DecisionModel(
                MockOperationProblem;
                horizon=24,
                interval=Hour(1),
                steps=2 * 24,
                name="DAUC",
            ),
            DecisionModel(
                MockOperationProblem;
                horizon=12,
                interval=Minute(5),
                steps=2 * 24 * 12,
                name="ED",
            ),
        ],
        EmulationModel(MockEmulationProblem; resolution=Minute(1), name="AGC"),
    )
end


text:
"""

Compute median, mad_sd, mean and std summary of distribition.

$(SIGNATURES)

## Positional arguments
```julia
* `model_df::DataFrame` # DataFrame holding the draws
* `pars::Vector{Symbol}` # Vector of symbols to be included
```

## Keyword arguments
```julia
* `digits = 2` # Number of decimal digits
```

"""
function model_summary(model_df::DataFrame,
    params::T = Symbol.(names(model_df));
    digits = 2) where {T}

    colnames = Symbol.(names(model_df))

    pars = Symbol[]
    for par in Symbol.(params)
        if par in colnames
            append!(pars, [par])
        else
            @warn ":$(par) not in $(colnames), will be dropped."
        end
    end

    if length(pars) > 0
        parameters = Pair{Symbol, Int}[]
        estimates = zeros(length(pars), 4)
        for (indx, par) in enumerate(pars)
            if par in colnames
                append!(parameters, [par => indx])
                vals = model_df[:, par]
                estimates[indx, :] = [median(vals), mad(vals), mean(vals), std(vals)]
            end
        end

        return NamedArray(
            round.(estimates; digits=digits),
            (OrderedDict(parameters...),
            OrderedDict(:median=>1, :mad_sd=>2, :mean=>3, :std=>4)),
                   ("Parameter", "Value")
        )
    else
        @warn "No parameters match the column names in model_df."
        return nothing
    end
end

export
    model_summary


text:
# Linear algebra functions

const σ0 = TensorMap(Matrix{ComplexF64}([1 0; 0 1]), ℂ^2, ℂ^2) # Identity matrix
const σ1 = TensorMap(Matrix{ComplexF64}([0 1; 1 0]), ℂ^2, ℂ^2) # Pauli X matrix
const σ2 = TensorMap(Matrix{ComplexF64}([0 -im; im 0]), ℂ^2, ℂ^2) # Pauli Y matrix
const σ3 = TensorMap(Matrix{ComplexF64}([1 0; 0 -1]), ℂ^2, ℂ^2) # Pauli Z matrix

const σp = 1 / 2 * (σ1 + im * σ2) # Pauli+ matrix
const σm = 1 / 2 * (σ1 - im * σ2) # Pauli- matrix

function LinearAlgebra.tr(t::PEPOTensor{S}) where {S<:IndexSpace}
    t = t.tensor
    @tensor tTr[1 2; 3 4] := t[1 2 x; 3 4 x]
end

function LinearAlgebra.ishermitian(p::PEPOTensor; phys_only=false)
    t = p.tensor
    phys_only || return LinearAlgebra.ishermitian(t)
    tT = permute(t, (1, 2, 6), (4, 5, 3))
    for (key, block) in blocks(t)
        block ≈ conj(blocks(tT)[key]) || return false
    end
    return true
end

function LinearAlgebra.vec(p::PEPOTensor)
    t = p.tensor
    return VectorPEPOTensor(_vec(t)...)
end

function _vec(t::TensorMap{S,3,3}) where {S<:EuclideanSpace}
    dom = TensorKit.domain(t)
    in = dom[3] # Physical index corresponding to the input
    cod = TensorKit.codomain(t)
    out = cod[3] # Physical index corresponding to the output
    tt = permute(t, (1, 2, 3, 6), (4, 5))
    u = unitary(fuse(out ⊗ in), out ⊗ flip(in))
    @tensor ttt[1 2 ph; 3 4] := u[ph; x1 x2] * tt[1 2 x1 x2; 3 4]
    return ttt, u
end

function unvec(p::VectorPEPOTensor)
    t = p.tensor
    u = p.unitary
    iu = inv(u)
    @tensor tt[1 2 x1 x2; 3 4] := iu[x1 x2; ph] * t[1 2 ph; 3 4]
    ttt = permute(tt, (1, 2, 3), (5, 6, 4))
    return PEPOTensor(ttt)
end

function conjugate(t::AbstractTensorMap)
    return TensorMap(conj(t.data), codomain(t), domain(t))
end

const cj = conjugate

@doc raw"""
    trace(t::PEPOTensor{S}, bond::S) where {S<:IndexSpace}

"""
function trace(t::PEPOTensor{S}, bond::S) where {S<:IndexSpace}
    @tensor pt[w s; n e] := t[w s x; n e x]
    guess = InfiniteMPS([codomain(pt)[2]], [bond])
    ts, _, _ = leading_boundary(guess, InfiniteMPO(pt), Vumps(; maxiter=400, verbose=true))
    return expectation_value(ts, pt, 1), ts
end

trace(t::PEPOTensor{ComplexSpace}, χ::Integer) = trace(t, ℂ^χ)

max_stars_repo_name: jack-dunham/PEPOKit

max_issues_repo_name: jack-dunham/PEPOKit

max_forks_repo_name: jack-dunham/PEPOKit


text:
using BinanceBot, Binance

client = Client()
cap_list = read_market_cap_list("cap100m.csv")
prices = read_price_data("prices.csv")
v = sorted_roundtrip(client, cap_list, prices)

#implement
#* trades
#* depth analysis
#* streaming


text:
module NGram

using TextAnalysis

include("ngram.jl")

export NGramModel

end # module


text:
@testset "Test sampling." begin
    samp = sample(TAMode.A549model, HMC(0.001, 10), 4)
end


text:
@testset "Fourier Shear" begin

    @testset "Complex and real shear produce similar results" begin
        function f(a, b, Δ)
            x = randn((30, 24, 13))
            xc = 0im .+ x
            xc2 = 1im  .* x
            @test shear(x, Δ, a, b) ≈ real(shear(xc, Δ, a, b))
            @test shear(x, Δ, a, b) ≈ imag(shear(xc2, Δ, a, b))
        end

        f(2, 3, 123.1)
        f(3, 2, 13.1)
        f(1, 2, 13.1)
        f(3, 1, 13.1)
    end

    @testset "Test that in-place works in-place" begin
        function f(a, b, Δ)
            x = randn((30, 24, 13))
            xc = randn(ComplexF32, (30, 24, 13))
            xc2 = 1im  .* x
            @test shear!(x, Δ, a, b) ≈ x
            @test shear!(xc, Δ, a, b) ≈ xc
            @test shear!(xc2, Δ, a, b) ≈ xc2
        end

        f(2, 3, 123.1)
        f(3, 2, 13.1)
        f(1, 2, 13.1)
        f(3, 1, 13.1)
    end

    @testset "Fix Nyquist" begin
        @test shear(shear([1 2; 3 4.0], 0.123), -0.123, fix_nyquist = true) == [1.0 2.0; 3.0 4.0]
        @test shear(shear([1 2; 3 4.0], 0.123), -0.123, fix_nyquist = false) != [1.0 2.0; 3.0 4.0]

    end

end


text:
using GLFW

includet("src/input.jl")

function demo_xbox()

    window = GLFW.CreateWindow(640, 480, "GLFW Callback Test")
    GLFW.MakeContextCurrent(window)

    # Input callbacks
    GLFW.SetJoystickCallback(joystick_callback)
    GLFW.SetKeyCallback(window, (_, key, scancode, action, mods) -> begin
        name = GLFW.GetKeyName(key, scancode)
        if name === nothing
            println("scancode $scancode ", action)
        else
            println("key $name ", action)
        end
    end)

    #add any joysticks connected before GLFW was imported
    init_joysticks()
    #add any joysticks connected since GLFW was imported
    GLFW.PollEvents()

    t_start = 0
    t_end = 10
    dt = 1

    t_wall = time()

    t = t_start
    while t < t_end

        #this executes the callbacks for all the events received since the last call
        #to PollEvents
        GLFW.PollEvents()
        #update joystick inputs for callbacks
        # cursor_pos = GLFW.GetCursorPos(window) #placeholder for joystick reads
        for joystick in values(connected_joysticks)
            update(joystick)
            println(interface.axes.data)
        end
        # println(cursor_pos)

        #take a simulation step
        #step!()

        #update simulation time
        t += dt

        #update wall time
        t_wall_next = t_wall + dt
        t_wall = t_wall_next

        # Swap front and back buffers
        GLFW.SwapBuffers(window)

        #busy wait until the end of this frame
        while (time() < t_wall) & !GLFW.WindowShouldClose(window)

            #nothing to do here. to increase responsiveness we could take the chance
            #to keep polling asynchronous events, but this increases processor load
            #(why? this is already a busy wait)
            # GLFW.PollEvents()

        end
        # println(time()-t_wall)

    end

    GLFW.DestroyWindow(window);

end


text:
using InvariantMeasures
using ValidatedNumerics

using Plots
using LaTeXStrings
using StatsPlots

"""
Simple and quick experiment to precompile all needed functions before timing measurements
"""

D = Iterate(mod1_dynamic(x->2*x+0.5*x*(1-x)), 4)
B = Ulam(8)
Q = DiscretizedOperator(B, D)

norms = powernormbounds(B, D; Q=Q)

B_fine = Ulam(64)
Q_fine = DiscretizedOperator(B_fine, D)

norms_fine = finepowernormbounds(B, B_fine, D, norms; Q_fine=Q_fine)
w_fine = invariant_vector(B_fine, Q_fine)
error_fine = distance_from_invariant(B_fine, D, Q_fine, w_fine, norms_fine)

D = Mod1Dynamic(x -> 4*x + 0.01*InvariantMeasures.sinpi(8*x))
B = Hat(64)
Q = DiscretizedOperator(B, D)

norms = powernormbounds(B, D; Q=Q)

B_fine = Hat(128)
Q_fine = DiscretizedOperator(B_fine, D)

norms_fine = finepowernormbounds(B, B_fine, D, norms; Q_fine=Q_fine)
w_fine = invariant_vector(B_fine, Q_fine)
error_fine = distance_from_invariant(B_fine, D, Q_fine, w_fine, norms_fine)


text:
module LocalitySensitive
    include("minhash.jl")
    include("simhash.jl")

    export MinHash, push!, find_similar, similar_pairs, fingerprint, estimate_jaccard, LSHIndex, shingle, fingerprint_all
    export SimHash, SimHashIndex, estimate_cosine
end # module