text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

module Entry

import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version
import OldPkg
import ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..Dir
using LibGit2
import ..PkgError
using ..Types
using Base.Printf: @printf

macro recover(ex)
    quote
        try $(esc(ex))
        catch err
            show(err)
            print('\n')
        end
    end
end

function edit(f::Function, pkg::AbstractString, args...)
    r = Reqs.read("REQUIRE")
    reqs = Reqs.parse(r)
    avail = Read.available()
    !haskey(avail,pkg) && !haskey(reqs,pkg) && return false
    rʹ = f(r,pkg,args...)
    rʹ == r && return false
    reqsʹ = Reqs.parse(rʹ)
    reqsʹ != reqs && resolve(reqsʹ,avail)
    Reqs.write("REQUIRE",rʹ)
    @info "Package database updated"
    return true
end

function edit()
    editor = get(ENV,"VISUAL",get(ENV,"EDITOR",nothing))
    editor !== nothing ||
        throw(PkgError("set the EDITOR environment variable to an edit command"))
    editor = Base.shell_split(editor)
    reqs = Reqs.parse("REQUIRE")
    run(`$editor REQUIRE`)
    reqsʹ = Reqs.parse("REQUIRE")
    reqs == reqsʹ && return @info "Nothing to be done"
    @info "Computing changes..."
    resolve(reqsʹ)
end

function add(pkg::AbstractString, vers::VersionSet)
    outdated = :maybe
    @sync begin
        @async if !edit(Reqs.add,pkg,vers)
            ispath(pkg) || throw(PkgError("unknown package $pkg"))
            @info "Package $pkg is already installed"
        end
        branch = Dir.getmetabranch()
        outdated = with(GitRepo, "METADATA") do repo
            if LibGit2.branch(repo) == branch
                if LibGit2.isdiff(repo, "origin/$branch")
                    outdated = :yes
                else
                    try
                        LibGit2.fetch(repo)
                        outdated = LibGit2.isdiff(repo, "origin/$branch") ? (:yes) : (:no)
                    catch
                    end
                end
            else
                :no # user is doing something funky with METADATA
            end
        end
    end
    if outdated != :no
        is = outdated == :yes ? "is" : "might be"
        @info """
            METADATA $is out-of-date — you may not have the latest version of $pkg
            Use `OldPkg.update()` to get the latest versions of your packages
            """
    end
end
add(pkg::AbstractString, vers::VersionNumber...) = add(pkg,VersionSet(vers...))

function rm(pkg::AbstractString)
    edit(Reqs.rm,pkg) && return
    ispath(pkg) || return @info "Package $pkg is not installed"
    @info "Removing $pkg (unregistered)"
    Write.remove(pkg)
end

function available()
    all_avail = Read.available()
    avail = AbstractString[]
    for (pkg, vers) in all_avail
        any(x->Types.satisfies("julia", VERSION, x[2].requires), vers) && push!(avail, pkg)
    end
    sort!(avail, by=lowercase)
end

function available(pkg::AbstractString)
    avail = Read.available(pkg)
    if !isempty(avail) || Read.isinstalled(pkg)
        return sort!(collect(keys(avail)))
    end
    throw(PkgError("$pkg is not a package (not registered or installed)"))
end

function installed()
    pkgs = Dict{String,VersionNumber}()
    for (pkg,(ver,fix)) in Read.installed()
        pkgs[pkg] = ver
    end
    return pkgs
end

function installed(pkg::AbstractString)
    avail = Read.available(pkg)
    if Read.isinstalled(pkg)
        res = typemin(VersionNumber)
        if ispath(joinpath(pkg,".git"))
            LibGit2.with(GitRepo, pkg) do repo
                res = Read.installed_version(pkg, repo, avail)
            end
        end
        return res
    end
    isempty(avail) && throw(PkgError("$pkg is not a package (not registered or installed)"))
    return nothing # registered but not installed
end

function status(io::IO; pkgname::AbstractString = "")
    if !isempty(pkgname) && !ispath(pkgname)
        throw(PkgError("Package $pkgname does not exist"))
    end
    showpkg(pkg) = isempty(pkgname) ? true : (pkg == pkgname)
    reqs = Reqs.parse("REQUIRE")
    instd = Read.installed()
    required = sort!(collect(keys(reqs)))
    if !isempty(required)
        showpkg("") && println(io, "$(length(required)) required packages:")
        for pkg in required
            if !haskey(instd, pkg)
                showpkg(pkg) && status(io,pkg,"not found")
            else
                ver,fix = pop!(instd,pkg)
                showpkg(pkg) && status(io,pkg,ver,fix)
            end
        end
    end
    additional = sort!(collect(keys(instd)))
    if !isempty(additional)
        showpkg("") && println(io, "$(length(additional)) additional packages:")
        for pkg in additional
            ver,fix = instd[pkg]
            showpkg(pkg) && status(io,pkg,ver,fix)
        end
    end
    if isempty(required) && isempty(additional)
        println(io, "No packages installed")
    end
end

status(io::IO, pkg::AbstractString) = status(io, pkgname = pkg)

function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool)
    if !isempty(pkg) && !ispath(pkg)
        throw(PkgError("Package $pkg does not exist"))
    end
    @printf io " - %-29s " pkg
    fix || return println(io,ver)
    @printf io "%-19s" ver
    if ispath(pkg,".git")
        prepo = GitRepo(pkg)
        try
            with(LibGit2.head(prepo)) do phead
                if LibGit2.isattached(prepo)
                    print(io, LibGit2.shortname(phead))
                else
                    print(io, string(LibGit2.GitHash(phead))[1:8])
                end
            end
            attrs = AbstractString[]
            isfile("METADATA",pkg,"url") || push!(attrs,"unregistered")
            LibGit2.isdirty(prepo) && push!(attrs,"dirty")
            isempty(attrs) || print(io, " (",join(attrs,", "),")")
        catch err
            printstyled(io, " broken-repo (unregistered)", color=Base.error_color())
        finally
            close(prepo)
        end
    else
        printstyled(io, "non-repo (unregistered)", color=Base.warn_color())
    end
    println(io)
end

function status(io::IO, pkg::AbstractString, msg::AbstractString)
    @printf io " - %-29s %-19s\n" pkg msg
end

function clone(url::AbstractString, pkg::AbstractString)
    @info "Cloning $pkg from $url"
    ispath(pkg) && throw(PkgError("$pkg already exists"))
    try
        LibGit2.with(LibGit2.clone(url, pkg)) do repo
            LibGit2.set_remote_url(repo, "origin", url)
        end
    catch err
        isdir(pkg) && Base.rm(pkg, recursive=true)
        rethrow(err)
    end
    @info "Computing changes..."
    if !edit(Reqs.add, pkg)
        isempty(Reqs.parse("$pkg/REQUIRE")) && return
        resolve()
    end
end

function url_and_pkg(url_or_pkg::AbstractString)
    if !(':' in url_or_pkg)
        # no colon, could be a package name
        url_file = joinpath("METADATA", url_or_pkg, "url")
        isfile(url_file) && return readchomp(url_file), url_or_pkg
    end
    # try to parse as URL or local path
    m = match(r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?$", url_or_pkg)
    m === nothing && throw(PkgError("can't determine package name from URL: $url_or_pkg"))
    return url_or_pkg, m.captures[1]
end

clone(url_or_pkg::AbstractString) = clone(url_and_pkg(url_or_pkg)...)

function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, do_pull::Bool)
    ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo"))
    @info "Checking out $pkg $branch..."
    with(GitRepo, pkg) do r
        LibGit2.transact(r) do repo
            LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing"))
            LibGit2.branch!(repo, branch, track=LibGit2.Consts.REMOTE_ORIGIN)
            do_merge && LibGit2.merge!(repo, fastforward=true) # merge changes
            if do_pull
                @info "Pulling $pkg latest $branch..."
                LibGit2.fetch(repo)
                LibGit2.merge!(repo, fastforward=true)
            end
            resolve()
        end
    end
end

function free(pkg::AbstractString)
    ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo"))
    Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be freed – not an installed package"))
    avail = Read.available(pkg)
    isempty(avail) && throw(PkgError("$pkg cannot be freed – not a registered package"))
    with(GitRepo, pkg) do repo
        LibGit2.isdirty(repo) && throw(PkgError("$pkg cannot be freed – repo is dirty"))
        @info "Freeing $pkg"
        vers = sort!(collect(keys(avail)), rev=true)
        while true
            for ver in vers
                sha1 = avail[ver].sha1
                LibGit2.iscommit(sha1, repo) || continue
                return LibGit2.transact(repo) do r
                    LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing"))
                    LibGit2.checkout!(repo, sha1)
                    resolve()
                end
            end
            isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue
            throw(PkgError("can't find any registered versions of $pkg to checkout"))
        end
    end
end

function free(pkgs)
    try
        for pkg in pkgs
            ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo"))
            Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be freed – not an installed package"))
            avail = Read.available(pkg)
            isempty(avail) && throw(PkgError("$pkg cannot be freed – not a registered package"))
            with(GitRepo, pkg) do repo
                LibGit2.isdirty(repo) && throw(PkgError("$pkg cannot be freed – repo is dirty"))
                @info "Freeing $pkg"
                vers = sort!(collect(keys(avail)), rev=true)
                for ver in vers
                    sha1 = avail[ver].sha1
                    LibGit2.iscommit(sha1, repo) || continue
                    LibGit2.checkout!(repo, sha1)
                    break
                end
            end
            isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue
            throw(PkgError("Can't find any registered versions of $pkg to checkout"))
        end
    finally
        resolve()
    end
end

function pin(pkg::AbstractString, head::AbstractString)
    ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo"))
    should_resolve = true
    with(GitRepo, pkg) do repo
        id = if isempty(head) # get HEAD commit
            # no need to resolve, branch will be from HEAD
            should_resolve = false
            LibGit2.head_oid(repo)
        else
            LibGit2.revparseid(repo, head)
        end
        commit = LibGit2.GitCommit(repo, id)
        try
            # note: changing the following naming scheme requires a corresponding change in Read.ispinned()
            branch = "pinned.$(string(id)[1:8]).tmp"
            if LibGit2.isattached(repo) && LibGit2.branch(repo) == branch
                @info "Package $pkg is already pinned" * (isempty(head) ? "" : " to the selected commit")
                should_resolve = false
                return
            end
            ref = LibGit2.lookup_branch(repo, branch)
            try
                if ref !== nothing
                    if LibGit2.revparseid(repo, branch) != id
                        throw(PkgError("Package $pkg: existing branch $branch has " *
                            "been edited and doesn't correspond to its original commit"))
                    end
                    @info "Package $pkg: checking out existing branch $branch"
                else
                    @info "Creating $pkg branch $branch"
                    ref = LibGit2.create_branch(repo, branch, commit)
                end

                # checkout selected branch
                with(LibGit2.peel(LibGit2.GitTree, ref)) do btree
                    LibGit2.checkout_tree(repo, btree)
                end
                # switch head to the branch
                LibGit2.head!(repo, ref)
            finally
                close(ref)
            end
        finally
            close(commit)
        end
    end
    should_resolve && resolve()
    nothing
end
pin(pkg::AbstractString) = pin(pkg, "")

function pin(pkg::AbstractString, ver::VersionNumber)
    ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo"))
    Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be pinned – not an installed package"))
    avail = Read.available(pkg)
    isempty(avail) && throw(PkgError("$pkg cannot be pinned – not a registered package"))
    haskey(avail,ver) || throw(PkgError("$pkg – $ver is not a registered version"))
    pin(pkg, avail[ver].sha1)
end

function update(branch::AbstractString, upkgs::Set{String})
    @info "Updating METADATA..."
    with(GitRepo, "METADATA") do repo
        try
            with(LibGit2.head(repo)) do h
                if LibGit2.branch(h) != branch
                    if LibGit2.isdirty(repo)
                        throw(PkgError("METADATA is dirty and not on $branch, bailing"))
                    end
                    if !LibGit2.isattached(repo)
                        throw(PkgError("METADATA is detached not on $branch, bailing"))
                    end
                    LibGit2.fetch(repo)
                    LibGit2.checkout_head(repo)
                    LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch")
                    LibGit2.merge!(repo)
                end
            end

            LibGit2.fetch(repo)
            ff_succeeded = LibGit2.merge!(repo, fastforward=true)
            if !ff_succeeded
                LibGit2.rebase!(repo, "origin/$branch")
            end
        catch err
            cex = CapturedException(err, catch_backtrace())
            throw(PkgError("METADATA cannot be updated. Resolve problems manually in " *
                OldPkg.dir("METADATA") * ".", cex))
        end
    end
    deferred_errors = CompositeException()
    avail = Read.available()
    # this has to happen before computing free/fixed
    for pkg in filter(Read.isinstalled, collect(keys(avail)))
        try
            Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]])
        catch err
            cex = CapturedException(err, catch_backtrace())
            push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex))
        end
    end
    instd = Read.installed(avail)
    reqs = Reqs.parse("REQUIRE")
    if !isempty(upkgs)
        for (pkg, (v,f)) in instd
            satisfies(pkg, v, reqs) || throw(PkgError("Package $pkg: current " *
                "package status does not satisfy the requirements, cannot do " *
                "a partial update; use `OldPkg.update()`"))
        end
    end
    dont_update = Query.partial_update_mask(instd, avail, upkgs)
    free  = Read.free(instd,dont_update)
    for (pkg,ver) in free
        try
            Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]])
        catch err
            cex = CapturedException(err, catch_backtrace())
            push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex))
        end
    end
    fixed = Read.fixed(avail,instd,dont_update)
    Base.shred!(LibGit2.CachedCredentials()) do creds
        stopupdate = false
        for (pkg,ver) in fixed
            ispath(pkg,".git") || continue
            pkg in dont_update && continue
            with(GitRepo, pkg) do repo
                if LibGit2.isattached(repo)
                    if LibGit2.isdirty(repo)
                        @warn "Package $pkg: skipping update (dirty)..."
                    elseif Read.ispinned(repo)
                        @info "Package $pkg: skipping update (pinned)..."
                    else
                        prev_sha = string(LibGit2.head_oid(repo))
                        success = true
                        try
                            LibGit2.fetch(repo, credentials=creds)
                            LibGit2.merge!(repo, fastforward=true)
                        catch err
                            cex = CapturedException(err, catch_backtrace())
                            push!(deferred_errors, PkgError("Package $pkg cannot be updated.", cex))
                            success = false
                            stopupdate = isa(err, InterruptException)
                        end
                        if success
                            post_sha = string(LibGit2.head_oid(repo))
                            branch = LibGit2.branch(repo)
                            @info "Updating $pkg $branch..." * (prev_sha != post_sha ?
                                  " $(prev_sha[1:8]) → $(post_sha[1:8])" : "")
                        end
                    end
                end
            end
            stopupdate && break
            if haskey(avail,pkg)
                try
                    Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]])
                catch err
                    cex = CapturedException(err, catch_backtrace())
                    push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex))
                end
            end
        end
    end
    @info "Computing changes..."
    resolve(reqs, avail, instd, fixed, free, upkgs)
    # Don't use instd here since it may have changed
    updatehook(sort!(collect(keys(installed()))))

    # Print deferred errors
    length(deferred_errors) > 0 && throw(PkgError("Update finished with errors.", deferred_errors))
    nothing
end

function resolve(
    reqs  :: Dict = Reqs.parse("REQUIRE"),
    avail :: Dict = Read.available(),
    instd :: Dict = Read.installed(avail),
    fixed :: Dict = Read.fixed(avail, instd),
    have  :: Dict = Read.free(instd),
    upkgs :: Set{String} = Set{String}()
)
    bktrc = Query.init_resolve_backtrace(reqs, fixed)
    orig_reqs = deepcopy(reqs)
    Query.check_fixed(reqs, fixed, avail)
    Query.propagate_fixed!(reqs, bktrc, fixed)
    deps, conflicts = Query.dependencies(avail, fixed)

    for pkg in keys(reqs)
        if !haskey(deps,pkg)
            if "julia" in conflicts[pkg]
                throw(PkgError("$pkg can't be installed because it has no versions that support $VERSION " *
                   "of julia. You may need to update METADATA by running `OldPkg.update()`"))
            else
                sconflicts = join(conflicts[pkg], ", ", " and ")
                throw(PkgError("$pkg's requirements can't be satisfied because " *
                    "of the following fixed packages: $sconflicts"))
            end
        end
    end

    Query.check_requirements(reqs, deps, fixed)

    deps = Query.prune_dependencies(reqs, deps, bktrc)
    want = Resolve.resolve(reqs, deps)

    if !isempty(upkgs)
        orig_deps, _ = Query.dependencies(avail)
        Query.check_partial_updates(orig_reqs, orig_deps, want, fixed, upkgs)
    end

    # compare what is installed with what should be
    changes = Query.diff(have, want, avail, fixed)
    isempty(changes) && return @info "No packages to install, update or remove"

    # prefetch phase isolates network activity, nothing to roll back
    missing = []
    for (pkg,(ver1,ver2)) in changes
        vers = String[]
        ver1 !== nothing && push!(vers,LibGit2.head(pkg))
        ver2 !== nothing && push!(vers,Read.sha1(pkg,ver2))
        append!(missing,
            map(sha1->(pkg,(ver1,ver2),sha1),
                Cache.prefetch(pkg, Read.url(pkg), vers)))
    end
    if !isempty(missing)
        msg = "Missing package versions (possible metadata misconfiguration):"
        for (pkg,ver,sha1) in missing
            msg *= "  $pkg v$ver [$sha1[1:10]]\n"
        end
        throw(PkgError(msg))
    end

    # try applying changes, roll back everything if anything fails
    changed = []
    imported = String[]
    try
        for (pkg,(ver1,ver2)) in changes
            if ver1 === nothing
                @info "Installing $pkg v$ver2"
                Write.install(pkg, Read.sha1(pkg,ver2))
            elseif ver2 === nothing
                @info "Removing $pkg v$ver1"
                Write.remove(pkg)
            else
                up = ver1 <= ver2 ? "Up" : "Down"
                @info "$(up)grading $pkg: v$ver1 => v$ver2"
                Write.update(pkg, Read.sha1(pkg,ver2))
                if Base.root_module_exists(Base.PkgId(pkg))
                    push!(imported, "- $pkg")
                end
            end
            push!(changed,(pkg,(ver1,ver2)))
        end
    catch err
        for (pkg,(ver1,ver2)) in reverse!(changed)
            if ver1 === nothing
                @info "Rolling back install of $pkg"
                @recover Write.remove(pkg)
            elseif ver2 === nothing
                @info "Rolling back deleted $pkg to v$ver1"
                @recover Write.install(pkg, Read.sha1(pkg,ver1))
            else
                @info "Rolling back $pkg from v$ver2 to v$ver1"
                @recover Write.update(pkg, Read.sha1(pkg,ver1))
            end
        end
        rethrow(err)
    end
    if !isempty(imported)
        @warn join(["The following packages have been updated but were already imported:",
            imported..., "Restart Julia to use the updated versions."], "\n")
    end
    # re/build all updated/installed packages
    build(map(x->x[1], filter(x -> x[2][2] !== nothing, changes)))
end

function build(pkg::AbstractString, build_file::AbstractString, errfile::AbstractString)
    # To isolate the build from the running Julia process, we execute each build.jl file in
    # a separate process. Errors are written to errfile for later reporting.
    code = """
        import OldPkg
        empty!(Base.LOAD_PATH)
        append!(Base.LOAD_PATH, $(repr(LOAD_PATH, context=:module=>nothing)))
        empty!(Base.DEPOT_PATH)
        append!(Base.DEPOT_PATH, $(repr(DEPOT_PATH)))
        empty!(Base.DL_LOAD_PATH)
        append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH)))
        open("$(escape_string(errfile))", "a") do f
            pkg, build_file = "$pkg", "$(escape_string(build_file))"
            try
                @info "Building \$pkg"
                cd(dirname(build_file)) do
                    evalfile(build_file)
                end
            catch err
                @error \"""
                    ------------------------------------------------------------
                    # Build failed for \$pkg
                    \""" exception=err,catch_backtrace()
                write(f, pkg); write(f, 0x00)
                write(f, sprint(showerror, err)); write(f, 0x00)
            end
        end
        """
    cmd = ```
        $(Base.julia_cmd()) -O0
        --color=$(Base.have_color ? "yes" : "no")
        --compiled-modules=$(Bool(Base.JLOptions().use_compiled_modules) ? "yes" : "no")
        --history-file=no
        --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
        --eval $code
        ```

    success(pipeline(cmd, stdout=stdout, stderr=stderr))
end

function build!(pkgs::Vector, seen::Set, errfile::AbstractString)
    for pkg in pkgs
        pkg == "julia" && continue
        pkg in seen ? continue : push!(seen,pkg)
        Read.isinstalled(pkg) || throw(PkgError("$pkg is not an installed package"))
        build!(Read.requires_list(pkg), seen, errfile)
        path = abspath(pkg,"deps","build.jl")
        isfile(path) || continue
        build(pkg, path, errfile) || error("Build process failed.")
    end
end

function build!(pkgs::Vector, errs::Dict, seen::Set=Set())
    mktemp() do errfile, f
        build!(pkgs, seen, errfile)
        while !eof(f)
            pkg = readuntil(f, '\0')
            err = readuntil(f, '\0')
            errs[pkg] = err
        end
    end
end

function build(pkgs::Vector)
    errs = Dict()
    build!(pkgs,errs)
    isempty(errs) && return
    @warn """
        ------------------------------------------------------------
        # Build error summary

        $(join(keys(errs),", "," and ")) had build errors.

         - packages with build errors remain installed in $(pwd())
         - build the package(s) and all dependencies with `OldPkg.build("$(join(keys(errs),"\", \""))")`
         - build a single package by running its `deps/build.jl` script
        """
end
build() = build(sort!(collect(keys(installed()))))

function updatehook!(pkgs::Vector, errs::Dict, seen::Set=Set())
    for pkg in pkgs
        pkg in seen && continue
        updatehook!(Read.requires_list(pkg),errs,push!(seen,pkg))
        path = abspath(pkg,"deps","update.jl")
        isfile(path) || continue
        @info "Running update script for $pkg"
        cd(dirname(path)) do
            try evalfile(path)
            catch err
                @error """
                    ------------------------------------------------------------
                    # Update hook failed for $pkg
                    """ exception=err,catch_backtrace()
                errs[pkg] = err
            end
        end
    end
end

function updatehook(pkgs::Vector)
    errs = Dict()
    updatehook!(pkgs,errs)
    isempty(errs) && return
    println(stderr)
    @warn """
        ------------------------------------------------------------
        # Update hook summary

        $(join(keys(errs),", "," and ")) had update errors.

        - Unrelated packages are unaffected
        - To retry, run OldPkg.update() again
        """
end

function test!(pkg::AbstractString,
               errs::Vector{AbstractString},
               nopkgs::Vector{AbstractString},
               notests::Vector{AbstractString}; coverage::Bool=false)
    reqs_path = abspath(pkg,"test","REQUIRE")
    if isfile(reqs_path)
        tests_require = Reqs.parse(reqs_path)
        if (!isempty(tests_require))
            @info "Computing test dependencies for $pkg..."
            resolve(merge(Reqs.parse("REQUIRE"), tests_require))
        end
    end
    test_path = abspath(pkg,"test","runtests.jl")
    if !isdir(pkg)
        push!(nopkgs, pkg)
    elseif !isfile(test_path)
        push!(notests, pkg)
    else
        @info "Testing $pkg"
        cd(dirname(test_path)) do
            try
                cmd = ```
                    $(Base.julia_cmd())
                    --code-coverage=$(coverage ? "user" : "none")
                    --color=$(Base.have_color ? "yes" : "no")
                    --compiled-modules=$(Bool(Base.JLOptions().use_compiled_modules) ? "yes" : "no")
                    --check-bounds=yes
                    --warn-overwrite=yes
                    --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
                    $test_path
                    ```
                run(cmd)
                @info "$pkg tests passed"
            catch err
                @error """
                    ------------------------------------------------------------
                    # Testing failed for $pkg
                    """ exception=err,catch_backtrace()
                push!(errs,pkg)
            end
        end
    end
    isfile(reqs_path) && resolve()
end

struct PkgTestError <: Exception
    msg::AbstractString
end

function Base.showerror(io::IO, ex::PkgTestError, bt; backtrace=true)
    printstyled(io, ex.msg, color=Base.error_color())
end

function test(pkgs::Vector{AbstractString}; coverage::Bool=false)
    errs = AbstractString[]
    nopkgs = AbstractString[]
    notests = AbstractString[]
    for pkg in pkgs
        test!(pkg,errs,nopkgs,notests; coverage=coverage)
    end
    if !all(isempty, (errs, nopkgs, notests))
        messages = AbstractString[]
        if !isempty(errs)
            push!(messages, "$(join(errs,", "," and ")) had test errors")
        end
        if !isempty(nopkgs)
            msg = length(nopkgs) > 1 ? " are not installed packages" :
                                       " is not an installed package"
            push!(messages, string(join(nopkgs,", ", " and "), msg))
        end
        if !isempty(notests)
            push!(messages, "$(join(notests,", "," and ")) did not provide a test/runtests.jl file")
        end
        throw(PkgTestError(join(messages, "and")))
    end
end

test(;coverage::Bool=false) = test(sort!(AbstractString[keys(installed())...]); coverage=coverage)

end # module

max_stars_repo_name: helgee/julia

max_issues_repo_name: helgee/julia

max_forks_repo_name: helgee/julia


text:
import Distributions
using MatrixCompletion
import Random
using TimerOutputs

const to = TimerOutput()

@testset "$(format("ADMM Algorithm: Small Input[Negative Binomial User Input]"))" begin
  let
    for i = 5:2:200
      @printf("[Small] Currently doing rank %d\n", i)
      Random.seed!(65536)
      io = open("./test_result/negbin/small/rank"*string(i)*".txt", "w")
      r_input  = 6
      input_size = 200
      input_rank = i
      truth_matrix = rand([(FixedRankMatrix(Distributions.NegativeBinomial(r_input, 0.8),
                                            rank = input_rank),
                            input_size,
                            input_size)])
      sample_model = provide(Sampler{BernoulliModel}(), rate = 0.8)
      input_matrix = sample_model.draw(truth_matrix)
      user_input_estimators = Dict(:NegativeBinomial=> Dict(:r=>r_input, :p=>0.8))
      manual_type_input = Array{Symbol}(undef, input_size, input_size)
      manual_type_input .= :NegativeBinomial
      # display(input_matrix)
      @timeit to "neg small"  completed_matrix, type_tracker = complete(A            = input_matrix,
                                                                        maxiter      = 200,
                                                                        ρ            = 0.3,
                                                                        use_autodiff = false,
                                                                        gd_iter      = 3,
                                                                        debug_mode   = false,
                                                                        type_assignment = manual_type_input,
                                                                        user_input_estimators = user_input_estimators,
                                                                        project_rank = input_rank,
                                                                        io           = io)
      # display(completed_matrix[1:10, 1:10])
      predicted_negative_binomial = predict(NegativeBinomial(),
                                            forward_map(NegativeBinomial(),
                                                        completed_matrix[type_tracker[:NegativeBinomial]],
                                                        r_estimate = r_input))
      truth_negative_binomial = truth_matrix[type_tracker[:NegativeBinomial]]
      summary_negative_binomial = provide(Diagnostics{Any}(),
                                          reference = truth_negative_binomial,
                                          input_data = predicted_negative_binomial)
      show(io, MIME("text/plain"), summary_negative_binomial)
      print(io, "\n")
      show(io, MIME("text/plain"), to)
      close(io)
    end
  end
end

@testset "$(format("ADMM Algorithm: Medium Input[Negative Binomial User Input]"))" begin
  let
    for i = 5:20:500
      @printf("[Medium] Currently doing rank %d\n ", i)
      Random.seed!(65536)
      io = open("./test_result/negbin/medium/rank"*string(i)*".txt", "w")
      # io = stdout
      r_input  = 6
      input_size = 2000
      input_rank = i
      truth_matrix = rand([(FixedRankMatrix(Distributions.NegativeBinomial(r_input, 0.8),
                                            rank = input_rank),
                            input_size,
                            input_size)])
      sample_model = provide(Sampler{BernoulliModel}(), rate = 0.8)
      input_matrix = sample_model.draw(truth_matrix)
      user_input_estimators = Dict(:NegativeBinomial=> Dict(:r=>r_input, :p=>0.8))
      manual_type_input = Array{Symbol}(undef, input_size, input_size)
      manual_type_input .= :NegativeBinomial
      # display(input_matrix)
      @timeit to "neg medium"  completed_matrix, type_tracker = complete(A            = input_matrix,
                                                                         maxiter      = 200,
                                                                         ρ            = 0.3,
                                                                         use_autodiff = true,
                                                                         gd_iter      = 3,
                                                                         debug_mode   = false,
                                                                         type_assignment = manual_type_input,
                                                                         user_input_estimators = user_input_estimators,
                                                                         project_rank = input_rank,
                                                                         io           = io)
      predicted_negative_binomial = predict(NegativeBinomial(),
                                            forward_map(NegativeBinomial(),
                                                        completed_matrix[type_tracker[:NegativeBinomial]],
                                                        r_estimate = r_input))
      truth_negative_binomial = truth_matrix[type_tracker[:NegativeBinomial]]
      summary_negative_binomial = provide(Diagnostics{Any}(),
                                          reference = truth_negative_binomial,
                                          input_data = predicted_negative_binomial)
      show(io, MIME("text/plain"), summary_negative_binomial)
      print(io, "\n")
      show(io, MIME("text/plain"), to)
      close(io)
    end
  end
end

ext: jl


text:
module MyExample

using ForwardDiff

#greet() = print("Hello World!")

include("extra_file.jl")

#export my_f

export my_f, derivative_of_my_f

end # module


text:
# x0y plane

using PCLCommon
using PCLIO
using PCLFilters
using PCLSegmentation
using PCLSampleConsensus
using PCLVisualization
using Cxx

function Base.convert{T}(Array, v::CxxStd.StdVector{T})
    r = T[]
    for e in v
        push!(r,T(e))
    end
    r
end

# IR camera parameters of Kinect v2
fx = 365.1177
fy = 365.1177
cx = 256.0793
cy = 204.4635

function depth2World(x, y, z, cx, fx, cy, fy)
    X = (Float64(x) - cx) * z /  fx
    Y = (Float64(y) - cy) * z / fy
    Z = z
    (X, Y, Z)
end

function world2Depth(X, Y, Z, cx, fx, cy, fy)
    x = fx * X / Z + cx
    y = fy * Y / Z + cy
    z = Z
    (x, y, z)
end

PT = PointXYZRGB

fname = joinpath(dirname(@__FILE__), "pcd",
    "2016-09-07T19:51:46.802_41015197239218.pcd")
cloud = PointCloud{PT}(joinpath(dirname(@__FILE__), fname))
rotated_model = PointCloud{PT}();

### Extract ground plane
cloud_filtered = deepcopy(cloud)
coefficients = ModelCoefficients()
inliers = PointIndices()

seg = SACSegmentation{PT}()
setOptimizeCoefficients(seg, true)
setModelType(seg, SACMODEL_PLANE)
setMethodType(seg, SAC_RANSAC)
setMaxIterations(seg, 1000)
setDistanceThreshold(seg, 0.01)

ex = ExtractIndices{PT}()
nr_points = length(cloud_filtered)

planes = []
coefs = []
while length(cloud_filtered) > 0.3 * nr_points
    setInputCloud(seg, cloud_filtered)
    segment(seg, inliers, coefficients)

    coefvalues = icxx"$(coefficients.handle)->values;"
    indices = icxx"$(inliers.handle)->indices;"

    push!(coefs, coefvalues)

    if length(indices) == 0
        error("Could not estimate a planar model for the given dataset.")
    end

    cloud_p = PointCloud{PT}()
    cloud_f = PointCloud{PT}()

    setInputCloud(ex, cloud_filtered)
    setIndices(ex, inliers)
    setNegative(ex, false)
    filter(ex, cloud_p)
    println("PointCloud representing the planar component:",
        length(cloud_p), " data points")

    setNegative(ex, true)
    filter(ex, cloud_f)
    cloud_filtered = cloud_f

    push!(planes, cloud_p)
    break
end

neg_plane = cloud_filtered

### Compute trasnsform matrix so that ground plane is parallel with x0y axis

xy_plane_normal = Float32[0.0,0.0,1.0]
ground_normal = convert(Array, coefs[1])[1:end-1]
rotation = cross(xy_plane_normal, ground_normal)
theta = -atan2(norm(rotation), dot(xy_plane_normal, ground_normal))
@show ground_normal
@show rotation
@show theta
@show rad2deg(theta)

# Make rotation a unit vector
rotation = normalize(rotation)
# @assert norm(rotation) == 1.0

#=
transform = icxx"""
              Eigen::Affine3f transform_2 = Eigen::Affine3f::Identity();
                transform_2.translation() << 0, 0, 0;
                transform_2.rotate (Eigen::AngleAxisf ($theta, Eigen::Vector3f::UnitY()));
           return transform_2;
       """
=#

transform = icxx"""
             Eigen::Affine3f transform_2 = Eigen::Affine3f::Identity();
               transform_2.translation() << 0, 0, 0;
               transform_2.rotate (Eigen::AngleAxisf ($theta, Eigen::Vector3f($(rotation[1]),$(rotation[2]),$(rotation[3]))));
          return transform_2;
      """
icxx"std::cout << $(transform).matrix() << std::endl;"

transformPointCloud(cloud, rotated_model, transform)

if isdefined(:vis) && vis
    viewer = PCLVisualizer("test");
    addCoordinateSystem(viewer, 1.2, 0,0,0)
    addPointCloud(viewer, cloud,
        PointCloudColorHandlerCustom(cloud, 0,255,0), id="input")
    addPointCloud(viewer, first(planes),
        PointCloudColorHandlerCustom(first(planes), 0,0,255), id="ground")
    addPointCloud(viewer, rotated_model,
        PointCloudColorHandlerRGBField(rotated_model), id="rotated_model")
    try
        spin(viewer)
    finally
        close(viewer);viewer=0;gc();gc()
    end
    error()
end

function getDepthFromPointCloud!(depth, cloud)
    # @assert size(depth) == (512,424)
    fill!(depth, 0.0)
    w, h = size(depth)
    depthp = pointer(depth)
    icxx"""
        const float fx = $fx;
        const float fy = $fy;
        const float cx = $cx;
        const float cy = $cy;
        for (size_t i = 0; i < $(cloud.handle)->points.size(); ++i) {
            const auto p = $(cloud.handle)->points[i];
            if (std::isnan(p.z) || p.z <= 0) {
                continue;
            }
            int r, c;
            float depth;
            c = fx * p.x / p.z + cx;
            r = fy * p.y / p.z + cy;
            depth = p.z * 1000.0;
            if (r >= 0 && r < $h && c >= 0 && c < $w) {
                $(depthp)[$w * r + c] = depth;
            }
        }
    """
    depth
end

using Libfreenect2
using Cxx
using CVCore
using CVHighGUI
using CVImgProc
using CVImgCodecs

"""Get point cloud from undistored depth and reigstered color"""
function getPointCloudXYZRGB(registration::Registration, undistorted, registered)
    w = Libfreenect2.width(undistorted)
    h = Libfreenect2.height(undistorted)
    cloud = PointCloud{PointXYZRGB}(w, h)
    icxx"$(cloud.handle)->is_dense = false;"
    pointsptr = icxx"&$(cloud.handle)->points[0];"
    icxx"""
    for (size_t ri = 0; ri < $h; ++ri) {
        for (size_t ci = 0; ci < $w; ++ci) {
            auto p = $(pointsptr) + $w * ri + ci;
            $(registration.handle)->getPointXYZRGB($(undistorted.handle),
                $(registered.handle), ri, ci, p->x, p->y, p->z, p->rgb);
        }
    }
    """
    cloud
end

isesc(key) = key == 27

f = Freenect2()
num_devices = enumerateDevices(f)
if num_devices <= 0
    error("No device!")
end
device = openDefaultDevice(f, OpenCLPacketPipeline())
listener = SyncMultiFrameListenerPtr()
setIrAndDepthFrameListener(device, listener)
setColorFrameListener(device, listener)

start(device)

# NOTE: must be called after start(device)
registration = Registration(getIrCameraParams(device),
    getColorCameraParams(device))
undistorted = FramePtr(512, 424, 4, key=Libfreenect2.FRAME_DEPTH)
registered = FramePtr(512, 424, 4, key=Libfreenect2.FRAME_COLOR)

tilt_img_scale_w = 2
tilt_img_scale_h = 1
tilt_compensatated_depth = zeros(Float64,
    Int(512*tilt_img_scale_w), Int(424*tilt_img_scale_h))

pass = PassThrough{PT}()
sor = VoxelGrid{PT}()

try
    viewer = PCLVisualizer("pcl visualizer")
    addCoordinateSystem(viewer, 1.2, 0,0,0)
    while !wasStopped(viewer)
        frames = waitForNewFrame(listener)
        color = frames[FrameType.COLOR]
        ir = frames[FrameType.IR]
        depth = frames[FrameType.DEPTH]

        # Depth and color registration
        t = @elapsed begin
            apply(registration, color, depth, undistorted, registered)
            cloud = getPointCloudXYZRGB(registration, undistorted, registered)
        end
        println("Registration and getPointCloudXYZRGB: $(1/t) Hz")

        # Point cloud density reduction
        cloud_filtered = PointCloud{PT}();

        setInputCloud(sor, cloud)
        setLeafSize(sor, 0.01, 0.01, 0.01)
        filter(sor, cloud_filtered)
        cloud = cloud_filtered

        setInputCloud(pass, cloud)
        setFilterFieldName(pass, "z")
        setFilterLimits(pass, 1.2, 3.0)
        filter(pass, cloud_filtered)
        cloud = cloud_filtered

        # tilt compensation
        rotated_model = PointCloud{PT}();
        t = @elapsed begin
            transformPointCloud(cloud, rotated_model, transform)
        end
        println("transformPointCloud: $(1/t) Hz")

        # Get tilt compensatd depth
        t = @elapsed begin
            getDepthFromPointCloud!(tilt_compensatated_depth, rotated_model);
        end
        println("getDepthFromPointCloud: $(1/t) Hz")

        scale!(tilt_compensatated_depth, 1/4500)

        # Get depth array for comparison
        deptharr = Array(depth)
        scale!(deptharr, 1/4500)

        # Update point cloud viewer
        cloud_color_handler = PointCloudColorHandlerCustom(cloud, 0,255,0)
        if !updatePointCloud(viewer, cloud, cloud_color_handler, id="input_cloud")
            addPointCloud(viewer, cloud, cloud_color_handler, id="input_cloud")
        end
        color_handler = PointCloudColorHandlerRGBField(cloud)
        if !updatePointCloud(viewer, rotated_model, color_handler, id="rotated_model")
            addPointCloud(viewer, rotated_model, color_handler, id="rotated_model")
        end
        spinOnce(viewer, 1)

        # Show depth
        imshow("original depth", deptharr)
        imshow("tilt compensated depth", tilt_compensatated_depth)
        key = waitKey(1)
        if key == Int('s')
            scale!(deptharr, 255)
            scale!(tilt_compensatated_depth, 255)
            depthmat = Mat(deptharr)
            flip!(depthmat, 1)
            tilt_compensatated_depth_mat = Mat(tilt_compensatated_depth)
            flip!(tilt_compensatated_depth_mat, 1)
            imwrite("original_depth.png", depthmat)
            imwrite("tilt_compensated_depth.png", tilt_compensatated_depth_mat)
        end
        isesc(key) && break

        Libfreenect2.release(listener, frames)

        rand() > 0.95 && gc(false)
    end
finally
    stop(device)
    close(device)
    close(viewer)
    destroyAllWindows()
    device=0;viewer=0;gc()
end


text:
using Revise
using LinearAlgebra, StatsBase, Statistics, Distributions, Random
#Random.seed!(11)

#for displaying matrices
printm(m) = show(IOContext(stdout, :limit=>false), MIME"text/plain"(), m)
function printmln(m)
  show(IOContext(stdout, :limit=>false), MIME"text/plain"(), m)
  println()
end

#generates a well-defined covariance matrix
function randomcovar(K, N=10^4; displaymatrix=false)
  ϵ = rand(Uniform(-1,1), N,K)
  ψ = rand(Uniform(-1,1), N)
  β = (k->k*(-1)^k).(1:K)

  X = β' .* ψ
  @assert size(X) == (N,K)
  X .+= ϵ
  Σ = cov(X)

  displaymatrix && printmln(Σ)
  return Σ
end

function checkmatrixalgebra(dims::NamedTuple; testiter=10^5)
  atol = dims.K/testiter^0.5
  #generate the characteristic matrix
  Ξ = rand(Laplace(), dims.N, dims.K+1)
  #Ξ .-= mean(Ξ, dims=1)
  Ξ[:,end] = rand(dims.N) #last column is the market

  #signal parameters
  δi = rand(dims.K+1)
  Σθ = randomcovar(dims.K+1)
  σ²ϵ = rand(dims.N) .+ 0.0001
  #Σϵ = Diagonal(σ²ϵ)
  Σϵ = randomcovar(dims.N)

  Rf = 1.2
  P = rand(TriangularDist(1.0,11.0), dims.N)
  N1 = ones(dims.N)

  #distributions
  distθ = MvNormal(δi, Σθ)
  distϵ = MvNormal(zeros(dims.N), Σϵ)

  #test covariance of portfolio
  wi = rand(dims.N) .+ 0.01
  wi ./= sum(wi)
  ωi = wi ./ P
  vervec = [wi'* (Ξ * rand(distθ) ./ P .+ rand(distϵ) ./ P .- Rf * N1) for i in 1:testiter]
  vervar = var(vervec)
  testvar = ωi' * (Ξ*Σθ*Ξ' .+ Σϵ) * ωi
  msg  = "covariance of portfolio test with Δ $(abs(vervar-testvar)) and max atol $atol"
  isapprox(testvar, vervar, atol=atol) ? @info("passed $msg") : @warn("failed $msg")

  #test expectation
  vermean = mean(vervec)
  testmean = ωi' * (Ξ * δi .- P * Rf)
  msg  = "expectation of portfolio test with Δ $(abs(vermean-testmean)) and max atol $atol"
  isapprox(testmean, vermean, atol=atol) ? @info("passed $msg") : @warn("failed $msg")

  #test woodbury
  Σθinv = svd(Σθ)\I
  Σϵinv = Σϵ\I
  IN = Diagonal(I,dims.N)

  verω = (Ξ *Σθ * Ξ' + Σϵ)\I * (Ξ * δi .- P .* Rf)
  testw1 = Σϵinv*(I - Ξ*((Σθinv .+ Ξ'*Σϵinv*Ξ)\I)*Ξ'*Σϵinv)*(Ξ*δi .- P .* Rf)
  @assert verω ≈ testw1
  @info "passed test prewoodbury"

  si = (I - ((Σθinv .+ Ξ'*Σϵinv*Ξ)\I) * (Ξ'*Σϵinv * Ξ)) * δi
  κ = (I - Ξ*((Σθinv .+ Ξ'*Σϵinv*Ξ)\I) * (Ξ'*Σϵinv))
  testw2 = Σϵinv*(Ξ*si .- κ * P .* Rf)
  @assert verω ≈ testw2

  testw3 = P .* Σϵinv*(Ξ*si  .- κ * P .* Rf)
  #testw3 = Σϵinv*(Ξ*si .- κ * P .* Rf)
  ver3 = P .* verω
  @assert ver3 ≈ testw3
  @info "passed test woodbury"

  ###Market amalgamation test
  Ξᴷ = Ξ[:,1:(end-1)]
  ξₘ = Ξ[:,end] .* si[end] .- κ * P .* Rf
  siᴷ = si[1:(end-1)]
  testamalg1 = ωi'*Σϵinv*Ξᴷ*siᴷ .+ ωi'*Σϵinv*ξₘ

  Ξᴺ=deepcopy(Ξ)
  Ξᴺ[:,end] .-= κ * P .* Rf/si[end]
  testamalg2 = ωi'*Σϵinv*Ξᴺ*si

  veramalg = ωi'*Σϵinv*(Ξ*si .- κ * P .* Rf)

  @assert testamalg1 ≈ veramalg
  @assert testamalg2 ≈ veramalg
  @info "passed market amalgamation test"

  #Signal test
  verxi = verω .* P#Σϵinv * (Ξ*si - κ*P*Rf) .* P
  testxi = [Σϵinv * Ξ[:,k] .* si[k] .* P for k ∈ 1:(dims.K+1)]
  testxi[end] .= Σϵinv * (Ξ[:,end] .* si[end] .- κ*P .* Rf) .* P
  @assert verxi ≈ sum(testxi)
  @info "passed portfolio weight test. Portfolio weights (net): $(sum.(testxi))"

end

function checkwork()
  dims = (N=1000, K=5)
  checkmatrixalgebra(dims,testiter=10^5)
end

@time checkwork()

max_stars_repo_name: clintonTE/CCA

max_issues_repo_name: clintonTE/CCA

max_forks_repo_name: clintonTE/CCA