text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Sockets, Random, Test
using Base: Experimental

# set up a watchdog alarm for 10 minutes
# so that we can attempt to get a "friendly" backtrace if something gets stuck
# (although this'll also terminate any attempted debugging session)
# expected test duration is about 5-10 seconds
function killjob(d)
    Core.print(Core.stderr, d)
    if Sys.islinux()
        SIGINFO = 10
    elseif Sys.isbsd()
        SIGINFO = 29
    end
    if @isdefined(SIGINFO)
        ccall(:uv_kill, Cint, (Cint, Cint), getpid(), SIGINFO)
        sleep(1)
    end
    ccall(:uv_kill, Cint, (Cint, Cint), getpid(), Base.SIGTERM)
    nothing
end
sockets_watchdog_timer = Timer(t -> killjob("KILLING BY SOCKETS TEST WATCHDOG\n"), 600)

@testset "parsing" begin
    @test ip"127.0.0.1" == IPv4(127,0,0,1)
    @test ip"192.0" == IPv4(192,0,0,0)

    # These used to work, but are now disallowed. Check that they error
    @test_throws ArgumentError parse(IPv4, "192.0xFFF") # IPv4(192,0,15,255)
    @test_throws ArgumentError parse(IPv4, "192.0xFFFF") # IPv4(192,0,255,255)
    @test_throws ArgumentError parse(IPv4, "192.0xFFFFF") # IPv4(192,15,255,255)
    @test_throws ArgumentError parse(IPv4, "192.0xFFFFFF") # IPv4(192,255,255,255)
    @test_throws ArgumentError parse(IPv4, "022.0.0.1") # IPv4(18,0,0,1)

    @test UInt(IPv4(0x01020304)) == 0x01020304
    @test Int(IPv4("1.2.3.4")) == Int(0x01020304) == Int32(0x01020304)
    @test Int128(IPv6("2001:1::2")) == 42540488241204005274814694018844196866
    @test_throws InexactError Int16(IPv4("1.2.3.4"))
    @test_throws InexactError Int64(IPv6("2001:1::2"))

    let ipv = parse(IPAddr, "127.0.0.1")
        @test isa(ipv, IPv4)
        @test ipv == ip"127.0.0.1"
        @test sprint(show, ipv) == "ip\"127.0.0.1\""
    end

    @test_throws ArgumentError parse(IPv4, "192.0xFFFFFFF")
    @test_throws ArgumentError IPv4(192,255,255,-1)
    @test_throws ArgumentError IPv4(192,255,255,256)

    @test_throws ArgumentError parse(IPv4, "192.0xFFFFFFFFF")
    @test_throws ArgumentError parse(IPv4, "192.")
    @test_throws ArgumentError parse(IPv4, "256.256.256.256")
    # too many fields
    @test_throws ArgumentError parse(IPv6, "256:256:256:256:256:256:256:256:256:256:256:256")

    @test_throws ArgumentError IPv4(-1)
    @test_throws ArgumentError IPv4(typemax(UInt32) + Int64(1))

    @test ip"::1" == IPv6(1)
    @test ip"2605:2700:0:3::4713:93e3" == IPv6(parse(UInt128,"260527000000000300000000471393e3", base = 16))

    @test ip"2001:db8:0:0:0:0:2:1" == ip"2001:db8::2:1" == ip"2001:db8::0:2:1"

    @test ip"0:0:0:0:0:ffff:127.0.0.1" == IPv6(0xffff7f000001)

    let ipv = parse(IPAddr, "0:0:0:0:0:ffff:127.0.0.1")
        @test isa(ipv, IPv6)
        @test ipv == ip"0:0:0:0:0:ffff:127.0.0.1"
    end

    @test_throws ArgumentError IPv6(-1)
    @test_throws ArgumentError IPv6(1,1,1,1,1,1,1,-1)
    @test_throws ArgumentError IPv6(1,1,1,1,1,1,1,typemax(UInt16)+1)

    @test IPv6(UInt16(1), UInt16(1), UInt16(1), UInt16(1), UInt16(1), UInt16(1), UInt16(1), UInt16(1)) == IPv6(1,1,1,1,1,1,1,1)

    @test_throws BoundsError Sockets.ipv6_field(IPv6(0xffff7f000001), -1)
    @test_throws BoundsError Sockets.ipv6_field(IPv6(0xffff7f000001), 9)
end

@testset "InetAddr constructor" begin
    inet = Sockets.InetAddr(IPv4(127,0,0,1), 1024)
    @test inet.host == ip"127.0.0.1"
    @test inet.port == 1024
    str = "Sockets.InetAddr{$(isdefined(Main, :IPv4) ? "" : "Sockets.")IPv4}(ip\"127.0.0.1\", 1024)"
    @test sprint(show, inet) == str
    inet = Sockets.InetAddr("127.0.0.1", 1024)
    @test inet.host == ip"127.0.0.1"
    @test inet.port == 1024
end
@testset "InetAddr invalid port" begin
    @test_throws InexactError Sockets.InetAddr(IPv4(127,0,0,1), -1)
    @test_throws InexactError Sockets.InetAddr(IPv4(127,0,0,1), typemax(UInt16)+1)
end

@testset "isless and comparisons" begin
    @test ip"1.2.3.4" < ip"1.2.3.7" < ip"2.3.4.5"
    @test ip"1.2.3.4" >= ip"1.2.3.4" >= ip"1.2.3.1"
    @test isless(ip"1.2.3.4", ip"1.2.3.5")
    @test_throws MethodError sort([ip"2.3.4.5", ip"1.2.3.4", ip"2001:1:2::1"])
end

@testset "broadcastable" begin
    @test size(ip"127.0.0.1" .== ip"127.0.0.1") == ()
    @test size(ip"::1" .== ip"::1") == ()
end

@testset "RFC 5952 Compliance" begin
    @test repr(ip"2001:db8:0:0:0:0:2:1") == "ip\"2001:db8::2:1\""
    @test repr(ip"2001:0db8::0001") == "ip\"2001:db8::1\""
    @test repr(ip"2001:db8::1:1:1:1:1") == "ip\"2001:db8:0:1:1:1:1:1\""
    @test repr(ip"2001:db8:0:0:1:0:0:1") == "ip\"2001:db8::1:0:0:1\""
    @test repr(ip"2001:0:0:1:0:0:0:1") == "ip\"2001:0:0:1::1\""
end

defaultport = rand(2000:4000)
@testset "writing to/reading from a socket, getsockname and getpeername" begin
    for testport in [0, defaultport]
        port = Channel(1)
        tsk = @async begin
            local (p, s) = listenany(testport)
            @test p != 0
            @test getsockname(s) == (Sockets.localhost, p)
            put!(port, p)
            for i in 1:3
                sock = accept(s)
                @test getsockname(sock) == (Sockets.localhost, p)
                let peer = getpeername(sock)::Tuple{IPAddr, UInt16}
                    @test peer[1] == Sockets.localhost
                    @test 0 != peer[2] != p
                end
                # test write call
                write(sock, "Hello World\n")

                # test "locked" println to a socket
                @Experimental.sync begin
                    for i in 1:100
                        @async println(sock, "a", 1)
                    end
                end
                close(sock)
            end
            close(s)
        end
        wait(port)
        let p = fetch(port)
            otherip = getipaddr()
            if otherip != Sockets.localhost
                @test_throws Base._UVError("connect", Base.UV_ECONNREFUSED) connect(otherip, p)
            end
            for i in 1:3
                client = connect(p)
                let name = getsockname(client)::Tuple{IPAddr, UInt16}
                    @test name[1] == Sockets.localhost
                    @test 0 != name[2] != p
                end
                @test getpeername(client) == (Sockets.localhost, p)
                @test read(client, String) == "Hello World\n" * ("a1\n"^100)
            end
        end
        wait(tsk)
    end

    mktempdir() do tmpdir
        socketname = Sys.iswindows() ? ("\\\\.\\pipe\\uv-test-" * randstring(6)) : joinpath(tmpdir, "socket")
        local nconn = 0
        srv = listen(socketname)
        t = accept(srv) do client
            write(client, "Hello World $(nconn += 1)\n")
            close(client)
            nconn == 3 && Base.wait_close(srv)
        end
        @test read(connect(socketname), String) == "Hello World 1\n"
        @test read(connect(socketname), String) == "Hello World 2\n"
        @test read(connect(socketname), String) == "Hello World 3\n"
        conn = connect(socketname)
        close(srv)
        wait(t)
        @test read(conn, String) == ""
    end
end

@testset "getsockname errors" begin
    sock = TCPSocket()
    serv = Sockets.TCPServer()
    @test_throws MethodError getpeername(serv)
    @test_throws Base._UVError("cannot obtain socket name", Base.UV_EBADF) getpeername(sock)
    @test_throws Base._UVError("cannot obtain socket name", Base.UV_EBADF) getsockname(serv)
    @test_throws Base._UVError("cannot obtain socket name", Base.UV_EBADF) getsockname(sock)
    close(sock)
    close(serv)
end

@testset "getnameinfo on some unroutable IP addresses (RFC 5737)" begin
    @test getnameinfo(ip"192.0.2.1") == "192.0.2.1"
    @test getnameinfo(ip"198.51.100.1") == "198.51.100.1"
    @test getnameinfo(ip"203.0.113.1") == "203.0.113.1"
    @test getnameinfo(ip"0.1.1.1") == "0.1.1.1"
    @test getnameinfo(ip"::ffff:0.1.1.1") == "::ffff:0.1.1.1"
    @test getnameinfo(ip"::ffff:192.0.2.1") == "::ffff:192.0.2.1"
    @test getnameinfo(ip"2001:db8::1") == "2001:db8::1"
end

@testset "getnameinfo on some valid IP addresses" begin
    @test !isempty(getnameinfo(ip"::")::String)
    @test !isempty(getnameinfo(ip"0.0.0.0")::String)
    @test !isempty(getnameinfo(ip"10.1.0.0")::String)
    @test !isempty(getnameinfo(ip"10.1.0.255")::String)
    @test !isempty(getnameinfo(ip"10.1.255.1")::String)
    @test !isempty(getnameinfo(ip"255.255.255.255")::String)
    @test !isempty(getnameinfo(ip"255.255.255.0")::String)
    @test !isempty(getnameinfo(ip"192.168.0.1")::String)
    @test !isempty(getnameinfo(ip"::1")::String)
end

@testset "getaddrinfo" begin
    @test getaddrinfo("127.0.0.1") == ip"127.0.0.1"
    @test getaddrinfo("::1") == ip"::1"
    let localhost = getnameinfo(ip"127.0.0.1")::String
        @test !isempty(localhost) && localhost != "127.0.0.1"
        @test !isempty(getalladdrinfo(localhost)::Vector{IPAddr})
        @test getaddrinfo(localhost, IPv4)::IPv4 != ip"0.0.0.0"
        @test try
            getaddrinfo(localhost, IPv6)::IPv6 != ip"::"
        catch ex
            isa(ex, Sockets.DNSError) && ex.code == Base.UV_EAI_NONAME && ex.host == localhost
        end
    end
    @test_throws Sockets.DNSError getaddrinfo(".invalid")
    @test_throws ArgumentError getaddrinfo("localhost\0") # issue #10994
    @test_throws Base._UVError("connect", Base.UV_ECONNREFUSED) connect(ip"127.0.0.1", 21452)
    e = (try; getaddrinfo(".invalid"); catch ex; ex; end)
    @test startswith(sprint(show, e), "DNSError:")
end

@testset "invalid port" begin
    @test_throws ArgumentError connect(ip"127.0.0.1", -1)
    @test_throws ArgumentError connect(ip"127.0.0.1", typemax(UInt16)+1)
    @test_throws ArgumentError connect(ip"0:0:0:0:0:ffff:127.0.0.1", -1)
    @test_throws ArgumentError connect(ip"0:0:0:0:0:ffff:127.0.0.1", typemax(UInt16)+1)
end

@testset "put! in listenany(defaultport)" begin
    (p, server) = listenany(defaultport)
    r = Channel(1)
    tsk = @async begin
        put!(r, :start)
        @test_throws Base._UVError("accept", Base.UV_ECONNABORTED) accept(server)
    end
    @test fetch(r) === :start
    close(server)
    wait(tsk)
end

# test connecting to a named port
let localhost = ip"127.0.0.1"
    global randport
    randport, server = listenany(localhost, defaultport)
    @async connect(localhost, randport)
    s1 = accept(server)
    @test_throws ErrorException("client TCPSocket is not in initialization state") accept(server, s1)
    @test_throws Base._UVError("listen", Base.UV_EADDRINUSE) listen(randport)
    port2, server2 = listenany(localhost, randport)
    @test randport != port2
    close(server)
    close(server2)
end

@test_throws Sockets.DNSError connect(".invalid", 80)

@testset "UDPSocket" begin
    # test show() function for UDPSocket()
    @test repr(UDPSocket()) ∈ ("Sockets.UDPSocket(init)", "UDPSocket(init)")

    let
        a = UDPSocket()
        b = UDPSocket()
        bind(a, ip"127.0.0.1", randport)
        bind(b, ip"127.0.0.1", randport + 1)

        @Experimental.sync begin
            let i = 0
                for _ = 1:30
                    @async let msg = String(recv(a))
                        @test msg == "Hello World $(i += 1)"
                    end
                end
            end
            yield()
            for i = 1:30
                send(b, ip"127.0.0.1", randport, "Hello World $i")
            end
        end
        let msg = Vector{UInt8}("fedcba9876543210"^36) # The minimum reassembly buffer size for IPv4 is 576 bytes
            tsk = @async @test recv(a) == msg
            @test send(b, ip"127.0.0.1", randport, msg) === nothing
            wait(tsk)
        end
        let msg = Vector{UInt8}("1234"^16377) # The maximum size of an IPv4 datagram is 65535 bytes, including the header
            @test_throws(Base._UVError("send", Base.UV_EMSGSIZE),
                         send(b, ip"127.0.0.1", randport, msg))
            pop!(msg)
            tsk = @async recv(a)
            try
                send(b, ip"127.0.0.1", randport, msg)
            catch ex
                if !(ex isa Base.IOError && ex.code == Base.UV_EMSGSIZE) || Sys.islinux() || Sys.iswindows()
                    # this is allowed failure on some platforms which might further restrict
                    # the maximum packet size being sent (even locally), such as BSD's `sysctl net.inet.udp.maxdgram`
                    rethrow()
                end
                empty!(msg)
                send(b, ip"127.0.0.1", randport, msg) # check that the socket is still alive
            end
            @test fetch(tsk) == msg
        end
        let tsk = @async send(b, ip"127.0.0.1", randport, "WORLD HELLO")
            (inetaddr, data) = recvfrom(a)
            @test inetaddr.host == ip"127.0.0.1" && String(data) == "WORLD HELLO"
            wait(tsk)
        end
        close(a)
        close(b)
    end

    @test_throws MethodError bind(UDPSocket(), randport)

    let
        a = UDPSocket()
        b = UDPSocket()
        bind(a, ip"::1", UInt16(randport))
        bind(b, ip"::1", UInt16(randport + 1))

        for i = 1:3
            tsk = @async begin
                let (inetaddr, data) = recvfrom(a)
                    @test inetaddr.host == ip"::1"
                    @test String(data) == "Hello World"
                end
            end
            send(b, ip"::1", randport, "Hello World")
            wait(tsk)
        end
    end
end

@testset "local ports" begin
    for (addr, porthint) in [
            (IPv4("127.0.0.1"), UInt16(11011)),
            (IPv6("::1"), UInt16(11012)),
            (getipaddr(), UInt16(11013)) ]
        port, listen_sock = listenany(addr, porthint)
        gsn_addr, gsn_port = getsockname(listen_sock)

        @test addr == gsn_addr
        @test port == gsn_port

        # connect to it
        client_sock = connect(addr, port)
        test_done = false
        @Experimental.sync begin
            @async begin
                Base.wait_readnb(client_sock, 1)
                test_done || error("Client disconnected prematurely.")
            end
            @async begin
                server_sock = accept(listen_sock)

                self_client_addr, self_client_port = getsockname(client_sock)
                peer_client_addr, peer_client_port = getpeername(client_sock)
                self_srvr_addr, self_srvr_port = getsockname(server_sock)
                peer_srvr_addr, peer_srvr_port = getpeername(server_sock)

                @test self_client_addr == peer_client_addr == self_srvr_addr == peer_srvr_addr

                @test peer_client_port == self_srvr_port
                @test peer_srvr_port == self_client_port
                @test self_srvr_port != self_client_port

                test_done = true

                close(listen_sock)
                close(client_sock)
                close(server_sock)
            end
        end
    end
end

@testset "Local-machine broadcast" begin
    let a, b, c
        # Apple does not support broadcasting on 127.255.255.255
        bcastdst = Sys.isapple() ? ip"255.255.255.255" : ip"127.255.255.255"

        function create_socket(addr::IPAddr, port)
            s = UDPSocket()
            bind(s, addr, port, reuseaddr = true, enable_broadcast = true)
            return s
        end

        # Wait for futures to finish with a given timeout
        function wait_with_timeout(recvs, TIMEOUT_VAL = 3*1e9)
            t0 = time_ns()
            recvs_check = copy(recvs)
            while ((length(filter!(t->!istaskdone(t), recvs_check)) > 0)
                  && (time_ns() - t0 < TIMEOUT_VAL))
                sleep(0.05)
            end
            length(recvs_check) > 0 && error("timeout")
            map(wait, recvs)
        end

        # First, test IPv4 broadcast
        port = 2000
        a, b, c = [create_socket(ip"0.0.0.0", port) for i in 1:3]
        try
            # bsd family do not allow broadcasting on loopbacks
            @static if !Sys.isbsd() || Sys.isapple()
                send(c, bcastdst, port, "hello")
                recvs = [@async @test String(recv(s)) == "hello" for s in (a, b)]
                wait_with_timeout(recvs)
            end
        catch e
            if isa(e, Base.IOError) && Base.uverrorname(e.code) == "EPERM"
                @warn "UDP IPv4 broadcast test skipped (permission denied upon send, restrictive firewall?)"
            else
                rethrow()
            end
        end
        [close(s) for s in [a, b, c]]

        # Test ipv6 broadcast groups
        a, b, c = [create_socket(ip"::", port) for i in 1:3]
        try
            # Exemplary Interface-local ipv6 multicast group, if we wanted this to actually be routed
            # to other computers, we should use a link-local or larger address scope group
            # bsd family and darwin do not allow broadcasting on loopbacks
            @static if !Sys.isbsd() && !Sys.isapple()
                group = ip"ff11::6a75:6c69:61"
                join_multicast_group(a, group)
                join_multicast_group(b, group)

                send(c, group, port, "hello")
                recvs = [@async @test String(recv(s)) == "hello" for s in (a, b)]
                wait_with_timeout(recvs)

                leave_multicast_group(a, group)
                leave_multicast_group(b, group)

                send(c, group, port, "hello")
                recvs = [@async @test String(recv(s)) == "hello" for s in (a, b)]
                # We only wait 200ms since we're pretty sure this is going to time out
                @test_throws ErrorException wait_with_timeout(recvs, 2e8)
            end
        catch e
            if isa(e, Base.IOError) && Base.uverrorname(e.code) == "EPERM"
                @warn "UDP IPv6 broadcast test skipped (permission denied upon send, restrictive firewall?)"
            else
                rethrow()
            end
        end
    end
end

@testset "Pipe" begin
    P = Pipe()
    Base.link_pipe!(P)
    write(P, "hello")
    @test bytesavailable(P) == 0
    @test !eof(P)
    @test read(P, Char) === 'h'
    @test !eof(P)
    @test read(P, Char) === 'e'
    @test isopen(P)
    t = @async begin
        # feed uv_read one more event so that it triggers the transition from active -> open
        write(P, "w")
        while P.out.status != Base.StatusOpen
            yield() # wait for that transition
        end
        close(P.in)
    end
    # on unix, this proves that the kernel can buffer a single byte
    # even with no registered active call to read
    # on windows, the kernel fails to do even that
    # causing the `write` call to freeze
    # so we end up forced to do a slightly weaker test here
    Sys.iswindows() || wait(t)
    @test isopen(P) # without an active uv_reader, P shouldn't be closed yet
    @test !eof(P) # should already know this,
    @test isopen(P) #  so it still shouldn't have an active uv_reader
    @test readuntil(P, 'w') == "llo"
    Sys.iswindows() && wait(t)
    @test eof(P)
    @test !isopen(P) # eof test should have closed this by now
    close(P) # should be a no-op, just make sure
    @test !isopen(P)
    @test eof(P)
end

@testset "connect!" begin
    # test the method matching connect!(::TCPSocket, ::Sockets.InetAddr{T<:Base.IPAddr})
    let addr = Sockets.InetAddr(ip"127.0.0.1", 4444)
        srv = listen(addr)
        r = @async close(accept(srv))
        close(connect(addr))
        fetch(r)
        close(srv)
    end

    let addr = Sockets.InetAddr(ip"127.0.0.1", 4444)
        srv = listen(addr)
        r = @async close(srv)
        @test_throws Base._UVError("accept", Base.UV_ECONNABORTED) accept(srv)
        fetch(r)
    end

    let addr = Sockets.InetAddr(ip"127.0.0.1", 4444)
        srv = listen(addr)
        s = Sockets.TCPSocket()
        Sockets.connect!(s, addr)
        r = @async close(s)
        @test_throws Base._UVError("connect", Base.UV_ECANCELED) Sockets.wait_connected(s)
        fetch(r)
    end
end

@testset "iswritable" begin
    let addr = Sockets.InetAddr(ip"127.0.0.1", 4445)
        srv = listen(addr)
        s = Sockets.TCPSocket()
        Sockets.connect!(s, addr)
        @test iswritable(s)
        close(s)
        @test !iswritable(s)
    end
end

@testset "TCPServer constructor" begin
    s = Sockets.TCPServer(; delay=false)
    if ccall(:jl_has_so_reuseport, Int32, ()) == 1
        @test 0 == ccall(:jl_tcp_reuseport, Int32, (Ptr{Cvoid},), s.handle)
    end
end

@testset "getipaddrs" begin
    @test getipaddr() in getipaddrs()
    try
        getipaddr(IPv6) in getipaddrs(IPv6)
    catch
        if !isempty(getipaddrs(IPv6))
            @test "getipaddr(IPv6) errored when it shouldn't have!"
        end
    end

    @testset "including loopback addresses" begin
        @test issubset(getipaddrs(), getipaddrs(loopback=true))
        @test issubset(getipaddrs(IPv6), getipaddrs(IPv6, loopback=true))
    end
end

@testset "address scope" begin
    @test islinklocaladdr(ip"169.254.1.0")
    @test islinklocaladdr(ip"169.254.254.255")
    @test islinklocaladdr(ip"fe80::")
    @test islinklocaladdr(ip"febf::")
    @test !islinklocaladdr(ip"127.0.0.1")
    @test !islinklocaladdr(ip"2001::")

end

@static if !Sys.iswindows()
    # Issue #29234
    @testset "TCPSocket stdin" begin
        let addr = Sockets.InetAddr(ip"127.0.0.1", 4455)
            srv = listen(addr)
            s = connect(addr)

            @test success(pipeline(`$(Base.julia_cmd()) --startup-file=no -e "exit()" -i`, stdin=s))

            close(s)
            close(srv)
        end
    end
end

# Issues #18818 and #24169
mutable struct RLimit
    cur::Int64
    max::Int64
end
function with_ulimit(f::Function, stacksize::Int)
    RLIMIT_STACK = 3 # from /usr/include/sys/resource.h
    rlim = Ref(RLimit(0, 0))
    # Get the current maximum stack size in bytes
    rc = ccall(:getrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim)
    @assert rc == 0
    current = rlim[].cur
    try
        rlim[].cur = stacksize * 1024
        ccall(:setrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim)
        f()
    finally
        rlim[].cur = current
        ccall(:setrlimit, Cint, (Cint, Ref{RLimit}), RLIMIT_STACK, rlim)
    end
    nothing
end
@static if Sys.isapple()
    @testset "Issues #18818 and #24169" begin
        with_ulimit(7001) do
            @test getaddrinfo("localhost") isa IPAddr
        end
    end
end

close(sockets_watchdog_timer)

max_stars_repo_name: udesou/julia

max_issues_repo_name: zhuangchaoxi/julia

max_forks_repo_name: zhuangchaoxi/julia


text:
module Molden

export makeMoldenFile

import ..Quiqbox: Molecule, checkFname, AtomicNumberList, centerCoordOf, flatten, groupedSort,
                  joinConcentricBFuncStr, alignNumSign, alignNum

"""

    makeMoldenFile(mol::Molecule; roundDigits::Int=15,
                   recordUMO::Bool=false, fileName::String = "MO") -> String

Write the information of input `Molecule` into a **Molden** file. `recordUMO` determines
whether to include the unoccupied molecular orbitals. `fileName` specifies the name of the
file, which is also the returned value. If `roundDigits < 0`, there won't be rounding for
recorded data.
"""
function makeMoldenFile(mol::Molecule;
                        roundDigits::Int=15, recordUMO::Bool=false, fileName::String = "MO")
    nucCoords = mol.nucCoords |> collect
    nuc = mol.nuc |> collect
    basis = mol.basis |> collect
    MOs = mol.orbital
    iNucPoint = 0
    groups = groupedSort(basis, centerCoordOf)
    strs = joinConcentricBFuncStr.(groups)
    strs = split.(strs, "\n", limit=2)
    gCoeffs = getindex.(strs, 2)
    rpadN = roundDigits < 0 ? 21 : (roundDigits+1)
    lpadN = 8

    text = """
           [Molden Format]
           made by Quiqbox

           [Atoms] (AU)
           """

    centers = getindex.(strs, 1)
    for cen in centers
        iNucPoint += 1
        coord = parse.(Float64, split(cen[5:end]))
        if (i = findfirst(x->prod(isapprox.(x, coord, atol=1e-15)), nucCoords);
            i !== nothing)
            n = popat!(nuc, i)
            atmName = rpad("$(n)", 5)
            atmNumber = rpad("$(AtomicNumberList[n])", 4)
            popat!(nucCoords, i)
        else
            atmName = "X    "
            atmNumber = "X   "
        end
        roundDigits > 0 && (coord = round.(coord, digits=roundDigits))
        coordStr = alignNum(coord[1], lpadN, rpadN; roundDigits) *
                   alignNum(coord[2], lpadN, rpadN; roundDigits) *
                   alignNum(coord[3], lpadN, 0; roundDigits)
        text *= atmName*rpad("$iNucPoint", 5)*atmNumber*coordStr*"\n"
    end
    for (n, coord) in zip(nuc, nucCoords)
        roundDigits >= 0 && (cv = round.(coord, digits=roundDigits))
        iNucPoint += 1
        text *= rpad("$(n)", 5) * rpad(iNucPoint, 5) * rpad("$(AtomicNumberList[n])", 4)*
                alignNum(cv[1], lpadN, rpadN; roundDigits) *
                alignNum(cv[2], lpadN, rpadN; roundDigits) *
                alignNum(cv[3], lpadN, 0; roundDigits) * "\n"
    end
    text *= "\n[GTO]"
    for (i, gs) in zip(1:length(gCoeffs), gCoeffs)
        text *= "\n$i 0\n" * gs
    end
    text *= "\n[MO]\n"
    recordUMO ? (l = length(MOs)) : (l = findfirst(isequal(0),
                                                   [x.occupancy for x in MOs]) - 1)
    for i = 1:l
        text *= "Sym=   $(MOs[i].symmetry)\n"
        moe = MOs[i].energy
        MOcoeffs = MOs[i].orbitalCoeffs
        if roundDigits > 0
            moe = round(moe, digits=roundDigits)
            MOcoeffs = round.(MOcoeffs, digits=roundDigits)
        end
        text *= "Ene=  "*alignNumSign(moe; roundDigits)*"\n"
        text *= "Spin=  $(MOs[i].spin)\n"
        text *= "Occup= $(MOs[i].occupancy)\n"
        text *= join([rpad("   $j", 6)*alignNum(c, lpadN, 0; roundDigits)*
                      "\n" for (j,c) in zip(1:length(MOcoeffs), MOcoeffs)])
    end

    fn = fileName*".molden" |> checkFname
    open(fn, "w") do f
        write(f, text)
    end

    fn
end

end

ext: jl


text:
module EnglishText

# code to text
include("semantics.jl")
include("articulate.jl")
include("list.jl")
include("numeric.jl")
include("pluralize.jl")
include("quantity.jl")

# text to code
include("text.jl")

end # module


text:
# Coverage
# ========
#
# Coverage of intervals.
#
# This file is a part of BioJulia.
# License is MIT: https://github.com/BioJulia/Bio.jl/blob/master/LICENSE.md

"""
    coverage(intervals)

Compute the coverage of a collection of intervals and return an `IntervalCollection` that contains run-length encoded coverage data.

For example, given intervals like:

    [------]     [------------]
       [---------------]

This function would return a new set of disjoint intervals with annotated coverage like:

    [1][-2-][-1-][--2--][--1--]

# Example

```jldoctest
julia> intervals = [
           Interval("chr1", 1, 8),
           Interval("chr1", 4, 20),
           Interval("chr1", 14, 27)];

julia> coverage(intervals)
IntervalCollection{UInt32} with 5 intervals:
  chr1:1-3  .  1
  chr1:4-8  .  2
  chr1:9-13  .  1
  chr1:14-20  .  2
  chr1:21-27  .  1
```
"""
function coverage(stream, seqname_isless::Function=isless)
    cov = IntervalCollection{UInt32}()
    lasts = Int64[]

    stream_next = iterate(stream)
    if stream_next === nothing
        return cov
    end

    current_coverage = 0
    coverage_seqname = ""
    coverage_first = 0
    last_interval_first = 0
    interval, stream_state = stream_next
    stream_next = iterate(stream, stream_state)

    while true
        if seqname(interval) != coverage_seqname
            coverage_process_lasts_heap!(cov, current_coverage, coverage_seqname, coverage_first, lasts)
            if !(isempty(coverage_seqname) || seqname_isless(coverage_seqname, seqname(interval)))
                error("Intervals must be sorted to compute coverage.")
            end

            coverage_seqname = seqname(interval)
            current_coverage = 0
            coverage_first = 0
            last_interval_first = 0
        end

        if leftposition(interval) < last_interval_first
            error("Intervals must be sorted to compute coverage.")
        end

        if !isempty(lasts) && lasts[1] < first(interval)
            pos = DataStructures.heappop!(lasts)
            if first(interval) == pos + 1
                DataStructures.heappush!(lasts, last(interval))
                if stream_next === nothing
                    break
                end
                last_interval_first = first(interval)
                interval, stream_state = stream_next
                stream_next = iterate(stream, stream_state)
            elseif pos == coverage_first - 1
                current_coverage -= 1
            else
                @assert pos >= coverage_first
                push!(cov, Interval{UInt32}(coverage_seqname, coverage_first, pos, STRAND_BOTH, current_coverage))
                current_coverage -= 1
                coverage_first = pos + 1
            end
        else
            if coverage_first == 0
                coverage_first = first(interval)
                current_coverage = 1
            elseif coverage_first == first(interval)
                current_coverage += 1
            else
                if current_coverage > 0
                    push!(cov, Interval{UInt32}(coverage_seqname, coverage_first, first(interval) - 1, STRAND_BOTH, current_coverage))
                end
                current_coverage += 1
                coverage_first = first(interval)
            end

            DataStructures.heappush!(lasts, last(interval))
            if stream_next === nothing
                break
            end
            last_interval_first = first(interval)
            interval, stream_state = stream_next
            stream_next = iterate(stream, stream_state)
        end
    end

    coverage_process_lasts_heap!(cov, current_coverage, coverage_seqname, coverage_first, lasts)

    return cov
end

function coverage(ic::IntervalCollection)
    return coverage(ic, isless)
end

# Helper function for coverage. Process remaining interval end points after
# all intervals have been read.
function coverage_process_lasts_heap!(cov::IntervalCollection{UInt32}, current_coverage, coverage_seqname, coverage_first, lasts)
    while !isempty(lasts)
        pos = DataStructures.heappop!(lasts)
        if pos == coverage_first - 1
            current_coverage -= 1
        else
            @assert pos >= coverage_first
            push!(cov, Interval{UInt32}(coverage_seqname, coverage_first, pos, STRAND_BOTH, current_coverage))
            current_coverage -= 1
            coverage_first = pos + 1
        end
    end
    @assert current_coverage == 0
end


text:
# These functions do most of the work in the package.
# They are all type-stable recusive methods for performance and extensibility.

"""
    dimsmatch([f], dim, lookup) => Bool
    dimsmatch([f], dims::Tuple, lookups::Tuple) => Bool

Compare 2 dimensions or `Tuple` of `Dimension` are of the same base type,
or are at least rotations/transformations of the same type.

`f` is `<:` by default, but can be `>:` to match abstract types to concrete types.
"""
@inline dimsmatch(dims, lookups) = dimsmatch(<:, dims, lookups)
@inline dimsmatch(f::Function, dims::Tuple, lookups::Tuple) =
    all(map((d, l) -> dimsmatch(f, d, l), dims, lookups))
@inline dimsmatch(f::Function, dim, lookup) = dimsmatch(f, typeof(dim), typeof(lookup))
@inline dimsmatch(f::Function, dim::Type, lookup) = dimsmatch(f, dim, typeof(lookup))
@inline dimsmatch(f::Function, dim, lookup::Type) = dimsmatch(f, typeof(dim), lookup)
@inline dimsmatch(f::Function, dim::Nothing, lookup::Type) = false
@inline dimsmatch(f::Function, dim::Type, ::Nothing) = false
@inline dimsmatch(f::Function, dim, lookup::Nothing) = false
@inline dimsmatch(f::Function, dim::Nothing, lookup) = false
@inline dimsmatch(f::Function, dim::Nothing, lookup::Nothing) = false
@inline dimsmatch(f::Function, dim::Type{D}, match::Type{M}) where {D,M} =
    f(basetypeof(unwrap(D)), basetypeof(unwrap(M))) ||
    f(basetypeof(unwrap(D)), basetypeof(dims(modetype(unwrap(M))))) ||
    f(basetypeof(dims(modetype(unwrap(D)))), basetypeof(unwrap(M)))

"""
    key2dim(s::Symbol) => Dimension
    key2dim(dims...) => Tuple{Dimension,Vararg}
    key2dim(dims::Tuple) => Tuple{Dimension,Vararg}

Convert a symbol to a dimension object. `:X`, `:Y`, `:Ti` etc will be converted.
to `X()`, `Y()`, `Ti()`, as with any other dims generated with the [`@dim`](@ref) macro.

All other `Symbol`s `S` will generate `Dim{S}()` dimensions.
"""
@inline key2dim(t::Tuple) = map(key2dim, t)
@inline key2dim(s::Symbol) = key2dim(Val{s}())
# Allow other things to pass through
@inline key2dim(d::Val{<:Dimension}) = d
@inline key2dim(d) = d

"""
    dim2key(dim::Dimension) => Symbol
    dim2key(dims::Type{<:Dimension}) => Symbol

Convert a dimension object to a simbol. `X()`, `Y()`, `Ti()` etc will be converted.
to `:X`, `:Y`, `:Ti`, as with any other dims generated with the [`@dim`](@ref) macro.

All other `Dim{S}()` dimensions will generate `Symbol`s `S`.
"""
@inline dim2key(dim::Dimension) = dim2key(typeof(dim))
@inline dim2key(dim::Val{D}) where D <: Dimension = dim2key(D)
@inline dim2key(dt::Type{<:Dimension}) = Symbol(Base.nameof(dt))

"""
    sortdims([f], tosort, order) => Tuple

Sort dimensions `tosort` by `order`. Dimensions
in `order` but missing from `tosort` are replaced with `nothing`.

`tosort` and `order` can be `Tuple`s or `Vector`s or Dimension
or dimension type. Abstract supertypes like [`TimeDim`](@ref)
can be used in `order`.

`f` is `<:` by default, but can be `>:` to sort abstract types by concrete types.
"""
@inline sortdims(args...) = _call(_sortdims, MaybeFirst(), args...)

@inline _sortdims(f, tosort, order::Tuple{<:Integer,Vararg}) = map(p -> tosort[p], order)
@inline _sortdims(f, tosort, order) = _sortdims_gen(f, tosort, order)

@generated _sortdims_gen(f, tosort::Tuple, order::Tuple) = begin
    expr = Expr(:tuple)
    allreadyfound = Int[]
    for (i, ord) in enumerate(order.parameters)
        # Make sure we don't find the same dim twice
        found = 0
        while true
            found = findnext((tosort.parameters...,), found + 1) do s
                dimsmatch(_asfunc(f), s, ord)
            end
            if found == nothing
                push!(expr.args, :(nothing))
                break
            elseif !(found in allreadyfound)
                push!(expr.args, :(tosort[$found]))
                push!(allreadyfound, found)
                break
            end
        end
    end
    expr
end

"""
    dims(x, lookup) => Tuple{Vararg{<:Dimension}}
    dims(x, lookup...) => Tuple{Vararg{<:Dimension}}

Get the dimension(s) matching the type(s) of the lookup dimension.

Lookup can be an Int or an Dimension, or a tuple containing
any combination of either.

## Arguments
- `x`: any object with a `dims` method, or a `Tuple` of `Dimension`.
- `lookup`: Tuple or a single `Dimension` or `Dimension` `Type`.

## Example
```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

```
"""
@inline dims(args...) = _call(_dims, MaybeFirst(), args...)

@inline _dims(f, dims, lookup) = _remove_nothing(_sortdims(f, dims, lookup))

"""
    commondims([f], x, lookup) => Tuple{Vararg{<:Dimension}}

This is basically `dims(x, lookup)` where the order of the original is kept,
unlike [`dims`](@ref) where the lookup tuple determines the order

Also unlike `dims`,`commondims` always returns a `Tuple`, no matter the input.
No errors are thrown if dims are absent from either `x` or `lookup`.

`f` is `<:` by default, but can be `>:` to sort abstract types by concrete types.

```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

julia> commondims(A, X)
X

julia> commondims(A, (X, Z))
X, Z

julia> commondims(A, Ti)
()

```
"""
@inline commondims(args...) = _call(_commondims, AlwaysTuple(), args...)

_commondims(f, ds, lookup) = _dims(f, ds, _dims(_flip_subtype(f), lookup, ds))

"""
    dimnum(x, lookup::Tuple) => NTuple{Int}
    dimnum(x, lookup) => Int

Get the number(s) of `Dimension`(s) as ordered in the dimensions of an object.

## Arguments
- `x`: any object with a `dims` method, a `Tuple` of `Dimension` or a single `Dimension`.
- `lookup`: Tuple, Array or single `Dimension` or dimension `Type`.

The return type will be a Tuple of `Int` or a single `Int`,
depending on wether `lookup` is a `Tuple` or single `Dimension`.

## Example

```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

julia> dimnum(A, (Z, X, Y))
(3, 1, 2)

julia> dimnum(A, Y)
2
```
"""
@inline function dimnum(args...)
    all(hasdim(args...)) || _errorextradims()
    _call(_dimnum, MaybeFirst(), args...)
end

@inline function _dimnum(f::Function, ds::Tuple, lookups::Tuple{Vararg{Int}})
    lookups
end
@inline function _dimnum(f::Function, ds::Tuple, lookups::Tuple)
    numbered = map(ds, ntuple(identity, length(ds))) do d, i
        basetypeof(d)(i)
    end
    map(val, _dims(f, numbered, lookups))
end

"""
    hasdim([f], x, lookup::Tuple) => NTUple{Bool}
    hasdim([f], x, lookups...) => NTUple{Bool}
    hasdim([f], x, lookup) => Bool

Check if an object `x` has dimensions that match or inherit from the `lookup` dimensions.

## Arguments
- `x`: any object with a `dims` method, a `Tuple` of `Dimension` or a single `Dimension`.
- `lookup`: Tuple or single `Dimension` or dimension `Type`.
- `f`: `<:` by default, but can be `>:` to match abstract types to concrete types.

Check if an object or tuple contains an `Dimension`, or a tuple of dimensions.

## Example
```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

julia> hasdim(A, X)
true

julia> hasdim(A, (Z, X, Y))
(true, true, true)

julia> hasdim(A, Ti)
false
```
"""
@inline hasdim(args...) = _call(_hasdim, MaybeFirst(), args...)

@inline _hasdim(f, dims, lookup) =
    map(d -> !(d isa Nothing), _sortdims(f, _commondims(f, dims, lookup), lookup))
@inline _hasdim(f, dims, lookup::Tuple{Vararg{Int}}) =
    map(l -> l in 1:length(dims), lookup)

"""
    otherdims(x, lookup) => Tuple{Vararg{<:Dimension,N}}

Get the dimensions of an object _not_ in `lookup`.

## Arguments
- `x`: any object with a `dims` method, a `Tuple` of `Dimension`.
- `lookup`: Tuple or single `Dimension` or dimension `Type`.
- `f`: `<:` by default, but can be `>:` to match abstract types to concrete types.

A tuple holding the unmatched dimensions is always returned.

## Example
```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

julia> otherdims(A, X)
Y, Z

julia> otherdims(A, (Y, Z))
X

julia> otherdims(A, Ti)
X, Y, Z
```
"""
@inline otherdims(args...) = _call(_otherdims_presort, AlwaysTuple(), args...)

@inline _otherdims_presort(f, ds, lookup) = _otherdims(f, ds, _sortdims(f, lookup, ds))
# Work with a sorted lookup where the missing dims are `nothing`
@inline _otherdims(f, ds::Tuple, lookup::Tuple) =
    (_dimifmatching(f, first(ds), first(lookup))..., _otherdims(f, tail(ds), tail(lookup))...)
@inline _otherdims(f, dims::Tuple{}, ::Tuple{}) = ()

@inline _dimifmatching(f, dim, lookup) = dimsmatch(f, dim, lookup) ? () : (dim,)

"""
    setdims(X, newdims) => AbstractArray
    setdims(::Tuple, newdims) => Tuple{Vararg{<:Dimension,N}}

Replaces the first dim matching `<: basetypeof(newdim)` with newdim,
and returns a new object or tuple with the dimension updated.

## Arguments
- `x`: any object with a `dims` method, a `Tuple` of `Dimension` or a single `Dimension`.
- `newdim`: Tuple or single `Dimension`, `Type` or `Symbol`.

# Example
```jldoctest
using DimensionalData

A = DimArray(ones(10, 10), (X, Y(10:10:100)))
B = setdims(A, Y('a':'j'))
index(B, Y)

# output

'a':1:'j'
```
"""
@inline setdims(x, d1, d2, ds...) = setdims(x, (d1, d2, ds...))
@inline setdims(x, newdims::Dimension) = rebuild(x; dims=setdims(dims(x), key2dim(newdims)))
@inline setdims(x, newdims::Tuple) = rebuild(x; dims=setdims(dims(x), key2dim(newdims)))
@inline setdims(dims::Tuple, newdim::Dimension) = setdims(dims, (newdim,))
@inline setdims(dims::Tuple, newdims::Tuple) = swapdims(dims, sortdims(newdims, dims))
@inline setdims(dims::Tuple, newdims::Tuple{}) = dims

"""
    swapdims(x::T, newdims) => T
    swapdims(dims::Tuple, newdims) => Tuple{Vararg{<:Dimension}}

Swap dimensions for the passed in dimensions, in the
order passed.

Passing in the `Dimension` types rewraps the dimension index,
keeping the index values and metadata, while constructed `Dimension`
objectes replace the original dimension. `nothing` leaves the original
dimension as-is.

## Arguments
- `x`: any object with a `dims` method or a `Tuple` of `Dimension`.
- `newdim`: Tuple of `Dimension` or dimension `Type`.

# Example
```jldoctest
julia> using DimensionalData

julia> A = DimArray(ones(10, 10, 10), (X, Y, Z));

```
"""
@inline swapdims(x, d1, d2, ds...) = swapdims(x, (d1, d2, ds...))
@inline swapdims(x, newdims::Tuple) =
    rebuild(x; dims=formatdims(x, swapdims(dims(x), newdims)))
@inline swapdims(dims::DimTuple, newdims::Tuple) =
    map((d, nd) -> _swapdims(d, nd), dims, newdims)

@inline _swapdims(dim::Dimension, newdim::DimType) =
    basetypeof(newdim)(val(dim), mode(dim), metadata(dim))
@inline _swapdims(dim::Dimension, newdim::Dimension) = newdim
@inline _swapdims(dim::Dimension, newdim::Nothing) = dim

"""
    slicedims(x, I) => Tuple{Tuple,Tuple}
    slicedims(f, x, I) => Tuple{Tuple,Tuple}

Slice the dimensions to match the axis values of the new array.

All methods return a tuple conatining two tuples: the new dimensions,
and the reference dimensions. The ref dimensions are no longer used in
the new struct but are useful to give context to plots.

Called at the array level the returned tuple will also include the
previous reference dims attached to the array.

# Arguments

- `f`: a function `getindex`,  `view` or `dotview`. This will be used for slicing
    `getindex` is the default if `f` is not included.
- `x`: An `AbstractDimArray`, `Tuple` of `Dimension`, or `Dimension`
- `I`: A tuple of `Integer`, `Colon` or `AbstractArray`
"""
function slicedims end
@inline slicedims(args...) = slicedims(getindex, args...)
@inline slicedims(f::Function, x, i1, i2, I...) = slicedims(f, x, (i1, i2, I...))
@inline slicedims(f::Function, x, I::CartesianIndex) = slicedims(f, x, Tuple(I))
@inline slicedims(f::Function, x, I::Tuple) = _slicedims(f, dims(x), refdims(x), I)
@inline slicedims(f::Function, dims::Tuple, I::Tuple) = _slicedims(f, dims, I)
@inline slicedims(f::Function, dims::Tuple, refdims::Tuple, i1, I...) = slicedims(f, dims, refdims, (i1, I...))
@inline slicedims(f::Function, dims::Tuple, refdims::Tuple, I) = _slicedims(f, dims, refdims, I)
@inline slicedims(f::Function, dims::Tuple, refdims::Tuple, I::CartesianIndex) =
    slicedims(f, dims, refdims, Tuple(I))

@inline _slicedims(f, dims::Tuple, refdims::Tuple, I::Tuple) = begin
    newdims, newrefdims = _slicedims(f, dims, I)
    newdims, (refdims..., newrefdims...)
end
@inline _slicedims(f, dims::Tuple, refdims::Tuple, I::Tuple{}) = dims, refdims
@inline _slicedims(f, dims::DimTuple, I::Tuple{}) = dims, ()
@inline _slicedims(f, dims::DimTuple, I::Tuple) = begin
    d = _slicedims(f, first(dims), first(I))
    ds = _slicedims(f, tail(dims), tail(I))
    (d[1]..., ds[1]...), (d[2]..., ds[2]...)
end
@inline _slicedims(f, dims::Tuple{}, I::Tuple) = (), ()
@inline _slicedims(f, dims::Tuple{}, I::Tuple{}) = (), ()

@inline _slicedims(f, d::Dimension, i) = _slicedims(f, mode(d), d, i)
@inline _slicedims(f, ::IndexMode, d::Dimension, i::Colon) = (d,), ()
@inline _slicedims(f, ::IndexMode, d::Dimension, i::Integer) =
    (), (rebuild(d, d[relate(d, i)], _slicemode(mode(d), val(d), i)),)
@inline _slicedims(f, ::NoIndex, d::Dimension, i::Integer) = (), (rebuild(d, i),)
# TODO deal with unordered arrays trashing the index order
@inline _slicedims(f, ::IndexMode, d::Dimension{<:Val{Index}}, i::AbstractArray) where Index =
    (rebuild(d, Val{Index[relate(d, i)]}(), _slicemode(mode(d), val(d), i)),), ()
@inline _slicedims(f, ::IndexMode, d::Dimension{<:AbstractArray}, i::AbstractArray) =
    (rebuild(d, f(val(d), relate(d, i)), _slicemode(mode(d), val(d), i)),), ()
@inline _slicedims(f, ::NoIndex, d::Dimension{<:AbstractArray}, i::AbstractArray) =
    (rebuild(d, f(val(d), relate(d, i))),), ()
# Should never happen, just for ambiguity
@inline _slicedims(f, ::NoIndex, d::Dimension{<:Val}, i::AbstractArray) =
    (rebuild(d, f(val(d), relate(d, i))),), ()

@inline relate(d::Dimension, i) = _maybeflipindex(relation(d), d, i)

@inline _maybeflipindex(::ForwardRelation, d, i) = i
@inline _maybeflipindex(::ReverseRelation, d, i::Integer) = lastindex(d) - i + 1
@inline _maybeflipindex(::ReverseRelation, d, i::AbstractArray) =
    reverse(lastindex(d) .- i .+ 1)

"""
    reducedims(x, dimstoreduce) => Tuple{Vararg{<:Dimension}}

Replace the specified dimensions with an index of length 1.
This is usually to match a new array size where an axis has been
reduced with a method like `mean` or `reduce` to a length of 1,
but the number of dimensions has not changed.

`IndexMode` traits are also updated to correspond to the change in
cell step, sampling type and order.
"""
@inline reducedims(x, dimstoreduce) = _reducedims(x, key2dim(dimstoreduce))

@inline _reducedims(x, dimstoreduce) = _reducedims(x, (dimstoreduce,))
@inline _reducedims(x, dimstoreduce::Tuple) = _reducedims(dims(x), dimstoreduce)
@inline _reducedims(dims::DimTuple, dimstoreduce::Tuple) =
    map(_reducedims, dims, sortdims(dimstoreduce, dims))
# Map numbers to corresponding dims. Not always type-stable
@inline _reducedims(dims::DimTuple, dimstoreduce::Tuple{Vararg{Int}}) =
    map(_reducedims, dims, sortdims(map(i -> dims[i], dimstoreduce), dims))
# Reduce matching dims but ignore nothing vals - they are the dims not being reduced
@inline _reducedims(dim::Dimension, ::Nothing) = dim
@inline _reducedims(dim::Dimension, ::DimOrDimType) = _reducedims(mode(dim), dim)
@inline _reducedims(mode::NoIndex, dim::Dimension) = rebuild(dim, Base.OneTo(1), NoIndex())
# TODO what should this do?
@inline _reducedims(mode::Unaligned, dim::Dimension) = rebuild(dim, [nothing], NoIndex)
# Categories are combined.
@inline _reducedims(mode::Categorical, dim::Dimension{Vector{String}}) =
    rebuild(dim, ["combined"], Categorical())
@inline _reducedims(mode::Categorical, dim::Dimension) =
    rebuild(dim, [:combined], Categorical())
@inline _reducedims(mode::AbstractSampled, dim::Dimension) =
    _reducedims(span(mode), sampling(mode), mode, dim)
@inline _reducedims(::Irregular, ::Points, mode::AbstractSampled, dim::Dimension) =
    rebuild(dim, _reducedims(Center(), dim::Dimension), mode)
@inline _reducedims(::Irregular, ::Intervals, mode::AbstractSampled, dim::Dimension) = begin
    newmode = rebuild(mode; order=Ordered())
    rebuild(dim, _reducedims(locus(mode), dim), newmode)
end
@inline _reducedims(span::Regular, ::Sampling, mode::AbstractSampled, dim::Dimension) = begin
    newspan = Regular(step(span) * length(dim))
    newmode = rebuild(mode; order=Ordered(), span=newspan)
    rebuild(dim, _reducedims(locus(mode), dim), newmode)
end
@inline _reducedims(
    span::Regular{<:Dates.CompoundPeriod}, ::Sampling, mode::AbstractSampled, dim::Dimension
) = begin
    newspan = Regular(Dates.CompoundPeriod(step(span).periods .* length(dim)))
    newmode = rebuild(mode; order=Ordered(), span=newspan)
    rebuild(dim, _reducedims(locus(mode), dim), newmode)
end
@inline _reducedims(span::Explicit, ::Intervals, mode::AbstractSampled, dim::Dimension) = begin
    index = _reducedims(locus(mode), dim)
    bnds = val(span)
    newspan = Explicit(reshape([bnds[1, 1]; bnds[2, end]], 2, 1))
    newmode = rebuild(mode; order=Ordered(), span=newspan)
    rebuild(dim, index, newmode)
end
# Get the index value at the reduced locus.
# This is the start, center or end point of the whole index.
@inline _reducedims(locus::Start, dim::Dimension) = [first(index(dim))]
@inline _reducedims(locus::End, dim::Dimension) = [last(index(dim))]
@inline _reducedims(locus::Center, dim::Dimension) = begin
    index = val(dim)
    len = length(index)
    if iseven(len)
        _centerval(index, len)
    else
        [index[len ÷ 2 + 1]]
    end
end

# Need to specialise for more types
@inline _centerval(index::AbstractArray{<:AbstractFloat}, len) =
    [(index[len ÷ 2] + index[len ÷ 2 + 1]) / 2]
@inline _centerval(index::AbstractArray, len) = [index[len ÷ 2 + 1]]

"""
    comparedims(A::AbstractDimArray...)
    comparedims(A::Tuple...)
    comparedims(a, b)

Check that dimensions or tuples of dimensions are the same,
and return the first valid dimension. If `AbstractDimArray`s
are passed as arguments their dimensions are compared.

Empty tuples and `nothing` dimension values are ignored,
returning the `Dimension` value if it exists.
"""
function comparedims end
@inline comparedims(x...; kw...) = comparedims(x; kw...)
@inline comparedims(A::Tuple; kw...) = comparedims(map(dims, A)...; kw...)
@inline comparedims(dims::Vararg{<:Tuple{Vararg{<:Dimension}}}; kw...) =
    map(d -> comparedims(first(dims), d), dims; kw...) |> first

@inline comparedims(a::DimTuple, ::Nothing; kw...) = a
@inline comparedims(::Nothing, b::DimTuple; kw...) = b
@inline comparedims(::Nothing, ::Nothing; kw...) = nothing
# Cant use `map` here, tuples may not be the same length
@inline comparedims(a::DimTuple, b::DimTuple; kw...) =
    (comparedims(first(a), first(b); kw...), comparedims(tail(a), tail(b); kw...)...)
@inline comparedims(a::DimTuple, b::Tuple{}; kw...) = a
@inline comparedims(a::Tuple{}, b::DimTuple; kw...) = b
@inline comparedims(a::Tuple{}, b::Tuple{}; kw...) = ()
@inline comparedims(a::AnonDim, b::AnonDim; kw...) = nothing
@inline comparedims(a::Dimension, b::AnonDim; kw...) = a
@inline comparedims(a::AnonDim, b::Dimension; kw...) = b
@inline function comparedims(a::Dimension, b::Dimension;
    type=true, length=true, mode=false, val=false, metadata=false
)
    type && basetypeof(a) != basetypeof(b) && _dimsmismatcherror(a, b)
    mode && typeof(DD.mode(a)) != typeof(DD.mode(b)) && _modeerror(a, b)
    length && Base.length(a) != Base.length(b) && _dimsizeerror(a, b)
    val && DD.val(a) != DD.val(b) && _valerror(a, b)
    metadata && DD.metadata(a) != DD.metadata(b) && _metadataerror(a, b)
    return a
end

"""
    combinedims(xs; check=true)

Combine the dimensions of each object in `xs`, in the order they are found.
"""
function combinedims end
# @inline combinedims(xs::Tuple) = combinedims(xs...)
@inline combinedims(xs...; kw...) = combinedims(map(dims, xs)...; kw...)
@inline combinedims(dt1::DimTuple; kw...) = dt1
@inline combinedims(dt1::DimTuple, dt2::DimTuple, dimtuples::DimTuple...; kw...) =
    reduce((dt2, dimtuples...); init=dt1) do dims1, dims2
        _combinedims(dims1, dims2; kw...)
    end
# Cant use `map` here, tuples may not be the same length
@inline _combinedims(a::DimTuple, b::DimTuple; check=true, kw...) = begin
    if check # Check the matching dims are the same
        common = commondims(a, b)
        comparedims(dims(a, common), dims(b, common); kw...)
    end
    # Take them from a, and add any extras from b
    (a..., otherdims(b, a)...)
end

"""
    dimstride(x, dim) => Int

Get the stride of the dimension relative to the other dimensions.

This may or may not be equal to the stride of the related array,
although it will be for `Array`.

## Arguments

- `x` is any object with a `dims` method, or a `Tuple` of `Dimension`.
- `dim` is a `Dimension`, `Dimension` type, or and `Int`. Using an `Int` is not type-stable.
"""
@inline dimstride(x, n) = dimstride(dims(x), n)
@inline dimstride(::Nothing, n) = _dimsnotdefinederror()
@inline dimstride(dims::DimTuple, d::DimOrDimType) = dimstride(dims, dimnum(dims, d))
@inline dimstride(dims::DimTuple, n::Int) = prod(map(length, dims)[1:n-1])

@inline basedims(x) = basedims(dims(x))
@inline basedims(ds::Tuple) = map(basedims, ds)
@inline basedims(d::Dimension) = basetypeof(d)()
@inline basedims(d::Symbol) = key2dim(d)
@inline basedims(T::Type{<:Dimension}) = basetypeof(T)()

# Utils
struct MaybeFirst end
struct AlwaysTuple end

# Call the function f with stardardised args
@inline _call(f::Function, t, args...) = _call(f, t, <:, _wraparg(args...)...)
@inline _call(f::Function, t, op::Function, args...) = _call1(f, t, op, _wraparg(args...)...)

@inline _call1(f, t, op::Function, x, l1, l2, ls...) = _call1(f, t, op, x, (l1, l2, ls...))
@inline _call1(f, t, op::Function, x, lookup) = _call1(f, t, op, dims(x), lookup)
@inline _call1(f, t, op::Function, x::Nothing, lookup) = _dimsnotdefinederror()
@inline _call1(f, t, op::Function, d::Tuple, lookup) = _call1(f, t, op, d, dims(lookup))
@inline _call1(f, t::AlwaysTuple, op::Function, d::Tuple, lookup::Union{Dimension,DimType,Val,Integer}) =
    _call1(f, t, op, d, (lookup,))
@inline _call1(f, t::MaybeFirst, op::Function, d::Tuple, lookup::Union{Dimension,DimType,Val,Integer}) =
    _call1(f, t, op, d, (lookup,)) |> _maybefirst
@inline _call1(f, t, op::Function, d::Tuple, lookup::Tuple) = map(unwrap, f(op, d, lookup))

_maybefirst(xs::Tuple) = first(xs)
_maybefirst(::Tuple{}) = nothing

@inline _kwdims(kw::Base.Iterators.Pairs) = _kwdims(kw.data)
@inline _kwdims(kw::NamedTuple{Keys}) where Keys = _kwdims(key2dim(Keys), values(kw))
@inline _kwdims(dims::Tuple, vals::Tuple) =
    (rebuild(first(dims), first(vals)), _kwdims(tail(dims), tail(vals))...)
@inline _kwdims(dims::Tuple{}, vals::Tuple{}) = ()

@inline _pairdims(pairs::Pair...) = map(p -> basetypeof(key2dim(first(p)))(last(p)), pairs)

@inline _remove_nothing(xs::Tuple) = _remove_nothing(xs...)
@inline _remove_nothing(x, xs...) = (x, _remove_nothing(xs...)...)
@inline _remove_nothing(::Nothing, xs...) = _remove_nothing(xs...)
@inline _remove_nothing() = ()

# This looks ridiculous, but gives seven arguments with constant-propagation,
# which means type stability using Symbols/types instead of objects.
@inline _wraparg(d1, d2, d3, d4, d5, d6, d7, ds...) =
    (_w(d1), _w(d2), _w(d3), _w(d4), _w(d5), _w(d6), _w(d7), _wraparg(ds...)...)
@inline _wraparg(d1, d2, d3, d4, d5, d6) = _w(d1), _w(d2), _w(d3), _w(d4), _w(d5), _w(d6)
@inline _wraparg(d1, d2, d3, d4, d5) = _w(d1), _w(d2), _w(d3), _w(d4), _w(d5)
@inline _wraparg(d1, d2, d3, d4) = _w(d1), _w(d2), _w(d3), _w(d4)
@inline _wraparg(d1, d2, d3) = _w(d1), _w(d2), _w(d3)
@inline _wraparg(d1, d2) = _w(d1), _w(d2)
@inline _wraparg(d1) = (_w(d1),)
@inline _wraparg() = ()

@inline _w(t::Tuple) = _wraparg(t...)
@inline _w(x) = x
@inline _w(s::Symbol) = key2dim(s)
@inline _w(x::Type{T}) where T = Val{T}()

@inline _asfunc(::Type{typeof(<:)}) = <:
@inline _asfunc(::Type{typeof(>:)}) = >:

@inline _flip_subtype(::typeof(<:)) = >:
@inline _flip_subtype(::typeof(>:)) = <:

# Error methods. @noinline to avoid allocations.

@noinline _dimsnotdefinederror() = throw(ArgumentError("Object does not define a `dims` method"))
@noinline _dimsmismatcherror(a, b) = throw(DimensionMismatch("$(basetypeof(a)) and $(basetypeof(b)) for dims on the same axis"))
@noinline _dimsizeerror(a, b) = throw(DimensionMismatch("Found both lengths $(length(a)) and $(length(b)) for $(basetypeof(a))"))
@noinline _modeerror(a, b) = throw(DimensionMismatch("Mode $(mode(a)) and $(mode(b)) do not match"))
@noinline _metadataerror(a, b) = throw(DimensionMismatch("Metadata $(metadata(a)) and $(madata(b)) do not match"))
@noinline _valerror(a, b) = throw(DimensionMismatch("Dimension index $(val(a)) and $(val(b)) do not match"))
@noinline _warnextradims(extradims) = @warn "$(map(basetypeof, extradims)) dims were not found in object"
@noinline _errorextradims() = throw(ArgumentError("Some dims were not found in object"))


text:
using Documenter, IRTools

makedocs(
  modules=[IRTools],
  sitename="IRTools",
  pages = [
        "Home" => "index.md",
        "Dynamo" => "dynamo.md",
        "Reference" => "reference.md"],
  format = Documenter.HTML(prettyurls = haskey(ENV, "CI")))

deploydocs(
    repo = "github.com/MikeInnes/IRTools.jl.git",
)


text:
# Support for ZeroMQ, a network and interprocess communication library

module ZMQ

using BinDeps
@BinDeps.load_dependencies

import Base: convert, ref, get, bytestring, length, size, stride, similar, getindex, setindex!, fd, wait

export
    #Types
    StateError,Context,Socket,Message,
    #functions
    close, get, set, bind, connect,send,recv,convert, ref,
    #Constants
    IO_THREADS,MAX_SOCKETS,PAIR,PUB,SUB,REQ,REP,ROUTER,DEALER,PULL,PUSH,XPUB,XPUB,XREQ,XREP,UPSTREAM,DOWNSTREAM,MORE,MORE,SNDMORE,POLLIN,POLLOUT,POLLERR,STREAMER,FORWARDER,QUEUE

# A server will report most errors to the client over a Socket, but
# errors in ZMQ state can't be reported because the socket may be
# corrupted. Therefore, we need an exception type for errors that
# should be reported locally.
type StateError <: Exception
    msg::String
end
show(io, thiserr::StateError) = print(io, "ZMQ: ", thiserr.msg)

# Basic functions
function jl_zmq_error_str()
    errno = ccall((:zmq_errno, zmq), Cint, ())
    c_strerror = ccall ((:zmq_strerror, zmq), Ptr{Uint8}, (Cint,), errno)
    if c_strerror != C_NULL
        strerror = bytestring(c_strerror)
        return strerror
    else
        return "Unknown error"
    end
end

const version = let major = zeros(Cint, 1), minor = zeros(Cint, 1), patch = zeros(Cint, 1)
    ccall((:zmq_version, zmq), Void, (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch)
    VersionNumber(major[1], minor[1], patch[1])
end

# define macro to enable version specific code
macro v2only(ex)
    version.major == 2 ? esc(ex) : :nothing
end
macro v3only(ex)
    version.major >= 3 ? esc(ex) : :nothing
end

## Sockets ##
type Socket
    data::Ptr{Void}

    # ctx should be ::Context, but forward type references are not allowed
    function Socket(ctx, typ::Integer)
        p = ccall((:zmq_socket, zmq), Ptr{Void},  (Ptr{Void}, Cint), ctx.data, typ)
        if p == C_NULL
            throw(StateError(jl_zmq_error_str()))
        end
        socket = new(p)
        finalizer(socket, close)
        push!(ctx.sockets, socket)
        return socket
    end
end

function close(socket::Socket)
    if socket.data != C_NULL
        rc = ccall((:zmq_close, zmq), Cint,  (Ptr{Void},), socket.data)
        if rc != 0
            throw(StateError(jl_zmq_error_str()))
        end
        socket.data = C_NULL
    end
end

## Contexts ##
# Provide the same constructor API for version 2 and version 3, even
# though the underlying functions are changing
type Context
    data::Ptr{Void}

    # need to keep a list of sockets for this Context in order to
    # close them before finalizing (otherwise zmq_term will hang)
    sockets::Vector{Socket}

    function Context(n::Integer)
        @v2only p = ccall((:zmq_init, zmq), Ptr{Void},  (Cint,), n)
        @v3only p = ccall((:zmq_ctx_new, zmq), Ptr{Void},  ())
        if p == C_NULL
            throw(StateError(jl_zmq_error_str()))
        end
        zctx = new(p, Array(Socket,0))
        finalizer(zctx, close)
        return zctx
    end
end
Context() = Context(1)

function close(ctx::Context)
    if ctx.data != C_NULL # don't close twice!
        for s in ctx.sockets
            close(s)
        end
        @v2only rc = ccall((:zmq_term, zmq), Cint,  (Ptr{Void},), ctx.data)
        @v3only rc = ccall((:zmq_ctx_destroy, zmq), Cint,  (Ptr{Void},), ctx.data)
        if rc != 0
            throw(StateError(jl_zmq_error_str()))
        end
        ctx.data = C_NULL
    end
end
term(ctx::Context) = close(ctx)

@v3only begin
function get(ctx::Context, option::Integer)
    val = ccall((:zmq_ctx_get, zmq), Cint, (Ptr{Void}, Cint), ctx.data, option)
    if val < 0
        throw(StateError(jl_zmq_error_str()))
    end
    return val
end

function set(ctx::Context, option::Integer, value::Integer)
    rc = ccall((:zmq_ctx_set, zmq), Cint, (Ptr{Void}, Cint, Cint), ctx.data, option, value)
    if rc != 0
        throw(StateError(jl_zmq_error_str()))
    end
end
end # end v3only

# Getting and setting socket options
# Socket options of integer type
let u64p = zeros(Uint64, 1), i64p = zeros(Int64, 1), ip = zeros(Cint, 1), u32p = zeros(Uint32, 1), sz = zeros(Uint, 1),
    pp = zeros(Ptr{Void},1)
opslist = {
    (:set_affinity,                :get_affinity,                 4, u64p)
    (:set_type,                    :get_type,                    16,   ip)
    (:set_linger,                  :get_linger,                  17,   ip)
    (:set_reconnect_ivl,           :get_reconnect_ivl,           18,   ip)
    (:set_backlog,                 :get_backlog,                 19,   ip)
    (:set_reconnect_ivl_max,       :get_reconnect_ivl_max,       21,   ip)
  }

@unix_only opslist = vcat(opslist, (nothing,     :get_fd,        14,   ip))
@windows_only opslist = vcat(opslist, (nothing,  :get_fd,        14,   pp))

if version.major == 2
    opslist = vcat(opslist, {
    (:set_hwm,                     :get_hwm,                      1, u64p)
    (:set_swap,                    :get_swap,                     3, i64p)
    (:set_rate,                    :get_rate,                     8, i64p)
    (:set_recovery_ivl,            :get_recovery_ivl,             9, i64p)
    (:_zmq_setsockopt_mcast_loop,  :_zmq_getsockopt_mcast_loop,  10, i64p)
    (:set_sndbuf,                  :get_sndbuf,                  11, u64p)
    (:set_rcvbuf,                  :get_rcvbuf,                  12, u64p)
    (nothing,                      :_zmq_getsockopt_rcvmore,     13, i64p)
    (nothing,                      :get_events,                  15, u32p)
    (:set_recovery_ivl_msec,       :get_recovery_ivl_msec,       20, i64p)
    })
else
    opslist = vcat(opslist, {
    (:set_rate,                    :get_rate,                     8,   ip)
    (:set_recovery_ivl,            :get_recovery_ivl,             9,   ip)
    (:set_sndbuf,                  :get_sndbuf,                  11,   ip)
    (:set_rcvbuf,                  :get_rcvbuf,                  12,   ip)
    (nothing,                      :_zmq_getsockopt_rcvmore,     13,   ip)
    (nothing,                      :get_events,                  15,   ip)
    (:set_maxmsgsize,              :get_maxmsgsize,              22,   ip)
    (:set_sndhwm,                  :get_sndhwm,                  23,   ip)
    (:set_rcvhwm,                  :get_rcvhwm,                  24,   ip)
    (:set_multicast_hops,          :get_multicast_hops,          25,   ip)
    (:set_ipv4only,                :get_ipv4only,                31,   ip)
    (:set_tcp_keepalive,           :get_tcp_keepalive,           34,   ip)
    (:set_tcp_keepalive_idle,      :get_tcp_keepalive_idle,      35,   ip)
    (:set_tcp_keepalive_cnt,       :get_tcp_keepalive_cnt,       36,   ip)
    (:set_tcp_keepalive_intvl,     :get_tcp_keepalive_intvl,     37,   ip)
    })
end
if version > v"2.1"
    opslist = vcat(opslist, {
    (:set_rcvtimeo,                :get_rcvtimeo,                27,   ip)
    (:set_sndtimeo,                :get_sndtimeo,                28,   ip)
    })
end

for (fset, fget, k, p) in opslist
    if fset != nothing
        @eval global ($fset)
        @eval function ($fset)(socket::Socket, option_val::Integer)
            ($p)[1] = option_val
            rc = ccall((:zmq_setsockopt, zmq), Cint,
                       (Ptr{Void}, Cint, Ptr{Void}, Uint),
                       socket.data, $k, $p, sizeof(eltype($p)))
            if rc != 0
                throw(StateError(jl_zmq_error_str()))
            end
        end
    end
    if fget != nothing
        @eval global($fget)
        @eval function ($fget)(socket::Socket)
            ($sz)[1] = sizeof(eltype($p))
            rc = ccall((:zmq_getsockopt, zmq), Cint,
                       (Ptr{Void}, Cint, Ptr{Void}, Ptr{Uint}),
                       socket.data, $k, $p, $sz)
            if rc != 0
                throw(StateError(jl_zmq_error_str()))
            end
            return int(($p)[1])
        end
    end
end
# For some functions, the publicly-visible versions should require &
# return boolean
if version.major == 2
    global set_mcast_loop
    set_mcast_loop(socket::Socket, val::Bool) = _zmq_setsockopt_mcast_loop(socket, val)
    global get_mcast_loop
    get_mcast_loop(socket::Socket) = bool(_zmq_getsockopt_mcast_loop(socket))
end
end  # let
# More functions with boolean prototypes
get_rcvmore(socket::Socket) = bool(_zmq_getsockopt_rcvmore(socket))
# And a convenience function
ismore(socket::Socket) = get_rcvmore(socket)

# Raw FD access
@unix_only fd(socket::Socket) = RawFD(get_fd(socket))
@windows_only fd(socket::Socket) = WindowsRawSocket(get_fd(socket))
wait(socket::Socket; readable=false, writable=false) = wait(fd(socket); readable=readable, writable=writable)

# Socket options of string type
let u8ap = zeros(Uint8, 255), sz = zeros(Uint, 1)
opslist = {
    (:set_identity,                :get_identity,                5)
    (:set_subscribe,               nothing,                      6)
    (:set_unsubscribe,             nothing,                      7)
    }
if version.major >= 3
    opslist = vcat(opslist, {
    (nothing,                      :get_last_endpoint,          32)
    (:set_tcp_accept_filter,       nothing,                     38)
    })
end
for (fset, fget, k) in opslist
    if fset != nothing
        @eval global ($fset)
        @eval function ($fset)(socket::Socket, option_val::ByteString)
            if length(option_val) > 255
                throw(StateError("option value too large"))
            end
            rc = ccall((:zmq_setsockopt, zmq), Cint,
                       (Ptr{Void}, Cint, Ptr{Uint8}, Uint),
                       socket.data, $k, option_val, length(option_val))
            if rc != 0
                throw(StateError(jl_zmq_error_str()))
            end
        end
    end
    if fget != nothing
        @eval global ($fget)
        @eval function ($fget)(socket::Socket)
            ($sz)[1] = length($u8ap)
            rc = ccall((:zmq_getsockopt, zmq), Cint,
                       (Ptr{Void}, Cint, Ptr{Uint8}, Ptr{Uint}),
                       socket.data, $k, $u8ap, $sz)
            if rc != 0
                throw(StateError(jl_zmq_error_str()))
            end
            return bytestring(convert(Ptr{Uint8}, $u8ap), int(($sz)[1]))
        end
    end
end
end  # let

function bind(socket::Socket, endpoint::String)
    rc = ccall((:zmq_bind, zmq), Cint, (Ptr{Void}, Ptr{Uint8}), socket.data, endpoint)
    if rc != 0
        throw(StateError(jl_zmq_error_str()))
    end
end

function connect(socket::Socket, endpoint::String)
    rc=ccall((:zmq_connect, zmq), Cint, (Ptr{Void}, Ptr{Uint8}), socket.data, endpoint)
    if rc != 0
        throw(StateError(jl_zmq_error_str()))
    end
end

## Messages ##
type Message <: AbstractArray{Uint8,1}
    # 32 bytes (for v3) + a pointer (for v2)
    w0::Int64
    w1::Int64
    w2::Int64
    w3::Int64
    w4::Int
    bufferorigin::Any # prevent source of buffer (if any) from being gc'ed

    # Create an empty message (for receive)
    function Message()
        zmsg = new()
        rc = ccall((:zmq_msg_init, zmq), Cint, (Ptr{Message},), &zmsg)
        if rc != 0
            throw(StateError(jl_zmq_error_str()))
        end
        finalizer(zmsg, close)
        return zmsg
    end
    # Create a message with a given buffer size (for send)
    function Message(len::Integer)
        zmsg = new()
        rc = ccall((:zmq_msg_init_size, zmq), Cint, (Ptr{Message}, Csize_t), &zmsg, len)
        if rc != 0
            throw(StateError(jl_zmq_error_str()))
        end
        finalizer(zmsg, close)
        return zmsg
    end
    # Create a message with a given String or Array as a buffer (for send)
    Message(m::ByteString) = Message(m, convert(Ptr{Uint8}, m), sizeof(m))
    function Message{T}(origin, m::Ptr{T}, len::Integer)
        zmsg = new()
        zmsg.bufferorigin = origin # should be origin of data pointed to by m
        rc = ccall((:zmq_msg_init_data, zmq), Cint, (Ptr{Message}, Ptr{T}, Csize_t, Ptr{Void}, Ptr{Void}), &zmsg, m, len, C_NULL, C_NULL)
        if rc != 0
            throw(StateError(jl_zmq_error_str()))
        end
        finalizer(zmsg, close)
        return zmsg
    end
    Message(m::ByteString) = Message(m, convert(Ptr{Uint8}, m), sizeof(m))
    Message{T}(m::Array{T}) = Message(m, convert(Ptr{T}, m),  sizeof(m))
    function Message(io::IOBuffer)
        if !io.readable || !io.seekable
            error("byte read failed")
        end
        Message(io, convert(Ptr{Uint8}, io.data), io.size)
    end
end

# AbstractArray behaviors:
similar(a::Message, T, dims::Dims) = Array(T, dims) # ?
length(zmsg::Message) = int(ccall((:zmq_msg_size, zmq), Csize_t, (Ptr{Message},), &zmsg))
size(zmsg::Message) = (length(zmsg),)
convert(::Type{Ptr{Uint8}}, zmsg::Message) = ccall((:zmq_msg_data, zmq), Ptr{Uint8}, (Ptr{Message},), &zmsg)
function getindex(a::Message, i::Integer)
    if i < 1 || i > length(a)
        throw(BoundsError())
    end
    unsafe_load(pointer(a), i)
end
function setindex!(a::Message, v, i::Integer)
    if i < 1 || i > length(a)
        throw(BoundsError())
    end
    unsafe_store(pointer(a), v, i)
end

# Convert message to string (copies data)
bytestring(zmsg::Message) = bytestring(pointer(zmsg), length(zmsg))

# Build an IOStream from a message
# Copies the data
function convert(::Type{IOStream}, zmsg::Message)
    s = IOBuffer()
    write(s, zmsg)
    return s
end
# Close a message. You should not need to call this manually (let the
# finalizer do it).
function close(zmsg::Message)
    rc = ccall((:zmq_msg_close, zmq), Cint, (Ptr{Message},), &zmsg)
    if rc != 0
        throw(StateError(jl_zmq_error_str()))
    end
end

@v3only begin
function get(zmsg::Message, property::Integer)
    val = ccall((:zmq_msg_get, zmq), Cint, (Ptr{Message}, Cint), &zmsg, property)
    if val < 0
        throw(StateError(jl_zmq_error_str()))
    end
    val
end
function set(zmsg::Message, property::Integer, value::Integer)
    rc = ccall((:zmq_msg_set, zmq), Cint, (Ptr{Message}, Cint, Cint), &zmsg, property, value)
    if rc < 0
        throw(StateError(jl_zmq_error_str()))
    end
end
end # end v3only

## Send/receive messages
#
# Julia defines two types of ZMQ messages: "raw" and "serialized". A "raw"
# message is just a plain ZeroMQ message, used for sending a sequence
# of bytes. You send these with the following:
#   send(socket, zmsg)
#   zmsg = recv(socket)

#Send/Recv Options
const NOBLOCK = 1
const DONTWAIT = 1
const SNDMORE = 2

@v2only begin
function send(socket::Socket, zmsg::Message, flag=int32(0))
    rc = ccall((:zmq_send, zmq), Cint, (Ptr{Void}, Ptr{Message}, Cint),
               socket.data, &zmsg, flag)
    if rc != 0
        throw(StateError(jl_zmq_error_str()))
    end
end
send(socket::Socket, msg::String, flag=int32(0)) =
  send(socket, Message(msg), flag)
send{T}(socket::Socket, msg::Array{T}, flag=int32(0)) =
  send(socket, Message(msg), flag)
end # end v2only

@v3only begin
function send(socket::Socket, zmsg::Message, flag=int32(0))
    if (get_events(socket) & POLLOUT) == 0
        wait(socket; writable = true)
    end
    rc = ccall((:zmq_msg_send, zmq), Cint, (Ptr{Void}, Ptr{Message}, Cint),
                &zmsg, socket.data, flag)
    if rc == -1
        throw(StateError(jl_zmq_error_str()))
    end
end
function send{T}(socket::Socket, msg::Ptr{T}, len, flag=int32(0))
    if (get_events(socket) & POLLOUT) == 0
        wait(socket; writable = true)
    end
    rc = ccall((:zmq_send, zmq), Cint,
            (Ptr{Void}, Ptr{T}, Csize_t, Cint),
            socket.data, msg, len, flag)
    if rc == -1
        throw(StateError(jl_zmq_error_str()))
    end
end
send(socket::Socket, msg::String, flag=int32(0)) = send(socket, convert(Ptr{Uint8}, msg), sizeof(msg), flag)
send{T}(socket::Socket, msg::Array{T}, flag=int32(0)) = send(socket, convert(Ptr{T}, msg), sizeof(msg), flag)
end # end v3only

@v2only begin
function recv(socket::Socket)
    zmsg = Message()
    while true
        rc = ccall((:zmq_recv, zmq), Cint, (Ptr{Void}, Ptr{Message},  Cint),
                    socket.data, &zmsg, NOBLOCK)
        if rc != 0
            if errno() == Base.EAGAIN
                while (get_events(socket) & POLLIN) == 0
                    wait(socket; readable = true)
                end
                continue
            end
            throw(StateError(jl_zmq_error_str()))
        end
        break
    end
    return zmsg
end
end # end v2only

@v3only begin
function recv(socket::Socket)
    zmsg = Message()
    while true
        rc = ccall((:zmq_msg_recv, zmq), Cint, (Ptr{Message}, Ptr{Void}, Cint),
                    &zmsg, socket.data, NOBLOCK)
        if rc == -1
            if errno() == Base.EAGAIN
                while (get_events(socket) & POLLIN) == 0
                    wait(socket; readable = true)
                end
                continue
            end
            throw(StateError(jl_zmq_error_str()))
        end
        break
    end
    return zmsg
end
end # end v3only

## Constants

# Context options
const IO_THREADS = 1
const MAX_SOCKETS = 2

#Socket Types
const PAIR = 0
const PUB = 1
const SUB = 2
const REQ = 3
const REP = 4
const DEALER = 5
const ROUTER = 6
const PULL = 7
const PUSH = 8
const XPUB = 9
const XSUB = 10
const XREQ = DEALER
const XREP = ROUTER
const UPSTREAM = PULL
const DOWNSTREAM = PUSH

#Message options
const MORE = 1

#IO Multiplexing
const POLLIN = 1
const POLLOUT = 2
const POLLERR = 4

#Built in devices
const STREAMER = 1
const FORWARDER = 2
const QUEUE = 3

end


text:
#####
##### Profile Asynchronous MCTS to figure out the optimal number of workers
#####

using AlphaZero
using ProgressMeter
using Plots

# It is important not to profile async mcts on tictactoe as the number
# of states in this game is small and so the mcts tree quickly contains
# every state.

# Do NOT run this from the REPL!

ENV["JULIA_CUDA_MEMORY_POOL"] = "split"

include("../games.jl")
GAME = get(ENV, "GAME", "connect-four")
SelectedGame = GAME_MODULE[GAME]
using .SelectedGame: Game, Training

NUM_GAMES = 10
MAX_LOG_NWORKERS = 8 # 2^8 = 256

TIME_FIG = "mcts_speed"
INFERENCE_TIME_RATIO_FIG = "inference_time_ratio"

network = Training.Network{Game}(Training.netparams)

function profile(nworkers, ngames)
  params = MctsParams(Training.self_play.mcts, num_workers=nworkers)
  player = MctsPlayer(network, params)
  GC.gc(true)
  time = @elapsed begin
    @showprogress for i in 1:ngames
      AlphaZero.play_game(player, player)
    end
  end
  return time, MCTS.inference_time_ratio(player.mcts)
end

println("Compile everything...")
profile(1, 1)
profile(2, 1)

results = []
ts = []
itrs = []
for i in 0:MAX_LOG_NWORKERS
  nworkers = 2 ^ i
  println("Profiling MCTS with $nworkers workers...")
  time, inference_time_ratio = profile(nworkers, NUM_GAMES)
  push!(ts, time)
  push!(itrs, inference_time_ratio)
end

xticks = (
  collect(0:MAX_LOG_NWORKERS),
  ["$(2^i)" for i in 0:MAX_LOG_NWORKERS])

plot(0:MAX_LOG_NWORKERS, ts[1] ./ ts,
  title="Async MCTS Speedup",
  ylabel="Speedup",
  xlabel="Number of workers",
  ylims=(0, Inf),
  legend=:none,
  xticks=xticks)

hline!([1])

savefig(TIME_FIG)

plot(0:MAX_LOG_NWORKERS, 100 * itrs,
  title="Percentage of time spent in inference",
  ylims=(0, 100),
  xlabel="Number of workers",
  legend=:none,
  xticks=xticks)

savefig(INFERENCE_TIME_RATIO_FIG)


text:
# Before running this, please make sure to activate and instantiate the
# environment with [this `Project.toml`](https://raw.githubusercontent.com/juliaai/DataScienceTutorials.jl/gh-pages/__generated/EX-GLM/Project.toml) and
# [this `Manifest.toml`](https://raw.githubusercontent.com/juliaai/DataScienceTutorials.jl/gh-pages/__generated/EX-GLM/Manifest.toml).
# For instance, copy these files to a folder 'EX-GLM', `cd` to it and
#
# ```julia
# using Pkg; Pkg.activate("."); Pkg.instantiate()
# ```

# **Main author**: [Clarman Cruz](https://github.com/drcxcruz).
#
# This juypter lab showcases MLJ in particular using the popular [GLM](https://github.com/JuliaStats/GLM.jl) Julia package. We are using two datasets.  One dataset was created manually for testing purposes.  The other data set is the CollegeDistance dataset from the [AER](https://cran.r-project.org/web/packages/AER/index.html) package in R.
#
# We can quickly define our models in MLJ and study their results.  It is very easy and consistent.

using MLJ, CategoricalArrays, PrettyPrinting
import DataFrames: DataFrame, nrow
using UrlDownload

LinearRegressor = @load LinearRegressor pkg=GLM
LinearBinaryClassifier = @load LinearBinaryClassifier pkg=GLM

# ## Reading the data
#
# The CollegeDistance dataset was stored in a CSV file.  Here, we read the input file.

baseurl = "https://raw.githubusercontent.com/tlienart/DataScienceTutorialsData.jl/master/data/glm/"

dfX = DataFrame(urldownload(baseurl * "X3.csv"))
dfYbinary = DataFrame(urldownload(baseurl * "Y3.csv"))
dfX1 = DataFrame(urldownload(baseurl * "X1.csv"))
dfY1 = DataFrame(urldownload(baseurl * "Y1.csv"));

# You can have a look at those using `first`:

first(dfX, 3)

# same for Y:

first(dfY1, 3)

# ## Defining the Linear Model
#
# Let see how many MLJ models handle our kind of target which is the y variable.

ms = models() do m
    AbstractVector{Count} <: m.target_scitype
end
foreach(m -> println(m.name), ms)

# How about if the type was Continuous:

ms = models() do m
    Vector{Continuous} <: m.target_scitype
end
foreach(m -> println(m.name), ms)

# We can quickly define our models in MLJ.  It is very easy and consistent.

X = copy(dfX1)
y = copy(dfY1)

coerce!(X, autotype(X, :string_to_multiclass))
yv = Vector(y[:, 1])

LinearRegressorPipe = Pipeline(
    Standardizer(),
    OneHotEncoder(drop_last = true),
    LinearRegressor()
)

LinearModel = machine(LinearRegressorPipe, X, yv)
fit!(LinearModel)
fp = fitted_params(LinearModel)

# ## Reading the Output of Fitting the Linear Model
#
# We can quickly read the results of our models in MLJ.  Remember to compute the accuracy of the linear model.

ŷ = MLJ.predict(LinearModel, X)
yhatResponse = [ŷ[i,1].μ for i in 1:nrow(y)]
residuals = y .- yhatResponse
r = report(LinearModel)

k = collect(keys(fp.fitted_params_given_machine))[3]
println("\n Coefficients:  ", fp.fitted_params_given_machine[k].coef)
println("\n y \n ", y[1:5,1])
println("\n ŷ \n ", ŷ[1:5])
println("\n yhatResponse \n ", yhatResponse[1:5])
println("\n Residuals \n ", y[1:5,1] .- yhatResponse[1:5])
println("\n Standard Error per Coefficient \n", r.linear_regressor.stderror[2:end])

# and get the accuracy

round(rms(yhatResponse, y[:,1]), sigdigits=4)

# ## Defining the Logistic Model

X = copy(dfX)
y = copy(dfYbinary)

coerce!(X, autotype(X, :string_to_multiclass))
yc = CategoricalArray(y[:, 1])
yc = coerce(yc, OrderedFactor)

LinearBinaryClassifierPipe = Pipeline(
    Standardizer(),
    OneHotEncoder(drop_last = true),
    LinearBinaryClassifier()
)

LogisticModel = machine(LinearBinaryClassifierPipe, X, yc)
fit!(LogisticModel)
fp = fitted_params(LogisticModel)

# ## Reading the Output from the Prediction of the Logistic Model
#
# The output of the MLJ model basically contain the same information as the R version of the model.

ŷ = MLJ.predict(LogisticModel, X)
residuals = [1 - pdf(ŷ[i], y[i,1]) for i in 1:nrow(y)]
r = report(LogisticModel)

k = collect(keys(fp.fitted_params_given_machine))[3]
println("\n Coefficients:  ", fp.fitted_params_given_machine[k].coef)
println("\n y \n ", y[1:5,1])
println("\n ŷ \n ", ŷ[1:5])
println("\n residuals \n ", residuals[1:5])
println("\n Standard Error per Coefficient \n", r.linear_binary_classifier.stderror[2:end])

# No logistic analysis is complete without the confusion matrix:

yMode = [mode(ŷ[i]) for i in 1:length(ŷ)]
y = coerce(y[:,1], OrderedFactor)
yMode = coerce(yMode, OrderedFactor)
confusion_matrix(yMode, y)

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


text:
using Test
using Distributions
using HMMBase
using VariationalH3M

import Base: rand

# TODO