import errno
import gc
import io
import logging
import os
import platform
import pprint
import re
import shutil
import signal
import stat
import sys
import time

from contextlib import closing, contextmanager
from functools import wraps
from tempfile import mkstemp, mkdtemp
from threading import Thread
from unittest import TestCase
from unittest.case import SkipTest
from weakref import ref

import pytest

from ioprocess import (
    IOProcess,
    ERR_IOPROCESS_CRASH,
    Closed,
    Timeout,
    config,
    clear_cloexec
)

elapsed_time = lambda: os.times()[4]

config.IOPROCESS_PATH = os.path.join(os.getcwd(),
                                     "../../src/ioprocess")
IOProcess._DEBUG_VALGRIND = os.environ.get("ENABLE_VALGRIND", False)

_VALGRIND_RUNNING = IOProcess._DEBUG_VALGRIND

IOProcess._TRACE_DEBUGGING = True

log = logging.getLogger("Test")

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s %(levelname)-7s (%(threadName)s) [%(name)s] %(message)s"
)

requires_unprivileged_user = pytest.mark.skipif(
    os.geteuid() == 0, reason="This test can not run as root")

def on_s390x():
    return platform.machine() == "s390x"

def skip_in_valgrind(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        if _VALGRIND_RUNNING:
            raise SkipTest("Tests can't be run in valgrind")

        return f(*args, **kwargs)

    return wrapper

class IOProcessTests(TestCase):

    def testMaxRequestsAfterFillingThreadPool(self):
        proc = IOProcess(timeout=10, max_threads=3, max_queued_requests=0)
        with closing(proc):
            t1 = Thread(target=proc.echo, args=("hello", 2))
            t2 = Thread(target=proc.echo, args=("hello", 2))
            t3 = Thread(target=proc.echo, args=("hello", 2))
            t1.start()
            t2.start()
            t3.start()

            for t in (t1, t2, t3):
                t.join()

            t1 = Thread(target=proc.echo, args=("hello", 2))
            t2 = Thread(target=proc.echo, args=("hello", 2))
            t1.start()
            t2.start()
            # Make sure the echo calls are sent prior to the ping otherwise one
            # of them would fail and ping() would pass
            time.sleep(0.5)
            proc.ping()
            t1.join()
            t2.join()

    def testPing(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertEquals(proc.ping(), "pong")

    def test2SubsequentCalls(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertEquals(proc.ping(), "pong")
            self.assertEquals(proc.ping(), "pong")

    def testEcho(self):
        data = """The Doctor: But I don't exist in your world!
                  Brigade Leader: Then you won't feel the bullets when we
                  shoot you."""  # (C) BBC - Doctor Who
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertEquals(proc.echo(data), data)

    def testUnicodeEcho(self):
        data = u'\u05e9\u05dc\u05d5\u05dd'
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertEquals(proc.echo(data), data)

    def testMultitask(self):
        """
        Makes sure that when multiple requests are sent the results come
        back with correct IDs
        """
        threadnum = 10
        # We want to run all requests in parallel, so have one ioprocess thread
        # per client thread.
        proc = IOProcess(timeout=10, max_threads=threadnum)
        with closing(proc):
            errors = []
            threads = []

            def test(n):
                if proc.echo(str(n), 1) != str(n):
                    errors.append(n)

        for i in range(threadnum):
            t = Thread(target=test, args=(i,))
            t.start()
            threads.append(t)

        for thread in threads:
            thread.join()

        self.assertEquals(len(errors), 0)

    def testRecoverAfterCrash(self):
        data = """Brigadier: Is there anything I can do?
                  Third Doctor: Yes, pass me a silicon rod.
                                [Stirs cup of tea with it]
                  Brigadier: I meant is there anything UNIT can do about this
                  space lightning business?"""  # (C) BBC - Doctor Who
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertEquals(proc.echo(data), data)
            self.assertTrue(proc.crash())
            self.assertEquals(proc.echo(data), data)

    def testPendingRequestInvalidationOnCrash(self):
        data = """The Doctor: A straight line may be the shortest distance
                  between two points, but it is by no means the most
                  interesting."""  # (C) BBC - Doctor Who
        proc = IOProcess(timeout=12, max_threads=5)
        with closing(proc):
            res = [False]

            def sendCmd():
                try:
                    proc.echo(data, 10)
                except OSError as e:
                    if e.errno == ERR_IOPROCESS_CRASH:
                        res[0] = True
                    else:
                        log.error("Got unexpected error", exc_info=True)

            t = Thread(target=sendCmd)
            t.start()

            time.sleep(1)
            proc.crash()
            t.join()
            self.assertTrue(res[0])

    def testTimeout(self):
        data = """Madge: Are you the new caretaker?
                  The Doctor: Usually called "The Doctor." Or "The Caretaker."
                  Or "Get off this planet." Though, strictly speaking, that
                  probably isn't a name."""  # (C) BBC - Doctor Who
        # Using smaller timeout to ensure the echo will time out.
        proc = IOProcess(timeout=1, max_threads=5)
        with closing(proc):
            try:
                self.assertEquals(proc.echo(data, 10), data)
            except Timeout:
                return

            self.fail("Exception not raised")

    @skip_in_valgrind
    def testManyRequests(self):
        data = """Lily: What's happening?
                  The Doctor: No idea. Just do what I do: hold tight and
                  pretend it's a plan."""  # (C) BBC - Doctor Who
        proc = IOProcess(timeout=30, max_threads=5)
        with closing(proc):
            # even though we theoretically go back to a stable state, some
            # objects might have increased their internal buffers and mem
            # fragmantation might have caused some data to be spanned on more
            # pages then it originally did.
            acceptableRSSIncreasKB = 100

            startRSS = proc.memstat()['rss']
            # This way we catch evey leak that is more then one 0.1KB per call
            many = 300
            for i in range(many):
                self.assertEquals(proc.echo(data), data)
            endRSS = proc.memstat()['rss']
            RSSDiff = endRSS - startRSS
            log.debug("RSS difference was %d KB, %d per request", RSSDiff,
                      RSSDiff / many)
            # This only tests for leaks in the main request\response process.
            self.assertTrue(RSSDiff < acceptableRSSIncreasKB,
                            "Detected a leak sized %d KB" % RSSDiff)

    @pytest.mark.xfail(on_s390x(), reason="Unknown")
    def testStatvfs(self):
        data = b'''Peter Puppy: Once again, evil is as rotting meat before
                                the maggots of justice!
                   Earthworm Jim: Thank you for cramming that delightful image
                                  into my brain, Peter.
                '''  # (C) Universal Cartoon Studios - Earth Worm Jim
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, path = mkstemp()
            try:
                os.write(fd, data)
                os.close(fd)
                pystat = os.statvfs(path)
                mystat = proc.statvfs(path)
                for f in ("f_bsize", "f_frsize", "f_blocks",
                          "f_fsid", "f_flag", "f_namemax"):

                    try:
                        getattr(pystat, f)
                    except AttributeError:
                        # The results might be more comprehansive then python
                        # implementation
                        continue

                    log.debug("Testing field '%s'", f)
                    self.assertEquals(getattr(mystat, f), getattr(pystat, f))
            finally:
                os.unlink(path)

    def testStatFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.stat("/I do not exist")
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testMissingArguemt(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc._sendCommand("echo", {}, proc.timeout)
            except OSError as e:
                self.assertEquals(e.errno, errno.EINVAL)

    def testNonExistingMethod(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc._sendCommand("Implode", {}, proc.timeout)
            except OSError as e:
                self.assertEquals(e.errno, errno.EINVAL)

    def testPathExists(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, path = mkstemp()
            try:
                os.close(fd)
                self.assertTrue(proc.pathExists(path))
            finally:
                os.unlink(path)

    def testPathDoesNotExist(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            self.assertFalse(proc.pathExists("/I do not exist"))

    def testRename(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, oldpath = mkstemp()
            newpath = oldpath + ".new"
            try:
                os.close(fd)
                self.assertTrue(proc.rename(oldpath, newpath))
            finally:
                try:
                    os.unlink(oldpath)
                except:
                    pass
                try:
                    os.unlink(newpath)
                except:
                    pass

    def testRenameFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.rename("/I/do/not/exist", "/Dsadsad")
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testUnlink(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, path = mkstemp()
            try:
                os.close(fd)
                self.assertTrue(proc.unlink(path))
            finally:
                try:
                    os.unlink(path)
                except:
                    pass

    def testUnlinkFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.unlink("/I do not exist")
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testLink(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, oldpath = mkstemp()
            newpath = oldpath + ".new"
            try:
                os.close(fd)
                self.assertTrue(proc.link(oldpath, newpath))
            finally:
                os.unlink(oldpath)
                try:
                    os.unlink(newpath)
                except:
                    pass

    def testLinkFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.link("/I/do/not/exist", "/Dsadsad")
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testSymlink(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, oldpath = mkstemp()
            newpath = oldpath + ".new"
            try:
                os.close(fd)
                self.assertTrue(proc.symlink(oldpath, newpath))
                self.assertEquals(os.path.realpath(newpath),
                                  os.path.normpath(oldpath))
            finally:
                os.unlink(oldpath)
                try:
                    os.unlink(newpath)
                except:
                    pass

    def testSymlinkFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.symlink("/Dsadsad", "/I/do/not/exist")
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testChmod(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            fd, path = mkstemp()
            targetMode = os.W_OK | os.R_OK
            try:
                os.chmod(path, 0)
                os.close(fd)
                self.assertFalse(os.stat(path).st_mode & targetMode)
                self.assertTrue(proc.chmod(path, targetMode))
                self.assertTrue(os.stat(path).st_mode & targetMode)
            finally:
                os.unlink(path)

    def testChmodFail(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            try:
                proc.chmod("/I/do/not/exist", 0)
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
            else:
                raise AssertionError("OSError was not raised")

    def testListdir(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            path = mkdtemp()
            matches = []
            for i in range(10):
                matches.append(os.path.join(path, str(i)))
                with open(matches[-1], "w") as f:
                    f.write("A")

            matches.sort()

            try:
                remoteMatches = proc.listdir(path)
                remoteMatches.sort()
                flist = os.listdir(path)
                flist.sort()
                self.assertEquals(remoteMatches, flist)
            finally:
                shutil.rmtree(path)

    def testGlob(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            path = mkdtemp()
            matches = []
            for i in range(10):
                matches.append(os.path.join(path, str(i)))
                with open(matches[-1], "w") as f:
                    f.write("A")

            matches.sort()

            try:
                remoteMatches = proc.glob(os.path.join(path, "*"))
                remoteMatches.sort()
                self.assertEquals(remoteMatches, matches)
            finally:
                shutil.rmtree(path)

    def testRmdir(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            path = mkdtemp()

            try:
                proc.rmdir(path)
                self.assertFalse(os.path.exists(path))
            finally:
                try:
                    shutil.rmtree(path)
                except:
                    pass

    def testMkdir(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            path = mkdtemp()
            shutil.rmtree(path)

            try:
                proc.mkdir(path)
                self.assertTrue(os.path.exists(path))
            finally:
                try:
                    shutil.rmtree(path)
                except:
                    pass

    def testLexists(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            path = "/tmp/linktest.ioprocesstest"
            try:
                os.unlink(path)
            except OSError:
                pass
            os.symlink("dsadsadsadsad", path)
            try:
                self.assertTrue(proc.lexists(path))
            finally:
                os.unlink(path)

    def testGlobNothing(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            remoteMatches = proc.glob(os.path.join("/dsadasd", "*"))
            self.assertEquals(remoteMatches, [])

    def test_closed(self):
        proc = IOProcess(timeout=10, max_threads=5)
        proc.close()
        self.assertRaises(Closed, proc.echo, "foo", 1)

    def test_close_terminates_process(self):
        for i in range(100):
            proc = IOProcess(timeout=10, max_threads=5)
            proc.close()
            self.assertFalse(os.path.exists("/proc/%d" % proc.pid),
                             "process %s did not terminate" % proc)

    def test_restart_child(self):
        proc = IOProcess(timeout=10, max_threads=5)
        with closing(proc):
            # Kill current child process.
            pid = proc.pid
            log.debug("Killing child %s", pid)
            os.kill(pid, signal.SIGTERM)

            log.debug("Waiting until child %s terminate", pid)
            while os.path.exists("/proc/%d" % pid):
                time.sleep(0.01)

            log.debug("Waiting for new child...")
            while not os.path.exists("/proc/%d" % proc.pid):
                time.sleep(0.01)

            log.debug("New child %s started", proc.pid)

    def test_close_unrelated_fds(self):
        # Make inheritable file descriptor.
        with open(__file__) as my_file:
            clear_cloexec(my_file.fileno())
            proc = IOProcess(timeout=10, max_threads=5)
            with closing(proc):
                # Wait until ready.
                proc.ping()
                proc_fd = "/proc/%d/fd" % proc.pid
                child_fds = [int(fd) for fd in os.listdir(proc_fd)]
                # My file descriptor must not be inherited.
                self.assertNotIn(my_file.fileno(), child_fds)

def test_max_requests():
    proc = IOProcess(timeout=10, max_threads=1, max_queued_requests=1)
    with closing(proc):
        t1 = Thread(target=proc.echo, args=("hello", 2))
        t2 = Thread(target=proc.echo, args=("hello", 2))
        t1.start()
        t2.start()
        # Make sure the echo calls are sent prior to the ping otherwise one
        # of them would fail and ping() would pass.
        time.sleep(0.5)

        try:
            with pytest.raises(OSError) as e:
                proc.ping()
            assert e.value.errno == errno.EAGAIN
        finally:
            t1.join()
            t2.join()

def test_fsyncpath_directory(tmpdir):
    proc = IOProcess(timeout=10, max_threads=1)
    with closing(proc):
        # No easy way to test that we actually fsync this path. Lets just
        # call it to make sure it does not fail.
        proc.fsyncPath(str(tmpdir))

def test_fsyncpath_missing(tmpdir):
    proc = IOProcess(timeout=10, max_threads=1)
    with closing(proc):
        with pytest.raises(OSError) as e:
            proc.fsyncPath("/no/such/file")
        assert e.value.errno == errno.ENOENT

def test_stat_file(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        file = tmpdir.join("file")
        file.write(b"x" * 100)
        file = str(file)
        check_stat(proc.stat(file), os.stat(file))

def test_stat_dir(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        dir = str(tmpdir)
        check_stat(proc.stat(dir), os.stat(dir))

def test_stat_link(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        src = tmpdir.join("src")
        src.write(b"x" * 100)
        src = str(src)
        link = str(tmpdir.join("link"))
        os.symlink(src, link)
        check_stat(proc.stat(link), os.stat(link))

def test_stat_link_missing(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        src = str(tmpdir.join("file"))
        link = str(tmpdir.join("link"))
        os.symlink(src, link)
        with pytest.raises(OSError) as myerror:
            proc.stat(link)
        with pytest.raises(OSError) as pyerror:
            os.stat(link)
        assert myerror.value.errno == pyerror.value.errno

def test_lstat_file(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        file = tmpdir.join("file")
        file.write(b"x" * 100)
        file = str(file)
        check_stat(proc.lstat(file), os.lstat(file))

def test_lstat_dir(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        dir = str(tmpdir)
        check_stat(proc.lstat(dir), os.lstat(dir))

def test_lstat_link(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        src = tmpdir.join("src")
        src.write(b"x" * 100)
        src = str(src)
        link = str(tmpdir.join("link"))
        os.symlink(src, link)
        check_stat(proc.lstat(link), os.lstat(link))

def test_lstat_link_missing(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        src = str(tmpdir.join("file"))
        link = str(tmpdir.join("link"))
        os.symlink(src, link)
        check_stat(proc.lstat(link), os.lstat(link))

def check_stat(mystat, pystat):
    for f in mystat._fields:
        if f in ("st_atime", "st_mtime", "st_ctime"):
            # These are float\double values and due to the many
            # conversion the values experience during marshaling
            # they cannot be equated. The rest of the fields are a
            # good enough test.
            continue
        assert getattr(mystat, f) == getattr(pystat, f)

@pytest.mark.parametrize("size", [0, 1, 512, 4096, 1024**2 + 1])
def test_writefile(tmpdir, size):
    data = b'x' * size
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        path = str(tmpdir.join("file"))
        proc.writefile(path, data)
        with io.open(path, 'rb') as f:
            written = f.read()
        assert written == data

# TODO: Use userstorage instead of assuming CI storage sector size.
@pytest.mark.parametrize("size", [
    0,
    pytest.param(
        512,
        marks=pytest.mark.xfail(
            on_s390x(),
            reason="Test assumes 512 block size but on out s390x slave "
                   "storage uses 4k sector size")),
    4096,
    1024**2
])
def test_writefile_direct_aligned(tmpdir, size):
    data = b'x' * size
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        path = str(tmpdir.join("file"))
        proc.writefile(path, data, direct=True)
        with io.open(path, 'rb') as f:
            written = f.read()
        assert written == data

def test_writefile_direct_unaligned(tmpdir):
    data = b'unaligned data'
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        path = str(tmpdir.join("file"))
        with pytest.raises(OSError) as e:
            proc.writefile(path, data, direct=True)
        assert e.value.errno == errno.EINVAL

@pytest.mark.parametrize("size", [0, 1, 42, 512, 4096, 1024**2 + 1])
def test_readfile(tmpdir, size):
    data = b'x' * size
    path = str(tmpdir.join("file"))
    with io.open(path, "wb") as f:
        f.write(data)

    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        read = proc.readfile(path)
        assert read == data

@pytest.mark.parametrize("size", [0, 1, 42, 512, 4096, 1024**2 + 1])
def test_readfile_direct(tmpdir, size):
    data = b'x' * size
    path = str(tmpdir.join("file"))
    with io.open(path, "wb") as f:
        f.write(data)
        os.fsync(f.fileno())

    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        read = proc.readfile(path, direct=True)
        assert read == data

@pytest.mark.parametrize("direct", [
    pytest.param(True, id="direct"),
    pytest.param(False, id="buffered"),
])
def test_readfile_missing(tmpdir, direct):
    path = str(tmpdir.join("file"))
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        with pytest.raises(OSError) as e:
            read = proc.readfile(path, direct=direct)
        assert e.value.errno == errno.ENOENT

ACCESS_PARAMS = [
    (0o755, os.R_OK, True),
    (0o300, os.R_OK, False),
    (0o744, os.W_OK, True),
    (0o444, os.W_OK, False),
    (0o755, os.X_OK, True),
    (0o400, os.X_OK, False),
    (0o300, os.W_OK | os.X_OK, True),
    (0o300, os.R_OK | os.W_OK | os.X_OK, False),
]

@pytest.mark.parametrize("mode, permission, expected_result", ACCESS_PARAMS)
@requires_unprivileged_user
def test_access_file(tmpdir, mode, permission, expected_result):
    proc = IOProcess(timeout=10, max_threads=5)
    f = tmpdir.join("file")
    f.write("")
    path = str(f)

    with closing(proc):
        with chmod(path, mode):
            assert proc.access(path, permission) == os.access(path, permission)
            assert proc.access(path, permission) == expected_result

@pytest.mark.parametrize("mode, permission, expected_result", ACCESS_PARAMS)
@requires_unprivileged_user
def test_access_directory(tmpdir, mode, permission, expected_result):
    proc = IOProcess(timeout=10, max_threads=5)
    d = tmpdir.mkdir("subdir")
    path = str(d)

    with closing(proc):
        with chmod(path, mode):
            assert proc.access(path, permission) == os.access(path, permission)
            assert proc.access(path, permission) == expected_result

# TODO: Use userstorage instead of assuming block size.
@pytest.mark.xfail(
    on_s390x(),
    reason="Assuming 512, on s390x we have 4k drives")
def test_probe_block_size_512(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        assert proc.probe_block_size(str(tmpdir)) == 512

# TODO: Use userstorage instead of assuming block size.
@pytest.mark.skipif(not on_s390x(), reason="Assuming block size 512")
@pytest.mark.xfail(reason="Test assumes 4k block size on s390x")
def test_probe_block_size_4096(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        assert proc.probe_block_size(str(tmpdir)) == 4096

def test_probe_block_size_unsupported():
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        # /dev/shm uses tempfs, does not support direct I/O.
        with pytest.raises(OSError) as e:
            proc.probe_block_size("/dev/shm")
        assert e.value.errno == errno.EINVAL

def test_probe_block_size_cleanup(tmpdir):
    proc = IOProcess(timeout=10, max_threads=5)
    with closing(proc):
        proc.probe_block_size(str(tmpdir))
    # Should not leave the probe temporary file around.
    assert tmpdir.listdir() == []

@requires_unprivileged_user
def test_probe_block_size_not_writable(tmpdir):
    no_write = str(tmpdir.mkdir("no-write-for-you"))
    # Remove write bit, so the probe file cannot be created.
    with chmod(no_write, 0o500):
        proc = IOProcess(timeout=10, max_threads=5)
        proc._sublog = FakeLogger()
        with closing(proc):
            with pytest.raises(OSError) as e:
                proc.probe_block_size(no_write)
            assert e.value.errno == errno.EACCES
            warning = re.compile(r"Failed to create a probe file: .+?, "
                                 r"error: 'Permission denied'")
            assert any(warning.search(msg) for msg in proc._sublog.messages)

def test_probe_block_size_concurrent(tmpdir):
    proc = IOProcess(timeout=10, max_threads=20)
    with closing(proc):
        probe_dir = str(tmpdir)
        threads = []
        results = []

        def worker():
            for i in range(20):
                res = proc.probe_block_size(probe_dir)
                results.append(res)

        try:
            for i in range(20):
                t = Thread(target=worker)
                t.deamon = True
                t.start()
                threads.append(t)
        finally:
            for t in threads:
                t.join()

    # We should have identical prob results.
    assert len(results) == 400
    assert len(frozenset(results)) == 1

    # And no leftover probe files.
    assert tmpdir.listdir() == []

@contextmanager
def chmod(path, mode):
    """Changes path permissions.

    Changes the path permissions to the requested ones and
    reverts to the original permissions at the end.

    Arguments:
        path (str): file/directory path
        mode (int): new mode
    """

    orig_mode = stat.S_IMODE(os.stat(path).st_mode)
    os.chmod(path, mode)

    try:
        yield
    finally:
        try:
            os.chmod(path, orig_mode)
        except Exception as e:
            logging.error("Failed to restore %r mode: %s", path, e)

class TestWeakref(TestCase):

    @pytest.mark.xfail(
        sys.version_info[:2] == (3, 7), reason="fails in python 3.7")
    def test_close_when_unrefed(self):
        """Make sure there is nothing keeping IOProcess strongly referenced.

        Since there is a communication background thread doing all the hard
        work we need to make sure it doesn't prevent IOProcess from being
        garbage collected.
        """
        proc = IOProcess(timeout=10, max_threads=5)
        proc = ref(proc)

        end = elapsed_time() + 5.0

        while True:
            gc.collect()
            real_proc = proc()
            if real_proc is None:
                break
            refs = gc.get_referrers(real_proc)
            log.info("Object referencing ioprocess instance: %s",
                     pprint.pformat(refs))
            if hasattr(refs[0], "f_code"):
                log.info("Function referencing ioprocess instance: %s",
                         pprint.pformat(refs[0].f_code))
            if elapsed_time() > end:
                raise AssertionError("These objects still reference "
                                     "ioprocess: %s" % refs)
            del refs
            del real_proc
            time.sleep(0.1)

class FakeLogger(object):

    def __init__(self):
        self.messages = []

    def debug(self, fmt, *args):
        msg = fmt % args
        self.messages.append(msg)

    info = debug
    warning = debug
    error = debug

class LoggingTests(TestCase):

    def test_partial_logs(self):
        threads = []
        proc = IOProcess(timeout=10, max_threads=10)
        proc._sublog = FakeLogger()

        def worker():
            for i in range(100):
                proc.stat(__file__)

        try:
            for i in range(4):
                t = Thread(target=worker)
                t.deamon = True
                t.start()
                threads.append(t)
        finally:
            for t in threads:
                t.join()
            proc.close()

        for msg in proc._sublog.messages:
            self.assertFalse('DEBUG|' in msg,
                             "Raw log data in log message: %r" % msg)


import random
from .magic import Spell
from .inventory import Item

# Creating colors for UI
class bcolors:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	ENDC = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

# Class Person, for players and enemies-------
class Person:
	def __init__(self, name, hp, mp, atk, df, magic, items):

		self.name = name
		self.hp = hp
		self.hpmax = hp
		self.mp = mp
		self.mpmax = mp
		self.atkl = atk - 10
		self.atkh = atk + 10
		self.magic = magic
		self.items = items
		self.df = df
		self.actions = ["Attack", "Magic", "Items"]

	# Creating Essential Functions---------------
	def take_damage(self, dmg):

		self.hp -= dmg
		if self.hp < 0:
			self.hp = 0

	def generate_damage(self):
		dmg = random.randrange(self.atkl, self.atkh)
		return dmg

	def get_hp(self):
		return self.hp

	def get_hp_max(self):
		return self.hpmax

	def get_mp(self):
		return self.mp

	def get_mp_max(self):
		return self.mpmax

	def reduce_mp(self, cost):
		self.mp -= cost
		return self.mp

	def heal(self, dmg):
		self.hp += dmg

	def put_hp_max(self):
		self.hp = self.hpmax

	def put_mp_max(self):
		self.mp = self.mpmax

	def get_name(self):
		return self.name

	# Functions for action selection----------------
	def choose_action(self):
		i = 1
		print("\n" + bcolors.BOLD + self.name + "'s Turns: "+ bcolors.ENDC)
		print(bcolors.FAIL + "Actions" + bcolors.ENDC)
		for item in self.actions:
			print("   " + str(i) + ":", item)
			i += 1
	def choose_magic(self):
		i = 1
		print(bcolors.OKBLUE + "Magic" + bcolors.ENDC + " (Write 0 to go back)")
		for spell in self.magic:
			print("   " + str(i) + ":", spell.name, "(cost:", str(spell.cost) + ")" )
			i += 1

	def choose_item(self):
		i = 1
		print(bcolors.OKGREEN + "Items " + bcolors.ENDC + "(Write 0 to go back)")
		for item in self.items:
			print("   " + str(i) + "." + item["items"].name + " : " + item["items"].description + " (x" + str(item["quantity"]) + ")")
			i += 1

	def choose_target(self, enemies):
		i = 1

		print("\n" + bcolors.FAIL + bcolors.BOLD + "TARGET: " + bcolors.ENDC)
		for enemy in enemies:
			if enemy.get_hp	() != 0:
				print(" " + str(i) + "." + enemy.name  )
				i += 1
		choice = int(input("Choose Target: ")) - 1
		return choice

	def choose_enemy_spell(self):
		magic_choice = random.randrange(0, len(self.magic))
		spell = self.magic[magic_choice]
		magic_dmg = spell.generate_damage()

		if self.mp < spell.cost:
			self.choose_enemy_spell()
		else:
			return spell, magic_dmg

	def choose_enemy_item(self):
		item_choice = random.randrange(0, len(self.items))
		item = self.items[item_choice]
		item_props = item["items"].generate_prop()

		if self.items[item_choice]["quantity"] == 0:
			self.choose_enemy_item()
		else:
			return item, item_props, item_choice

	# Functions to print HP/MP Bars
	def enemy_get_stats(self):
		hp_bar = ""
		hp_mult = (self.hp / self.hpmax) * 100 / 2

		while hp_mult > 0:
			hp_bar += "█"
			hp_mult -= 1

		while len(hp_bar) < 50:
			hp_bar += " "

		hp_string = str(self.hp) + "/" + str(self.hpmax)
		current_hp = ""

		if len(hp_string) < 9:
			decreased = 9 - len(hp_string)
			while decreased > 0:
				current_hp += " "
				decreased -= 1
			current_hp += hp_string
		else:
			current_hp = hp_string

		print(bcolors.BOLD + self.name + ":" +
			  "                        " + bcolors.ENDC +
			  current_hp + " |" + bcolors.FAIL + hp_bar + bcolors.ENDC + bcolors.BOLD
			  + "|")

	# Updating HP/MP bars----------------------------------
	def get_stats(self):
		hp_bar = ""
		hp_mult = (self.hp / self.hpmax) * 100 / 4

		mp_bar = ""
		mp_mult = (self.mp / self.mpmax) * 100 / 10

		while hp_mult > 0:
			hp_bar += "█"
			hp_mult -= 1

		while len(hp_bar) < 25:
			hp_bar += " "

		while mp_mult > 0:
			mp_bar += "█"
			mp_mult -= 1

		while len(mp_bar) < 10:
				mp_bar += " "

		# Creating White Space at the beginning of the HP/MP string to maintain the UI order
		hp_string = str(self.hp) + "/" + str(self.hpmax)
		current_hp = ""

		if len(hp_string) < 9:
			decreased = 9 - len(hp_string)
			while decreased > 0:
				current_hp += " "
				decreased -= 1
			current_hp += hp_string
		else:
			current_hp = hp_string

		mp_string = str(self.mp) + "/" + str(self.mpmax)
		current_mp = ""

		if len(mp_string) < 9:
			decreased = 9 - len(mp_string)
			while decreased > 0:
				current_mp += " "
				decreased -= 1

			current_mp += mp_string
		else:
			current_mp = mp_string

		# Printing HP/MP Bars
		print(bcolors.BOLD + self.name + ":" +
			"                        " + bcolors.ENDC +
			current_hp + " |" + bcolors.OKGREEN + hp_bar + bcolors.ENDC + bcolors.BOLD
			+ "|     " +
			current_mp + " |" + bcolors.OKBLUE + mp_bar + bcolors.ENDC + "|")


import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPalette, QColor
from PyQt5 import QtGui, QtCore, QtWidgets
from tagasimaksegraafik import tagasimaksegraafik
from Tagasimakseaken import Tagasimakse
from themes import *
class Algaken(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setFixedSize(640,440)
        self.centralwidget = QWidget()
        self.setCentralWidget(self.centralwidget)
        self.setWindowIcon(QtGui.QIcon('pics/logo.jpg'))
        self.setWindowTitle("Laenukalkulaator")
        self.setStyleSheet(styledef)

        self.algsummaLabel = QLabel("Laenatud summa")
        self.algsummatekst = QLineEdit(self)
        self.intressLabel = QLabel("Aastane intress")
        self.intresstekst = QLineEdit(self)
        self.perioodLabel = QLabel("Laenuperiood")
        self.perioodtekst = QLineEdit(self)
        self.genereerinupp = QPushButton("genereeri tagasimaksegraafik")
        self.metsnupp = QPushButton("style:mets")
        self.dunenupp = QPushButton("style:dune")
        self.darknupp = QPushButton("style:dark")
        self.lightnupp = QPushButton("style:light")

        lay = QGridLayout(self.centralwidget)
        #algsumma
        lay.addWidget(self.algsummaLabel,0,0)
        self.algsummaLabel.setStyleSheet(darkstyle)
        lay.addWidget(self.algsummatekst,0,1,1,3)
        self.algsummatekst.setStyleSheet(darkstyle)
        #intress
        lay.addWidget(self.intressLabel,1,0)
        self.intressLabel.setStyleSheet(darkstyle)
        lay.addWidget(self.intresstekst,1,1,1,3)
        self.intresstekst.setStyleSheet(darkstyle)
        #periood
        lay.addWidget(self.perioodLabel,2,0)
        self.perioodLabel.setStyleSheet(darkstyle)
        lay.addWidget(self.perioodtekst,2,1,1,3)
        self.perioodtekst.setStyleSheet(darkstyle)
        #genereit, lisa tühje labeleid vahele et ruumi tekitada?- doesnt really wörk
        lay.addWidget(self.genereerinupp,3,1,1,2)
        self.genereerinupp.setStyleSheet(darkstyle)
        self.genereerinupp.clicked.connect(self.genereeri)

        lay.addWidget(self.metsnupp,4,0)
        self.metsnupp.setStyleSheet(darkstyle)
        self.metsnupp.clicked.connect(self.mets)

        lay.addWidget(self.dunenupp,4,3)
        self.dunenupp.setStyleSheet(darkstyle)
        self.dunenupp.clicked.connect(self.dune)

        lay.addWidget(self.darknupp,4,2)
        self.darknupp.setStyleSheet(darkstyle)
        self.darknupp.clicked.connect(self.darkmode)

        lay.addWidget(self.lightnupp,4,1)
        self.lightnupp.setStyleSheet(darkstyle)
        self.lightnupp.clicked.connect(self.lightmode)

    def mets(self):
        self.setStyleSheet(stylemets)
    def dune(self):
        self.setStyleSheet(styledune)
    def darkmode(self):
        self.setStyleSheet(styledef)
    def lightmode(self):
        self.setStyleSheet(stylelight)

    def genereeri(self):
        algsumma = self.algsummatekst.text()
        intress = self.intresstekst.text()
        periood = self.perioodtekst.text()
        tagasimaksegraafik(algsumma, intress, periood)
        self.hide()
        next = Tagasimakse()
        next.__init__()


import random
from pydub.utils import mediainfo
from moviepy.editor import VideoFileClip, AudioFileClip

# Define constants
VIDEO_FILENAME = "./Assets/landscape.mp4"
AUDIO_FILENAME = "audio.mp3"
OUTPUT_FILENAME = "final_video.mp4"  # Output filename

def random_start(length, clip_length):
    """Return a random start time for the clip."""
    return random.uniform(0, length - clip_length)

def cut_clip(video_filename, audio_filename, start, clip_length, output_filename):
    """Cut clip and save to file."""
    end = start + clip_length

    # Load video and extract clip
    video = VideoFileClip(video_filename).subclip(start, end)

    # Load audio file
    audio = AudioFileClip(audio_filename)

    # Set the audio of the video clip
    video.set_audio(audio).write_videofile(output_filename, codec='libx264')

    print(f"Saved clip to {output_filename}")

def main():
    # Get the duration of the audio file
    audio_info = mediainfo(AUDIO_FILENAME)
    audio_length = float(audio_info['duration'])

    # Get the duration of the video file
    video_info = mediainfo(VIDEO_FILENAME)
    video_length = float(video_info['duration'])

    start_time = random_start(video_length, audio_length)
    cut_clip(VIDEO_FILENAME, AUDIO_FILENAME, start_time, audio_length, OUTPUT_FILENAME)

if __name__ == "__main__":
    main()


class Trie:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.root = {}

    def insert(self, word: str) -> None:
        """
        Inserts a word into the trie.
        """
        cur = self.root
        for c in word:
            if c not in cur:
                cur[c] = {}

            cur = cur[c]

        cur['is_word'] = True

    def search(self, word: str) -> bool:
        """
        Returns if the word is in the trie.
        """
        cur = self.root
        for c in word:
            if c not in cur:
                return False

            cur = cur[c]

        return cur.get('is_word', False)

    def startsWith(self, prefix: str) -> bool:
        """
        Returns if there is any word in the trie that starts with the given prefix.
        """
        cur = self.root
        for c in prefix:
            if c not in cur:
                return False

            cur = cur[c]

        return True

class Solution:
    moves = [(-1, 0), (0, 1), (1, 0), (0, -1)]

    def dfs(self, cur, board, r, c, route, ans):
        if cur.get('is_word'):
            ans.append(route)
            cur['is_word'] = False

        if not(0 <= r < len(board)) or not(0 <= c < len(board[0])):
            return

        char = board[r][c]
        cur = cur.get(char)
        if not cur:
            return

        board[r][c] = '*'
        for mr, mc in self.moves:
            self.dfs(cur, board, r+mr, c+mc, route+char, ans)
        board[r][c] = char

        return

    def findWords(self, board: List[List[str]], words: List[str]) -> List[str]:
        ans = []
        trie = Trie()
        cur = trie.root
        for word in words:
            trie.insert(word)

        row, col = len(board), len(board[0])
        for r in range(row):
            for c in range(col):
                self.dfs(cur, board, r, c, '', ans)

        return ans


import os

import pytest
from globus_compute_sdk import Client  # type: ignore
from globus_compute_sdk.sdk.login_manager.manager import LoginManager  # type: ignore
from globus_sdk import (
    AuthAPIError,
    AuthClient,
    ClientCredentialsAuthorizer,
    ConfidentialAppAuthClient,
    OAuthTokenResponse,
    SearchClient,
)

from garden_ai import GardenClient
from garden_ai.client import AuthException

is_gha = os.getenv("GITHUB_ACTIONS")

def test_client_no_previous_tokens(
    mocker, mock_authorizer_tuple, token, mock_keystore, identity_jwt
):
    mock_authorizer_constructor, mock_authorizer = mock_authorizer_tuple
    # Mocks for KeyStore
    mock_keystore.file_exists.return_value = False

    # Mocks for Login Flow
    mock_auth_client = mocker.MagicMock(AuthClient)
    mock_auth_client.oauth2_get_authorize_url = mocker.Mock(
        return_value="https://globus.auth.garden"
    )
    mock_auth_client.oauth2_start_flow = mocker.Mock()
    mocker.patch("garden_ai.client.Prompt.ask").return_value = "my token"
    mocker.patch("garden_ai.client.typer.launch")

    mock_search_client = mocker.MagicMock(SearchClient)

    mock_token_response = mocker.MagicMock(OAuthTokenResponse)
    mock_token_response.by_resource_server = {
        "groups.api.globus.org": token,
        "search.api.globus.org": token,
        "0948a6b0-a622-4078-b0a4-bfd6d77d65cf": token,
        "funcx_service": token,
        "auth.globus.org": token,
    }
    mock_token_response.data = {"id_token": identity_jwt}
    mock_auth_client.oauth2_exchange_code_for_tokens = mocker.Mock(
        return_value=mock_token_response
    )

    # Mocks compute client login
    mock_login_manager = mocker.MagicMock(LoginManager)
    mock_login_manager.ensure_logged_in = mocker.Mock(return_value=True)
    mocker.patch(
        "globus_compute_sdk.sdk.client.LoginManager"
    ).return_value = mock_login_manager

    # Call the Garden constructor
    gc = GardenClient(auth_client=mock_auth_client, search_client=mock_search_client)

    assert gc.auth_key_store == mock_keystore
    mock_auth_client.oauth2_exchange_code_for_tokens.assert_called_with("my token")

    mock_auth_client.oauth2_start_flow.assert_called_with(
        refresh_tokens=True,
        requested_scopes=[
            "openid",
            "email",
            "urn:globus:auth:scope:groups.api.globus.org:view_my_groups_and_memberships",
            "urn:globus:auth:scope:search.api.globus.org:all",
            "https://auth.globus.org/scopes/0948a6b0-a622-4078-b0a4-bfd6d77d65cf/test_scope",
            "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all",
        ],
    )

    mock_keystore.store.assert_called_with(mock_token_response)
    mock_authorizer_constructor.assert_called_with(
        "MyRefreshToken",
        mock_auth_client,
        access_token="MyAccessToken",
        expires_at=1024,
        on_refresh=mock_keystore.on_refresh,
    )

    assert gc.groups_authorizer == mock_authorizer

def test_client_previous_tokens_stored(
    mocker, mock_authorizer_tuple, token, mock_keystore
):
    mock_authorizer_constructor, mock_authorizer = mock_authorizer_tuple
    mock_auth_client = mocker.MagicMock(AuthClient)

    mock_keystore.file_exists.return_value = True
    mock_keystore.get_token_data.return_value = token

    # Mocks compute client login
    mock_login_manager = mocker.MagicMock(LoginManager)
    mock_login_manager.ensure_logged_in = mocker.Mock(return_value=True)
    mocker.patch(
        "globus_compute_sdk.sdk.client.LoginManager"
    ).return_value = mock_login_manager

    # Call the Garden constructor
    gc = GardenClient(auth_client=mock_auth_client)

    assert gc.auth_key_store == mock_keystore
    mock_auth_client.oauth2_exchange_code_for_tokens.assert_not_called()

    mock_authorizer_constructor.assert_called_with(
        "MyRefreshToken",
        mock_auth_client,
        access_token="MyAccessToken",
        expires_at=1024,
        on_refresh=mock_keystore.on_refresh,
    )

    assert gc.groups_authorizer == mock_authorizer

def test_client_invalid_auth_token(
    mocker, mock_authorizer_tuple, token, mock_keystore, identity_jwt
):
    mock_authorizer_constructor, mock_authorizer = mock_authorizer_tuple
    # Mocks for KeyStore
    mock_keystore.file_exists.return_value = False

    # Mocks for Login Flow
    mock_auth_client = mocker.MagicMock(AuthClient)
    mock_auth_client.oauth2_get_authorize_url = mocker.Mock(
        return_value="https://globus.auth.garden"
    )
    mock_auth_client.oauth2_start_flow = mocker.Mock()
    mocker.patch("garden_ai.client.Prompt.ask").return_value = "my token"
    mocker.patch("garden_ai.client.typer.launch")

    mock_token_response = mocker.MagicMock(OAuthTokenResponse)
    mock_token_response.data = {"id_token": identity_jwt}
    mock_token_response.by_resource_server = {"groups.api.globus.org": token}
    mock_token_response.status_code = "X"
    mock_token_response.request = mocker.Mock()
    mock_token_response.request.headers = {"Authorization": "Yougotit"}
    mock_token_response.request._underlying_response = mocker.Mock()
    mock_token_response.url = "http://foo.bar.baz"

    # Add a json() method to the mock_token_response
    mock_token_response.json = mocker.Mock(return_value={"error": "mock error"})

    # Add the 'reason' attribute to the mock_token_response
    mock_token_response.reason = mocker.Mock(return_value="Mock Reason")

    mock_auth_client.oauth2_exchange_code_for_tokens.side_effect = AuthAPIError(
        r=mock_token_response
    )
    # Call the Garden constructor and expect an auth exception
    with pytest.raises(AuthException):
        GardenClient(auth_client=mock_auth_client)

@pytest.mark.skipif((not is_gha), reason="Test only works in Github Actions.")
def test_client_credentials_grant(cc_grant_tuple):
    # Must run as github action to get client id and client secret from env vars
    client_id = cc_grant_tuple[0]
    client_secret = cc_grant_tuple[1]

    os.environ["FUNCX_SDK_CLIENT_ID"] = client_id
    os.environ["FUNCX_SDK_CLIENT_SECRET"] = client_secret

    confidential_client = ConfidentialAppAuthClient(client_id, client_secret)
    gc = GardenClient(auth_client=confidential_client)

    assert isinstance(gc.openid_authorizer, ClientCredentialsAuthorizer)
    assert isinstance(gc.groups_authorizer, ClientCredentialsAuthorizer)
    assert isinstance(gc.search_authorizer, ClientCredentialsAuthorizer)
    assert isinstance(gc.compute_authorizer, ClientCredentialsAuthorizer)
    assert isinstance(gc.garden_authorizer, ClientCredentialsAuthorizer)

    assert isinstance(gc.compute_client, Client)
    assert isinstance(gc.search_client, SearchClient)

    assert isinstance(gc.auth_client, ConfidentialAppAuthClient)

    assert gc.auth_client.oauth2_validate_token(gc.openid_authorizer.access_token)[
        "active"
    ]
    assert gc.auth_client.oauth2_validate_token(gc.groups_authorizer.access_token)[
        "active"
    ]
    assert gc.auth_client.oauth2_validate_token(gc.search_authorizer.access_token)[
        "active"
    ]
    assert gc.auth_client.oauth2_validate_token(gc.compute_authorizer.access_token)[
        "active"
    ]
    assert gc.auth_client.oauth2_validate_token(gc.garden_authorizer.access_token)[
        "active"
    ]


from functools import partial
from http.server import HTTPServer

from app.routes import routes
from core.environment import PORT
from core.request import RequestHandler

def get_http_server(port: int) -> HTTPServer:
    return HTTPServer(
        ('', port),
        partial(RequestHandler, routes=routes),
    )

if __name__ == '__main__':
    server = get_http_server(PORT)

    print('Starting HTTP server at port {}'.format(PORT))
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass
    finally:
        print('Stopping HTTP server')
        server.server_close()


import os
from .client import Client
from .fixture import get_latest_fixtures
from .context import file_func
from .club import get_club_details,determine_opponent_club

def calc_percentageplayed(raw_data):
    return scoreboard_get_percentageplayed(raw_data.get("scores"),5)

def calc_averagescore(raw_data):
    l15_score = scoreboard_get_averageScore(raw_data.get("scores"),15)
    if l15_score == 0:
        return 0
    l5_score = scoreboard_get_averageScore(raw_data.get("scores"),5)

    return ((l5_score * 100) / l15_score ) - 100

def eval_percentageplayed(value):
    """
    100 % => 10
    0% => 0
    """
    return value / 10

def eval_averagescore(value):
    """
    +20 % => 10
    0% => 0
    """
    if value > 20:
        return 10
    if value <= 0:
        return 0
    return value / 2

def calc_nextopponent(raw_data):
    l15_score = scoreboard_get_averageScore(raw_data.get("scores"),15)
    if l15_score == 0:
        return 0

    oppAverage = 0
    if raw_data.get("nextTeamScores",None) == None:
        return 0
    if len(raw_data.get("nextTeamScores")) == 0:
        return 0
    for oppScore in raw_data.get("nextTeamScores"):
        oppAverage = oppAverage + oppScore
    oppAverage = oppAverage / len(raw_data.get("nextTeamScores"))

    return ((oppAverage * 100) / l15_score ) - 100

def eval_nextopponent(value):
    """
    +20 % => 10
    0% => 0
    """
    if value > 20:
        return 10
    if value <= 0:
        return 0
    return value / 2

def data_reader_nextopponent(raw_data,score):
   score["gw"] = raw_data.get("nextOpponentGW")
   return

# ========== Age ==========
def calc_age(raw_data):
   return raw_data.get("head").get("age")
def eval_age(age):
  if age < 23:
     return 10
  if age < 27:
     return 9
  if age < 30:
     return 8
  value = 37-age
  if value < 0:
     value = 0
  return value

def is_player_slug_in_scoreboard_list(player_slug:str,scoreboard_list:list[dict])->bool:
  for scoreboard in scoreboard_list:
      if player_slug == scoreboard.get("player").get("slug"):
         return True
  return False

def get_player_scoreboard(client:Client,player_slug:str, options:dict = {}):
    fixture = get_latest_fixtures(client)[0]
    cachefile_scores_plus = os.path.dirname(os.path.abspath(__file__))+"/../../temp/sorare/cache/player/"+player_slug+"/scores_plus.json"
    cachefile = os.path.dirname(os.path.abspath(__file__))+"/../../temp/sorare/cache/player/"+player_slug+"/scoreboard.json"

    raw_data = get_scores_of_player(client,player_slug)
    scores = raw_data.get("scores")
    next_opponent = None

    # Next games of team
    if raw_data.get("head").get("activeClub",None) == None:
      my_club_slug = None
    else:
      my_club_slug = raw_data.get("head").get("activeClub").get("slug")
      clubDetails = get_club_details(client,my_club_slug)
      next_opponent = None
      next_opp_gw = None
      for upcomingGame in clubDetails.get("upcomingGames"):
        if upcomingGame.get("so5Fixture").get("gameWeek") < fixture.gameWeek:
           continue
        opponent = determine_opponent_club(my_club_slug,upcomingGame)
        next_opp_gw = upcomingGame.get("so5Fixture").get("gameWeek")
        print(opponent)
        next_opponent = opponent.get("opponent")
        break
      if next_opponent != None:
        print("Next opponent:"+next_opponent)
        last_3_scores = []
        for score in raw_data.get("scores"):
          if len(last_3_scores) >= 3:
            break
          opp = determine_opponent_club(my_club_slug,score.get("game")).get("opponent")
          #print(opp)
          if opp == next_opponent:
            last_3_scores.append(score.get("score"))
            #print(">>>>>>>>"+str(score.get("score")))
        #print(last_3_scores)
        raw_data["nextTeamScores"] = last_3_scores
        raw_data["nextOpponentSlug"] = opponent.get("opponent")
        raw_data["nextOpponentGW"] = next_opp_gw
    file_func.write_json_to_file(raw_data,cachefile_scores_plus)
    #print(scoreboard_get_percentageplayed(scores,5))
    #print(scoreboard_get_percentageplayed(scores,15))
    #print(scoreboard_get_averageScore(scores,5))
    #print(scoreboard_get_averageScore(scores,15))
    scoreboard = [{
        "description": "Percentage played",
        "calculator": calc_percentageplayed,
        "evaluator": eval_percentageplayed,
        "weight": 0.5
    },{
        "description": "Score improvement",
        "calculator": calc_averagescore,
        "evaluator": eval_averagescore,
        "weight": 0.5
    },{
        "description": "Next opponent",
        "dataReader": data_reader_nextopponent,
        "calculator": calc_nextopponent,
        "evaluator": eval_nextopponent,
        "weight": 0.5
    }]
    if options.get("includeAge",False) == True:
      scoreboard.append({
        "description": "Age",
        "calculator": calc_age,
        "evaluator": eval_age,
        "weight": 0.2
      })
    sum = 0
    weight_sum = 0
    for score in scoreboard:
        if score.get("dataReader") != None:
           score.get("dataReader")(raw_data,score)
           del score["dataReader"]

        score["calculated"] = score["calculator"](raw_data)
        del score["calculator"]
        if score["calculated"] != None:
          weight_sum = weight_sum + score["weight"]
          score["evaluated"] = score["evaluator"](score["calculated"])
        del score["evaluator"]
    for score in scoreboard:
      if score["calculated"] != None:
        sum = sum +  ( score["evaluated"] * ( score["weight"]  / weight_sum ) )

    #print(scoreboard)
    #print(sum)
    cache = {
        "player": {
           "slug": player_slug,
           "name": raw_data.get("head").get("displayName"),
           "position": raw_data.get("head").get("position"),
           "age": raw_data.get("head").get("age"),
        },
        "team": {
           "slug": my_club_slug
        },
        "nextGame": next_opponent,
        "scoreboard": scoreboard,
        "scoreboard_total": sum
    }
    file_func.write_json_to_file(cache,cachefile)
    return cache

def scoreboard_get_percentageplayed(score_list,number):
    scores = score_list[:number]
    count = 0
    for score in scores:
        if score.get("playerGameStats").get("minsPlayed") != None :
            if score.get("playerGameStats").get("minsPlayed") != 0:
              count =count + 1
    return (count * 100) / number

def scoreboard_get_averageScore(score_list,number):
    scores = score_list[:number]
    sum = 0
    count = 0
    for score in scores:
        if score.get("score") > 0:
            sum = sum + score.get("score")
            count = count + 1
    if count == 0:
        return 0
    return sum / count

def get_scores_of_player(client:Client,player_slug:str):
    cachefile = os.path.dirname(os.path.abspath(__file__))+"/../../temp/sorare/cache/player/"+player_slug+"/scores.json"

    body = """
query Player($endCursor: String $slug: String! $first: Int!) {
  football {
    player(slug: $slug) {
      displayName
      age
      averageScore(type: LAST_FIFTEEN_SO5_AVERAGE_SCORE)
      position
      activeClub {
        slug
        name
      }
      activeNationalTeam {
        slug
      }
      activeSuspensions {
        reason
        competition {
          slug
        }
      }
      activeInjuries {
        kind
        expectedEndDate
      }

      allSo5Scores (first:$first after:$endCursor){
        pageInfo { endCursor hasNextPage hasPreviousPage startCursor }
        nodes {
          score
          decisiveScore {
            totalScore
          }
          game {
            status
            so5Fixture {
              gameWeek
            }
             homeTeam {
          __typename ... on TeamInterface {
            name
            slug
          }
        }
        awayTeam {
          __typename ... on TeamInterface {
            name
            slug
          }
        }
            competition {
          name
          slug
        }
          }
          playerGameStats {
            goals
            minsPlayed
            team {
              __typename ... on TeamInterface {
                name
                id
                slug
              }
            }
          }
        }
      }
    }
  }
}

"""
    variables = {
        "slug": player_slug,
        "first": 50
    }
    options = {
        "resultSelector": ["data","football","player","allSo5Scores","nodes"],
        "pagination": {
            "targetNumber": 100000,
            "paginationVariable": "endCursor",
            "cursorSelector": ["data","football","player","allSo5Scores","pageInfo","endCursor"]
        }
    }
    result_list = client.request(body,variables,options)
    head_data = client.request(body,{
        "slug": player_slug,
        "first": 0
    },{
        "resultSelector": ["data","football","player"],
    })
    cache = {
        "head": head_data,
        "scores": result_list,
    }
    file_func.write_json_to_file(cache,cachefile)

    return cache


# https://programmers.co.kr/learn/courses/30/lessons/67260?language=python3

# solution 1 : recursion

import sys
sys.setrecursionlimit(1000000)

from collections import defaultdict

def solution(n, path, order):

    graph = defaultdict(list)
    for i, j in path:
        graph[i].append(j)
        graph[j].append(i)

    directed_graph = defaultdict(list)
    stack = [0]
    visited = [0]*n

    while stack:
        i = stack.pop()
        visited[i] = 1
        for j in graph[i]:
            if visited[j] == 0:
                directed_graph[i].append(j)
                stack.append(j)

    for i, j in order:
        directed_graph[i].append(j)

    stack = [0]
    visited = [0]*n
    visiting = [0]*n

    def is_tree(i):
        visited[i] = 1
        visiting[i] = 1
        for j in directed_graph[i]:
            if visiting[j]:
                return False
            if visited[j] == 0 and not is_tree(j):
                return False
        visiting[i] = 0
        return True

    return is_tree(0)

# solution 2: iteration

from collections import defaultdict

def solution(n, path, order):

    graph = defaultdict(list)
    requirement = {}
    unlock = {}

    for node1, node2 in path:
        graph[node1].append(node2)
        graph[node2].append(node1)

    for node1, node2 in order:
        requirement[node2] = node1
        unlock[node1] = node2

    visited = [0]*n
    stack = [0]
    plan = set()

    while stack:
        curr = stack.pop()

        # 방문 조건이 만족 안되는 경우
        if curr in requirement and visited[requirement[curr]] == 0:
            plan.add(requirement[curr])
            continue

        # 방문 조건 만족한 경우
        visited[curr] = 1
        for node in graph[curr]:
            if visited[node] == 0:
                stack.append(node)

        if curr in plan:
            stack.append(unlock[curr])
            plan.remove(curr)

    return not plan


from datetime import datetime

import requests
from decouple import config
from flask import Flask, render_template, request, jsonify
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config[
    'SQLALCHEMY_DATABASE_URI'] = f'postgresql://{config("DB_USER")}:{config("DB_PASSWORD")}@db/{config("DB_NAME")}'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

db = SQLAlchemy(app)

# migrate = Migrate(app, db)

class Weather:
    def __init__(self, city):
        self.city = city
        self.url = f"http://api.openweathermap.org/data/2.5/weather?q={city}&appid={config('API_KEY')}&units=metric"

    def get_weather_info(self):
        while True:
            self.city = self.city.strip().replace(" ", "%20")

            if self.city.lower() == "stop":
                print("Exiting program.")
                exit(0)

            response = requests.post(self.url)
            if response.status_code == 200:
                return response.json()
            else:
                raise Exception(f"Error getting weather data for {self.city}.")

class WeatherModel(db.Model):
    __tablename__ = 'weather'
    id = db.Column(db.Integer, primary_key=True)
    city = db.Column(db.String(50), unique=True, nullable=False)
    description = db.Column(db.String(50), nullable=False)
    temperature = db.Column(db.Float, nullable=False)
    humidity = db.Column(db.Integer, nullable=False)
    created_at = db.Column(db.DateTime, default=datetime.utcnow(), nullable=False)
    is_deleted = db.Column(db.Boolean, default=False, nullable=False)

    def __repr__(self):
        return f"Weather(city='{self.city}', temperature='{self.temperature}', humidity='{self.humidity}', description='{self.description}')"

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/refresh_data", methods=["POST"])
def refresh_data():
    city = request.form["city"].capitalize()
    weather = WeatherModel.query.filter(WeatherModel.city == city).filter(
        db.func.DATE_PART("day", db.func.now() - WeatherModel.created_at) < 1
    ).order_by("created_at").first()
    if not weather:
        return jsonify({"error": "Weather data not available for the selected city from today."})
    result = {
        "city": weather.city,
        "temp": weather.temperature,
        "humidity": weather.humidity,
        "description": weather.description,
    }
    return jsonify(result)

@app.route("/get_data", methods=["POST"])
def get_data():
    city = request.form["city"].capitalize()
    weather = Weather(city)
    data = weather.get_weather_info()
    description = data["weather"][0]["description"]
    temp = data["main"]["temp"]
    humidity = data["main"]["humidity"]
    # update weather data in the database
    existing_weather = WeatherModel.query.filter_by(city=city).first()
    if existing_weather:
        existing_weather.temperature = temp
        existing_weather.humidity = humidity
        existing_weather.description = description
        db.session.commit()
    else:
        new_weather = WeatherModel(city=city, temperature=temp, humidity=humidity, description=description)
        db.session.add(new_weather)
        db.session.commit()
    result = {
        "city": city,
        "temp": temp,
        "humidity": humidity,
        "description": description,
    }
    return jsonify(result)

if __name__ == "__main__":
    with app.app_context():
        db.create_all()
    app.run(debug=True, host="0.0.0.0")


from django import template

register = template.Library()

@register.filter
def mergelistclient(items):
    email_client = [item.email for item in items]
    return ', '.join(email_client)

@register.filter
def mergelistserverresponse(items):
    email_client = [f'{item.get_status_display()}: {item.server_response}' for item in items]
    return '\n'.join(email_client)


from unittest import mock

import celery  # type: ignore
import pytest

from config.settings import IMPORTANT_PORTS
from modules.scan.port_scan.port_scan import PortScan
from tests.conftest import (
    TEST_PORTS,
    TEST_PORT_SCAN_TARGET,
    TEST_TARGET,
    PORT_SCAN_MODULE_PATH,
)
from utils.custom_dataclasses import PortScanInput

class TestPortScan:
    test_socket_request_target = TEST_PORT_SCAN_TARGET
    test_target = TEST_TARGET
    test_ports = TEST_PORTS
    important_ports = IMPORTANT_PORTS
    module_path = PORT_SCAN_MODULE_PATH

    def test_port_scan_object_created_successfully(self, port_scan_fixture):
        """
        Test PortScan object is created successfully
        """
        assert port_scan_fixture.target == self.test_target
        assert port_scan_fixture.formatted_target == self.test_socket_request_target
        assert port_scan_fixture.ports == self.test_ports

    def test_all_ports_open_and_scanned_successfully(
        self,
        mocker,
        port_scan_fixture,
        mock_socket_request_task,
        mock_celery_group,
        mock_port_scan_convert_list_or_set_to_dict,
        mock_port_scan_store_module_results_in_database,
    ):
        """
        Test all calls for all ports are made
        """
        mocker.patch(f"{self.module_path}.socket_request", mock_socket_request_task)
        mocker.patch.object(celery, "group", mock_celery_group)

        port_scan_fixture._run_with_celery()

        assert mock_socket_request_task.s.call_count == len(self.test_ports)
        assert mock_socket_request_task.s.call_args_list == [
            mock.call(
                target=self.test_socket_request_target,
                port=port,
                module=self.module_path,
            )
            for port in self.test_ports
        ]

    @pytest.mark.parametrize(
        "port_scan_type, expected_ports",
        [
            ("all", set(range(1, 65536))),
            ("important", IMPORTANT_PORTS),
            ("top_1000", set(range(1, 1001))),
            ("custom", set()),
        ],
    )
    def test_port_scan_types(self, port_scan_type, expected_ports):
        port_scan_input = PortScanInput(port_scan_type=port_scan_type, ports=set())
        port_scan = PortScan(target=self.test_target, port_scan_input=port_scan_input)

        port_scan._determine_use_type()

        assert port_scan.ports == expected_ports

    @pytest.mark.parametrize(
        "invalid_port_scan_type",
        [
            None,
            "invalid",
            123,
        ],
    )
    def test_port_scan_type_none_fail(self, invalid_port_scan_type):
        port_scan_input = PortScanInput(
            port_scan_type=invalid_port_scan_type, ports=set()
        )
        port_scan = PortScan(target=self.test_target, port_scan_input=port_scan_input)

        with pytest.raises(ValueError):
            port_scan._determine_use_type()


__version__ = '2020-03-23'

def confMatplotlib(GlobFontSize, CanvasSize, Unit='inches'):
    """
    Configures the rcParameters in Matplotlib.

    Parameters
    ----------
    GlobFontSize : int
        Font size for all text in the plots
    CanvasSize : list of length 2
        Size of the drawing canvas in inches
    Unit : str, optional
        Either 'inches' (default) or 'cm'
    """
    import matplotlib.pyplot as plt

    if Unit == 'cm':
        CanvasSize = (CanvasSize[0] / 2.54, CanvasSize[1] / 2.54)
    elif Unit == 'inches':
        pass
    else:
        raise ValueError('{} is not a valid unit'.format(Unit))

    params = {
        'font.size': GlobFontSize,
        'font.family': 'arial',
        'figure.figsize': CanvasSize
    }
    plt.rcParams.update(params)

def getCustomLogger(ExpName, ExpFolder=None,
                    ConsoleLvl='INFO', FileLvl='DEBUG'):
    """
    Creates a custom logger for use in other modules.
    The logging data is saved to a .txt file related to ExpName

    Parameters
    ----------
    ExpName : str
        Name for the log file.
    ExpFolder : Path object, optional
        Path to the save location. If no path is given, the logfile
        will be saved in the same folder as the script that used
        'getCustomLogger' first.
    ConsoleLvl : str, optional
        The logging level shown in the shell. Case insensitive.
        One of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'].
        Default is 'INFO'.
    FileLvl : str, optional
        The logging level shown in the logfile. Case insensitive.
        One of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'].
        Default is 'DEBUG'.

    Returns
    -------
    logging.Logger object
        Custom logger
    """
    import logging
    import sys

    # create the logging object, use same ID name in all modules
    Sentinel = logging.getLogger("Watchtower")
    Sentinel.propagate = False  # do not report to root
    Sentinel.setLevel(logging.DEBUG)

    # Direct the logging from the Sentinel to the shell / console
    ShellLog = logging.StreamHandler(sys.stdout)
    ShellLog.setFormatter(logging.Formatter("%(message)s"))
    ShellLog.setLevel(getattr(logging, ConsoleLvl))
    Sentinel.addHandler(ShellLog)

    if ExpFolder is None:
        name = ExpName + '.log'
    else:
        name = ExpFolder / (ExpName + '.log')

    # Define a log file for the Sentinel output
    LogFile = logging.FileHandler(filename=name,
                                  mode="w", encoding="utf-8")
    LogFormatter = logging.Formatter("%(asctime)s : %(levelname)-8s : "
                                     "%(module)-25s : Line %(lineno)-4s : "
                                     "%(funcName)-25s : %(message)s",
                                     "%Y-%m-%d %H:%M:%S")
    LogFile.setFormatter(LogFormatter)
    LogFile.setLevel(getattr(logging, FileLvl))
    Sentinel.addHandler(LogFile)

    return Sentinel


import psutil
import os
listOfProcessNames = list()
# Iterate over all running processes
for proc in psutil.process_iter():
   # Get process detail as dictionary
   pInfoDict = proc.as_dict(attrs=['pid', 'name', 'cpu_percent'])
   # Append dict of process detail in list
   listOfProcessNames.append(pInfoDict)

found = False
for proc in listOfProcessNames:
    if proc['name'].lower() == 'smartscreen.exe':
        pid = proc['pid']
        print("Killing smartscreen process {} pid={}".format(proc,pid))
        os.kill(pid,9)
        found = True

if not found:
    print('smartscreen.exe was not running')


import json
import csv

file = open('xxx.bbox-0.264.json', 'r')
result = csv.writer(open('pseudo.csv', 'w'))
f = json.load(file)
pred = open('pred.csv', 'r')
pred = csv.reader(pred)

abnormal = []

for item in pred:
    if float(item[1]) > 0.08:
        abnormal.append(item[0])

category_match = {
    1: "Aortic enlargement",
    2: "Atelectasis",
    3: "Calcification",
    4: "Cardiomegaly",
    5: "Consolidation",
    6: "ILD",
    7: "Infiltration",
    8: "Lung Opacity",
    9: "Nodule/Mass",
    10: "Other lesion",
    11: "Pleural effusion",
    12: "Pleural thickening",
    13: "Pneumothorax",
    14: "Pulmonary fibrosis",
}

for i in f:
    if i['image_id'] in abnormal:
        # thres 0.5
        # if i['score'] > 0.5:
        #     result.writerow([i['image_id'], i['bbox'][0], i['bbox'][1], i['bbox'][0]+i['bbox'][2], i['bbox'][1]+i['bbox'][3], category_match[i['category_id']]])
        # scheduled thres
        if (i['score'] > 0.5) or ((i['category_id'] != 1 or 4) and i['score'] > 0.4):
            result.writerow([i['image_id'], i['bbox'][0], i['bbox'][1], i['bbox'][0]+i['bbox'][2], i['bbox'][1]+i['bbox'][3], category_match[i['category_id']]])


from socket import *
import threading
import sys # In order to terminate the program

class server():
    def __init__(self, port):
        self.port = port
        self.ip = ''
        self.threads = []

    def start_socket(self):
        try:
            self.server = socket(AF_INET, SOCK_STREAM)
            self.server.bind((self.ip, self.port))
            self.server.listen(10)
        except OSError as err:
            if self.server:
                self.server.close()
            print(err)
            sys.exit(1)

    def run(self):
        self.start_socket()
        run = True
        while run:
            try:
                c = Client(self.server.accept())
                c.start()
                self.threads.append(c)
            except KeyboardInterrupt:
                run = False
        for thrd in self.threads:
            thrd.join()

class Client(threading.Thread):
    def __init__(self, conn):
         threading.Thread.__init__(self)
         self.clientSocket = conn[0]
         self.address = conn[1]
         self.size = 1024

    def run(self):
        message = self.clientSocket.recv(self.size).decode()
        filename = message.split()[1][1:]
        try:
            with open(filename) as f:
                data = f.read()

            okay = "HTTP/1.1 200 OK\n Content-Type: text/html\n\n".encode()
            self.clientSocket.send(okay)
            for i in range(0, len(data)):
                self.clientSocket.send(data[i].encode())

            self.clientSocket.send("\r\n".encode())
            self.clientSocket.close()

        except FileNotFoundError:
            not_found = "HTTP/1.1 404 Not Found\n \n\n".encode()
            self.clientSocket.send(not_found)
            self.clientSocket.close()

s = server(8888)
s.run()


# https://leetcode.com/problems/lowest-common-ancestor-of-a-binary-tree/

class TreeNode:

    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

def lowestCommonAncestor(root: TreeNode, p: TreeNode, q: TreeNode):

    ans = None

    def dfs(node: TreeNode):
        if not node:
            return False
        lson = dfs(node.left)
        rson = dfs(node.right)
        nonlocal ans
        if (lson and rson) or ((node.val == p.val or node.val == q.val) and
                               (lson or rson)):
            ans = node

        return lson or rson or (node.val == p.val or node.val == q.val)

    dfs(root)
    return ans


import csv
import sys
import requests

def main():
    print(getCAPM('V', 'S&P 500'))
    # print(getTreasuryBonds())

def getCAPM(ticker, index):
    rf = getTreasuryBonds()
    rm = 10
    beta = covariance(ticker, index) / variance(index)
    betaNew = (1 - 1/3) * beta + 1/3
    # betaNew = beta
    print(rf, rm, betaNew)
    return rf + betaNew * (rm - rf)

def getTreasuryBonds():
    response = requests.get("https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/avg_interest_rates?filter=security_desc:eq:Treasury Bonds&sort=-record_date")
    return float(response.json()["data"][0]["avg_interest_rate_amt"])

def getData(ticker, data):
    with open(f'data/{ticker} Historical Data.csv', mode='r') as csv_file:
        csv_reader = csv.DictReader(csv_file)
        prices = {}
        line_count = 0
        for row in csv_reader:
            newRow = convert_row(row)
            if line_count == 0:
                print(f"Column names are {', '.join(newRow)}")
            print(newRow)
            if data == "Change %":
                prices[newRow["\"Date\""]] = float(newRow[data][:-1].replace(',', ''))
            else:
                prices[newRow["\"Date\""]] = float(newRow[data].replace(',', ''))
            line_count += 1
    return prices

def covariance(ticker1, ticker2):
    prices1 = getData(ticker1, "Change %")
    prices2 = getData(ticker2, "Change %")
    average1 = sum(prices1.values()) / len(prices1)
    average2 = sum(prices2.values()) / len(prices2)
    tot = 0
    for i in range(len(prices1)):
        tot += (list(prices1.values())[i] - average1) * (list(prices2.values())[i] - average2)
    return tot / (len(prices1) - 1)

def variance(ticker):
    prices = getData(ticker, "Change %")
    average = sum(prices.values()) / len(prices)
    tot = 0
    for i in range(len(prices)):
        tot += (list(prices.values())[i] - average) ** 2
    return tot / (len(prices) - 1)

def convert_row( row ):
    row_dict = {}
    for key, value in row.items():
        keyAscii = key.encode('ascii', 'ignore' ).decode()
        valueAscii = value.encode('ascii','ignore').decode()
        row_dict[ keyAscii ] = valueAscii
    return row_dict

main()


from flask import Blueprint, render_template, redirect, request
from modules.controller.list_controller import *
from flask_login import login_required, current_user
from model import Device, Sensor, Camera, Actuator

_device = Blueprint("devices", __name__, static_folder="../static", template_folder="../view")

@_device.route("/list", methods=["GET"])
@login_required
def device_list():
    return render_template("device_list.html")

@_device.route("/list/camera")
@login_required
def device_list_camera():
    devices = Camera.get_cameras()
    return render_template("cameras_list.html", devices = devices)

@_device.route("/list/actuator")
@login_required
def device_list_actuator():
    devices = Actuator.get_actuators()
    return render_template("actuators_list.html", devices = devices)

@_device.route("/list/sensor")
@login_required
def device_list_sensor():
    devices = Sensor.get_sensors()
    return render_template("sensors_list.html", devices = devices)

@_device.route("/register")
@login_required
def device_register():
    return render_template("device_register.html")

@_device.route("/register/camera")
@login_required
def device_register_camera():
    return render_template("camera_register.html")

@_device.route("/register/actuator")
@login_required
def device_register_actuator():
    return render_template("actuator_register.html")

@_device.route("/register/sensor")
@login_required
def device_register_sensor():
    return render_template("sensor_register.html")

@_device.route("/register/camera/save", methods=["POST"])
@login_required
def camera_save():
    name = request.form.get("name")
    brand = request.form.get("brand")
    model = request.form.get("model")
    description = request.form.get("description")
    voltage = request.form.get("voltage")
    status_checkbox = request.form.get("status")
    resolution = request.form.get("resolution")
    status = False
    if status_checkbox == 'on':
        status = True

    Camera.save_camera(name, brand, model, description, voltage, status, resolution)

    return redirect('/devices/list/camera')

@_device.route("/register/sensor/save", methods=["POST"])
@login_required
def sensor_save():
    name = request.form.get("name")
    brand = request.form.get("brand")
    model = request.form.get("model")
    description = request.form.get("description")
    voltage = request.form.get("voltage")
    status_checkbox = request.form.get("status")
    status = False
    if status_checkbox == 'on':
        status = True
    measure = "0"

    Sensor.save_sensor(name, brand, model, description, voltage, status, measure)

    return redirect('/devices/list/sensor')

@_device.route("/register/actuator/save", methods=["POST"])
@login_required
def actuator_save():
    name = request.form.get("name")
    brand = request.form.get("brand")
    model = request.form.get("model")
    description = request.form.get("description")
    voltage = request.form.get("voltage")
    status_checkbox = request.form.get("status")
    _type = request.form.get("type")
    if status_checkbox == 'on':
        status = True

    Sensor.save_sensor(name, brand, model, description, voltage, status, _type)

    return redirect('/devices/list/actuator')


import unittest

from core.Config import Config
from core.Flows import FlowType,PatternType,Flows

class Test(unittest.TestCase):

    def setUp(self):
        ##data init
        print("Start test...")
        self.cfgdict = {'srcInterfaceID': 'MSF21ADT', 'dstInterfaceID': 'AMPSTC21ADT', 'dstFlowName': 'CIM_2_NSTMMC21_AMPSTCADT', 'vf': 'NSTMMC21_MSFADT_VALID_2_MAP', 'mf': 'NSTMMC21_MSFADT_MAP_2_CIM', 'fac': 'NSTMMC21'}
        self.flowdict =  {'name': 'CIM_2_NSTMMC21_AMPSTCADT', 'flowType': FlowType.VF, 'config': self.cfgdict, 'patternType': PatternType.AMP}

    def test_config(self):
        """
        Test Config object
        """
        print("Start test of Config class...")
        c = Config('MSF21ADT','AMPSTC21ADT','CIM_2_NSTMMC21_AMPSTCADT')
        c.set_derived_val('NSTMMC21_MSFADT_VALID_2_MAP','NSTMMC21_MSFADT_MAP_2_CIM','NSTMMC21')
        data = vars(c)
        result = len(data)
        self.assertEqual(result, 6)
        self.assertDictEqual(data,self.cfgdict)

    def test_flows(self):
        """
        Test Flows object
        """
        print("Start test of Flows class...")
        c = Config('MSF21ADT','AMPSTC21ADT','CIM_2_NSTMMC21_AMPSTCADT')
        c.set_derived_val('NSTMMC21_MSFADT_VALID_2_MAP','NSTMMC21_MSFADT_MAP_2_CIM','NSTMMC21')
        flow = Flows(c.dstFlowName,FlowType.VF,vars(c),patternType=PatternType.AMP)
        data = vars(flow)
        #result = len(data)
        #self.assertEqual(result, 6)
        self.assertDictEqual(data,self.flowdict)

if __name__ == '__main__':
    unittest.main()


# import getpass
import os
import numpy as np
import pandas as pd
import copy
import umap
from collections import Counter
import getpass
from SciServer import Authentication, CasJobs
from sklearn import preprocessing
import warnings
import time
warnings.simplefilter("ignore")
# model_dict={}
from code.prepro.label import get_stellar_pd

ckeys=['ug','ur', 'ui','uz','gz', 'gi', 'gr','rz','ri','iz']
mkeys=['z']
# SPEC_DATA='../sloan_stars/DR13v1/sql_544k_lbt.csv'
ftr=ckeys+mkeys
dtype='int64'
PRETRAIN_PATH='./pretrain'

def get_norm_pd(df,r=0.01,std=False):
    df1=(df-df.mean())/df.std() if std else df
    vmin=df1.quantile(r)
    vrng=df1.quantile(1-r)-vmin
    df_new=((df1-vmin)/vrng).clip(0,1)
    return df_new,vmin,vrng

def get_prepro_pds(df_full,ftr=ftr, r=0.01):
    df_norm,vmin,vrng=get_norm_pd(df_full[ftr], r=r,std=False)
    df_label=get_stellar_pd(df_full[['class','subclass']])
    return df_norm,vmin,vrng,df_label

def prepro_specs(SPEC_DATA,ftr=ftr, r=0.01,w=True,wpath=PRETRAIN_PATH):
    dfspec=pd.read_csv(SPEC_DATA)
    dfspec_norm,vmin,vrng,df_label=get_prepro_pds(dfspec,r=0.01)
    print(dfspec_norm.head(),vmin,vrng,)
    print('label',df_label.head() )
    if w:
        print(f'writing to {wpath}')
        np.savetxt(f'{wpath}/vmin.txt',vmin)
        np.savetxt(f'{wpath}/vrng.txt',vrng)
        dfspec_norm.to_csv(f'{wpath}/spec_norm.csv',index=False)
        df_label.to_csv(f'{wpath}/spec_lbl.csv', index=False)
    return dfspec_norm,vmin,vrng,df_label

def load_norm_params(vpath):
    vmin=np.loadtxt(f'{vpath}/vmin.txt')
    vrng=np.loadtxt(f'{vpath}/vrng.txt')
    return vmin, vrng

def prepro_photos(PHOTO_DATA, vmin, vrng, base, w=False, ftr=ftr,vpath=PRETRAIN_PATH):
    dfphoto=pd.read_csv(PHOTO_DATA)
    assert len(ftr)==len(vmin) & len(ftr)==len(vrng)
    df=((dfphoto[ftr]-vmin)/vrng).clip(0,1)
    if w:
        print(f'writing photo_norm_{base} to {vpath}')
        df.to_csv(f'{vpath}/photo_norm_{base}.csv', index=False)
    return df

    # df_label['encode']=full_encode;

#######################ENCODE######################
def horner_encode(mat,base,dtype):
    r,c=mat.shape
    print('samples:',r,'ftrs:',c, 'base:',base)
    encode=np.zeros((r),dtype=dtype)
    for ii, key in enumerate(mat.keys()):
        val=(mat[key].values).astype(dtype)
        encode= encode + val*(base**ii)
#         print(ii,val, encode)
    return encode

def horner_decode(encode,base, dim,dtype):
    arr=copy.deepcopy(np.array(encode))
    decode=np.zeros((len(arr),dim), dtype=dtype)
    for ii in range(dim-1,-1,-1):
        digits=arr//(base**ii)
        decode[:,ii]=digits
        arr= arr% (base**ii)
#         print(digits,arr)
    return decode

def get_encode_stream1D(df_norm, base,dtype):
    mat=(df_norm*(base-1)).round()
    assert (mat.min().min()>=0) & (mat.max().max()<=base-1)
    mat_encode=horner_encode(mat,base,dtype)
    mat_decode=horner_decode(mat_encode,base,len(mat.keys()),dtype)
    try:
        assert np.sum(abs(mat_decode-mat.values))<=0.0001
    except:
        print(np.nonzero(np.sum(abs(mat_decode-mat.values),axis=1)), np.sum(abs(mat_decode-mat.values)))
        raise 'overflow, try lower base or fewer features'
    return mat_encode


import csv

action = ''
result = True
'''items = [{'name':['Milk','Sugar','Flour','Coffe','Tea']}, {'quantity':[1,2,3,4,5]},{'unit':['ml','mg','mg','mg','lll']},{'unit_price':[1,2,3,4,5]}]'''

items_modified= [{'name':'Milk','quantity':1,'unit':'ml','unit_price':1}, {'name':'Coffe','quantity':2,'unit':'ml','unit_price':2}, {'name':'Tea','quantity':3,'unit':'ml','unit_price':3}, {'name':'Wine','quantity':4,'unit':'ml','unit_price':4}]
sold_items = []

#func that exports shop resouces status to the .csv file
def export_file_to_csv():
    with open('magazyn.csv', 'w', newline='') as csvfile:
        fieldnames = ['name','quantity','unit','unit_price']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        for product in items_modified:
            writer.writerow({'name':product['name'],'quantity':product['quantity'],'unit':product['unit'],'unit_price':product['unit_price']})
        print('Successfully exported data to magazyn.csv.')

#func that imports shop resouces status from the .csv file
def import_items_from_csv():
    with open('magazyn.csv', newline='') as csvfile:
        reader = csv.DictReader(csvfile)

        for row in reader:
            items_modified.append({'name':row['name'], 'quantity':row['quantity'], 'unit':row['unit'], 'unit_price':row['unit_price']})
        print('Successfully loaded data from magazyn.csv.')
        return items_modified

#adding new items to shop func
def add_item(name='', quantity ='', unit='', unit_price=''):
    print('Adding to warehouse...')
    items_modified.append({'name':input('Item name: ').capitalize(),'quantity':input('Item quantity: ').capitalize(), 'unit':input('Item unit of measure. Eg. l, kg, pcs: ').capitalize(), 'unit_price':input('Item price in PLN: ').capitalize()})

    print('Successfully added to warehouse. The current status is: ')
    get_items()

#func that sums up the items that were sold
def sold_items_summary(name, quantity,unit,unit_price):
    sold_items.append({'name':name, 'quantity':quantity, 'unit':unit, 'unit_price':unit_price})

#costs calculation func
def get_costs(sum_of_costs=0,list_of_costs=[]):
    list_of_costs = [product['quantity'] * product['unit_price'] for product in items_modified]
    sum_of_costs = sum(list_of_costs)
    return sum_of_costs

#income calculation func
def get_income(sum_of_income=0,list_of_income=[]):
    list_of_income = [product['quantity'] * product['unit_price'] for product in sold_items]
    sum_of_income = sum(list_of_income)
    return sum_of_income

#revenue calculation func
def show_revenue():
    print('Revenue breakdown (PLN)')
    print(f'Income:', get_income())
    print(f'Costs:', get_costs())
    print('------------')
    print(f'Revenue:', get_income() - get_costs(),' PLN')

#selling items func
def sell_item(name='', quantity=int()):
    name = input('Item name: ')
    name = name.capitalize()
    i = len(items_modified)
    for product in items_modified:
        if name not in product.values():
            i -= 1
            if i == 0:
                print('There is no such product in the shop.')
        else:
            quantity = int(input('Quantity to sell: '))

            if product['quantity'] >= quantity:
                product['quantity'] = product['quantity']- quantity
                print('Successfully sold', product['quantity'], 'of ', name)
                sold_items_summary(name, quantity,unit = product['unit'],unit_price = product['unit_price'])
                get_items()
            else:
                buy_all = input('There is no enough quantity to fulfil your request. Do you want to buy what is available? (yes/no')
                if buy_all == 'yes':
                    product['quantity'] = product['quantity']- quantity
                    if product['quantity'] < 0:
                        quantity = quantity + product['quantity']
                        product['quantity'] = 0
                        print('Successfully sold', product['quantity'], 'of ', name)
                        sold_items_summary(name, quantity,unit = product['unit'],unit_price = product['unit_price'])

                        get_items()
                else:
                    print('You can choose another item or different quantity. ')

#func that shows current shop resurces status
def get_items():
    print('Name\tQuantity\tUnit\tUnit Price (PLN)')
    print('----\t--------\t----\t----------------')

    for item in items_modified:
        print(item['name'],'\t',item['quantity'], '\t       ',item['unit'],'\t',item['unit_price'])

#function that triggers actions
def to_do(action):
    if action == 'show':
        get_items()
    elif action == 'add':
        add_item()
    elif action == 'sell':
        sell_item()
    elif action == 'show rev':
        show_revenue()
    elif action == 'save':
        export_file_to_csv()
    elif action == 'load':
        items_modified.clear()
        import_items_from_csv()
    elif action == 'exit':
        return True
    else:
        return False

#main program logic
import_items_from_csv()
while to_do(action) != True:
    action = str(input('What would you like to do? '))


'''
Working with categorical data
=============================

use of dummy variables, group statistics, within and between statistics
examples for efficient matrix algebra

dummy versions require that the number of unique groups or categories is not too large
group statistics with scipy.ndimage can handle large number of observations and groups
scipy.ndimage stats is missing count

new: np.bincount can also be used for calculating values per label
'''
from statsmodels.compat.python import lrange
import numpy as np

from scipy import ndimage

#problem: ndimage does not allow axis argument,
#   calculates mean or var corresponding to axis=None in np.mean, np.var
#   useless for multivariate application

def labelmeanfilter(y, x):
    # requires integer labels
    # from mailing list scipy-user 2009-02-11
    labelsunique = np.arange(np.max(y)+1)
    labelmeans = np.array(ndimage.mean(x, labels=y, index=labelsunique))
    # returns label means for each original observation
    return labelmeans[y]

#groupcount: i.e. number of observation by group/label
#np.array(ndimage.histogram(yrvs[:,0],0,10,1,labels=yrvs[:,0],index=np.unique(yrvs[:,0])))

def labelmeanfilter_nd(y, x):
    # requires integer labels
    # from mailing list scipy-user 2009-02-11
    # adjusted for 2d x with column variables

    labelsunique = np.arange(np.max(y)+1)
    labmeansdata = []
    labmeans = []

    for xx in x.T:
        labelmeans = np.array(ndimage.mean(xx, labels=y, index=labelsunique))
        labmeansdata.append(labelmeans[y])
        labmeans.append(labelmeans)
    # group count:
    labelcount = np.array(ndimage.histogram(y, labelsunique[0], labelsunique[-1]+1,
                        1, labels=y, index=labelsunique))

    # returns array of lable/group counts and of label/group means
    #         and label/group means for each original observation
    return labelcount, np.array(labmeans), np.array(labmeansdata).T

def labelmeanfilter_str(ys, x):
    # works also for string labels in ys, but requires 1D
    # from mailing list scipy-user 2009-02-11
    unil, unilinv = np.unique(ys, return_index=False, return_inverse=True)
    labelmeans = np.array(ndimage.mean(x, labels=unilinv, index=np.arange(np.max(unil)+1)))
    arr3 = labelmeans[unilinv]
    return arr3

def groupstatsbin(factors, values):
    '''uses np.bincount, assumes factors/labels are integers
    '''
    n = len(factors)
    ix,rind = np.unique(factors, return_inverse=1)
    gcount = np.bincount(rind)
    gmean = np.bincount(rind, weights=values)/ (1.0*gcount)
    meanarr = gmean[rind]
    withinvar = np.bincount(rind, weights=(values-meanarr)**2) / (1.0*gcount)
    withinvararr = withinvar[rind]
    return gcount, gmean , meanarr, withinvar, withinvararr

def convertlabels(ys, indices=None):
    '''convert labels based on multiple variables or string labels to unique
    index labels 0,1,2,...,nk-1 where nk is the number of distinct labels
    '''
    if indices is None:
        ylabel = ys
    else:
        idx = np.array(indices)
        if idx.size > 1 and ys.ndim == 2:
            ylabel = np.array(['@%s@' % ii[:2].tostring() for ii in ys])[:,np.newaxis]
            #alternative
    ##        if ys[:,idx].dtype.kind == 'S':
    ##            ylabel = nd.array([' '.join(ii[:2]) for ii in ys])[:,np.newaxis]
        else:
            # there might be a problem here
            ylabel = ys

    unil, unilinv = np.unique(ylabel, return_index=False, return_inverse=True)
    return unilinv, np.arange(len(unil)), unil

def groupsstats_1d(y, x, labelsunique):
    '''use ndimage to get fast mean and variance'''
    labelmeans = np.array(ndimage.mean(x, labels=y, index=labelsunique))
    labelvars = np.array(ndimage.var(x, labels=y, index=labelsunique))
    return labelmeans, labelvars

def cat2dummy(y, nonseq=0):
    if nonseq or (y.ndim == 2 and y.shape[1] > 1):
        ycat, uniques, unitransl =  convertlabels(y, lrange(y.shape[1]))
    else:
        ycat = y.copy()
        ymin = y.min()
        uniques = np.arange(ymin,y.max()+1)
    if ycat.ndim == 1:
        ycat = ycat[:,np.newaxis]
    # this builds matrix nobs*ncat
    dummy = (ycat == uniques).astype(int)
    return dummy

def groupsstats_dummy(y, x, nonseq=0):
    if x.ndim == 1:
        # use groupsstats_1d
        x = x[:,np.newaxis]
    dummy = cat2dummy(y, nonseq=nonseq)
    countgr = dummy.sum(0, dtype=float)
    meangr = np.dot(x.T,dummy)/countgr
    meandata = np.dot(dummy,meangr.T) # category/group means as array in shape of x
    xdevmeangr = x - meandata  # deviation from category/group mean
    vargr = np.dot((xdevmeangr * xdevmeangr).T, dummy) / countgr
    return meangr, vargr, xdevmeangr, countgr


from app.db_setup import User_profile, User_session

from uuid import uuid4
import datetime
import hashlib
import random
import string

# Функция хеширования
_hash = lambda x : hashlib.md5((x).encode()).hexdigest()

class DB_Activate():
    def __init__(self, session) -> None:
        self.session = session

    def db_decorator(func):
        def db_use(self, *args, **kwargs):
            result = None
            session = None

            try:
                session = self.session()
                result = func(*args, **kwargs, session=session)
            except Exception as err:
                print("ERR: ", err)
                if session is not None:
                    session.rollback()
            if session is not None:
                session.close()
            return result
        return db_use

    @db_decorator
    def get_first_filter(db_table, session=None, search=()):
        return session.query(db_table).filter(search).first()

    @db_decorator
    def get_all_filter(db_table, session=None, search=()):
        return session.query(db_table).filter(search).all()

    @db_decorator
    def add(db_table, session=None):
        session.add(db_table)
        session.commit()
        return db_table.id

    @db_decorator
    def update(db_table, reload={}, session=None, search=()):
        session.query(db_table).filter(search).update(reload)
        session.commit()

    @db_decorator
    def dell(db_table, session=None, search=()):
        db_list = session.query(db_table).filter(search).all()
        for db in db_list:
            session.delete(db)
        session.commit()

    @db_decorator
    def new_session(id, session=None):
        date_now = datetime.datetime.now(datetime.timezone.utc)
        user = User_session(user_id=id, last_using=date_now, session=(uuid4().hex))
        session.add(user)
        session.commit()

        return user.session

    @db_decorator
    def new_user(email, password, name, session=None):
        date_now = datetime.datetime.now(datetime.timezone.utc)
        _salt = ''.join(random.choice(string.ascii_letters) for x in range(30))
        _hashpass = _hash(password + _salt)

        db_user = User_profile(email=email, hashpass=_hashpass, salt=_salt, name=name)
        session.add(db_user)
        session.commit()

        db_user_session = User_session(user_id=db_user.id, last_using=date_now, session=(uuid4().hex))
        session.add(db_user_session)
        session.commit()

        return db_user_session.session


import sqlite3
import yaml

try:

    yml_file = "../Config/conig.yml"
    with open(yml_file, "r") as f:
        data = yaml.safe_load(f)

    fields = data['fields']

    jsonFilePath = data['output_file']
    dataBaseFilePath = data['data_file']
    tableName = data['table_name']

    connection = sqlite3.connect(dataBaseFilePath)
    cursor = connection.cursor()

    cursor.execute(f"SELECT * FROM {tableName}")
    rows = cursor.fetchall()

    for row in rows:
        print(row)

    connection.close()

except:
    print("File not found!")