import logging
import os
import sys

import boto3
from botocore.exceptions import ClientError
from pyspark import SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.functions import *

logging.basicConfig()
LOGGER = logging.getLogger(__name__)
LOGGER.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'DEBUG')))
LOGGER.debug(__name__)
GLUE = boto3.client('glue')

def get_database(database):

    try:
        return GLUE.get_database(Name=database)['Database']
    except ClientError as err:
        LOGGER.warning("Database not found: %s", err)
        GLUE.create_database(
            DatabaseInput={
                'Name': database,
                'Description': 'gsg bike index',
                'Parameters': {
                    'System': 'bike'
                }
            })
    return GLUE.get_database(Name=database)['Database']

def create_table(database, table, columns, location, partition) -> dict:
    table_input = {
        'Name': table,
        'Description': 'bike lake',
        'TableType': 'EXTERNAL_TABLE',
        'Parameters': dict(
            System='bike',
            External=str(True)),
        'StorageDescriptor': {
            'Columns': columns,
            'Location': location.replace("s3a", "s3"),
            'InputFormat': 'org.apache.hudi.hadoop.HoodieParquetInputFormat',
            'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat',
            'NumberOfBuckets': -1,
            'SerdeInfo': {'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
                          "Parameters": {
                              "serialization.format": "1"
                          }}}}
    if partition:
        table_input['PartitionKeys'] = [dict(Name='part', Type='string')]
    return GLUE.create_table(
        DatabaseName=get_database(database)['Name'],
        TableInput=table_input)

def get_catalog(event, dataframe) -> dict:
    try:
        return boto3.client('glue').get_table(DatabaseName=event['database'], Name=event['table']).get('Table')
    except ClientError as e:
        if e.response['Error'].get('Code') == 'EntityNotFoundException':
            create_table(event['database'], event['table'], columns(dataframe, event), event['location'],
                         event['partition'])
    return boto3.client('glue').get_table(DatabaseName=event['database'], Name=event['table']).get('Table')

def columns(dataframe, event) -> list:
    columns = list()
    for col in dataframe.dtypes:
        name = str(col[0]).lower()
        type = str(col[1]).lower()
        if name != "part":
            columns.append(dict(
                Name=name,
                Type=type))

    return columns

def get_config_hudi(key, partition, table, tiebreaker):
    configs = {
        "hoodie.datasource.write.keygenerator.class": "org.apache.hudi.keygen.ComplexKeyGenerator",
        "hoodie.datasource.write.payload.class": "org.apache.hudi.common.model.DefaultHoodieRecordPayload",
        "hoodie.datasource.hive_sync.partition_extractor_class": "org.apache.hudi.hive.MultiPartKeysValueExtractor",
        "hoodie.table.name": table,
        "hoodie.datasource.write.recordkey.field": key,
        "hoodie.datasource.write.partitionpath.field": 'part',
        "hoodie.datasource.write.hive_style_partitioning": "true",
        "hoodie.datasource.write.precombine.field": tiebreaker,
        "hoodie.datasource.write.operation": "upsert",

        #"hoodie.write.concurrency.mode": "single_writer",
        "hoodie.cleaner.commits.retained": 2,

        "hoodie.bloom.index.use.caching": True,
        "hoodie.parquet.compression.codec": "snappy",
        #"hoodie.parquet.max.file.size": 64
    }
    return configs

def read_data_frame(spark, event):
    dataframe = spark.read.format(event['file_format']).load(event['raw_data'])
    return dataframe

def hudi_upsert(dataframe, config, event):
    try:
        dataframe.write.format("hudi").options(**config).mode("append").save(event['location'])
    except Exception as err:
        LOGGER.error('ERROR: %s', err)
        raise err

def apply_partition(dataframe, event):
    #todo: should be a function for any partition
    dataframe = dataframe.withColumn('part', from_unixtime(col(event['partition']), "yyyyMMdd"))
    return dataframe

def hudi_handler(event, spark):
    dataframe = read_data_frame(spark, event)
    dataframe = apply_partition(dataframe, event)

    catalog = get_catalog(event, dataframe)

    configs = get_config_hudi(event['key'], event['partition'], event['table'], event['tiebreaker'])

    #partition = event['partition'].split(',')

    hudi_upsert(dataframe, configs, event)

    #partitions_list = [row.asDict() for row in dataframe.select(partition).distinct().collect()]
    #todo: update partition catalog
    # update_catalog(events, paths, partitions_list)

def init_spark():
    conf = SparkConf()
    conf.set("spark.jars.packages",
             "org.apache.hudi:hudi-spark3-bundle_2.12:0.8.0," "org.apache.hadoop:hadoop-aws:3.2.0,")
    conf.set("spark.sql.sources.partitionOverwriteMode", "dynamic")
    conf.set("spark.sql.files.ignoreMissingFiles", "true")
    conf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
    conf.set("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
    conf.set("spark.hadoop.fs.s3n.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
    conf.set("spark.hadoop.fs.AbstractFileSystem.s3a.impl", "org.apache.hadoop.fs.s3a.S3A")
    conf.set("spark.hadoop.fs.AbstractFileSystem.s3n.impl", "org.apache.hadoop.fs.s3a.S3A")
    conf.set("spark.hadoop.fs.AbstractFileSystem.s3.impl", "org.apache.hadoop.fs.s3a.S3A")
    conf.set("spark.hadoop.fs.s3a.aws.credentials.provider",
                        "com.amazonaws.auth.DefaultAWSCredentialsProviderChain")
                       #"com.amazonaws.auth.EnvironmentVariableCredentialsProvider")
    spark = SparkSession.builder.master("local[*]").config(conf=conf).getOrCreate()
    spark.sparkContext.setLogLevel("INFO")
    return spark

def mock_event_bike_search():
    event = dict()
    event['key'] = 'hash'
    event['partition'] = 'bike_created_at'
    event['table'] = 'search'
    event['tiebreaker'] = 'current_ts'
    event['database'] = 'bikeindex'
    event['location'] = get_step() + 'analytics/bikeindex/search/'
    event['raw_data'] = 'raw_data/bike-search/'
    event['file_format'] = 'json'
    return event

def mock_event_bike():
    event = dict()
    event['key'] = 'id'
    event['partition'] = 'registration_created_at'
    event['table'] = 'bike'
    event['tiebreaker'] = 'registration_updated_at'
    event['database'] = 'bikeindex'
    event['location'] = get_step() + 'analytics/bikeindex/bike'
    event['raw_data'] = 'raw_data/bike/'
    event['file_format'] = 'json'
    return event

def mock_event_bike_stolen():
    event = dict()
    event['key'] = 'key'
    event['partition'] = 'bike_created_at'
    event['table'] = 'stolen'
    event['tiebreaker'] = 'bike_updated_at'
    event['database'] = 'bikeindex'
    event['location'] = get_step() + 'analytics/bikeindex/stolen/'
    event['raw_data'] = 'raw_data/bike-stolen/'
    event['file_format'] = 'json'
    return event

def get_step():
    for count, item in enumerate(sys.argv):
        if item == '--bucket_name':
            return (sys.argv[count + 1])
    return ""

if __name__ == "__main__":

    spark = init_spark()

    event_bike_search = mock_event_bike_search()
    hudi_handler(event_bike_search, spark)

    event_bike = mock_event_bike()
    hudi_handler(event_bike, spark)

    event_bike_stolen = mock_event_bike_stolen()
    hudi_handler(event_bike_stolen, spark)


"""
``tractor`` testing!!
"""
import sys
import subprocess
import os
import random
import signal
import platform
import pathlib
import time
import inspect
from functools import partial, wraps

import pytest
import trio
import tractor

pytest_plugins = ['pytester']

def tractor_test(fn):
    """
    Use:

    @tractor_test
    async def test_whatever():
        await ...

    If fixtures:

        - ``arb_addr`` (a socket addr tuple where arbiter is listening)
        - ``loglevel`` (logging level passed to tractor internals)
        - ``start_method`` (subprocess spawning backend)

    are defined in the `pytest` fixture space they will be automatically
    injected to tests declaring these funcargs.
    """
    @wraps(fn)
    def wrapper(
        *args,
        loglevel=None,
        arb_addr=None,
        start_method=None,
        **kwargs
    ):
        # __tracebackhide__ = True

        if 'arb_addr' in inspect.signature(fn).parameters:
            # injects test suite fixture value to test as well
            # as `run()`
            kwargs['arb_addr'] = arb_addr

        if 'loglevel' in inspect.signature(fn).parameters:
            # allows test suites to define a 'loglevel' fixture
            # that activates the internal logging
            kwargs['loglevel'] = loglevel

        if start_method is None:
            if platform.system() == "Windows":
                start_method = 'trio'

        if 'start_method' in inspect.signature(fn).parameters:
            # set of subprocess spawning backends
            kwargs['start_method'] = start_method

        if kwargs:

            # use explicit root actor start

            async def _main():
                async with tractor.open_root_actor(
                    # **kwargs,
                    arbiter_addr=arb_addr,
                    loglevel=loglevel,
                    start_method=start_method,

                    # TODO: only enable when pytest is passed --pdb
                    # debug_mode=True,

                ):
                    await fn(*args, **kwargs)

            main = _main

        else:
            # use implicit root actor start
            main = partial(fn, *args, **kwargs)

        return trio.run(main)

    return wrapper

_arb_addr = '127.0.0.1', random.randint(1000, 9999)

# Sending signal.SIGINT on subprocess fails on windows. Use CTRL_* alternatives
if platform.system() == 'Windows':
    _KILL_SIGNAL = signal.CTRL_BREAK_EVENT
    _INT_SIGNAL = signal.CTRL_C_EVENT
    _INT_RETURN_CODE = 3221225786
    _PROC_SPAWN_WAIT = 2
else:
    _KILL_SIGNAL = signal.SIGKILL
    _INT_SIGNAL = signal.SIGINT
    _INT_RETURN_CODE = 1 if sys.version_info < (3, 8) else -signal.SIGINT.value
    _PROC_SPAWN_WAIT = 0.6 if sys.version_info < (3, 7) else 0.4

no_windows = pytest.mark.skipif(
    platform.system() == "Windows",
    reason="Test is unsupported on windows",
)

def repodir() -> pathlib.Path:
    '''
    Return the abspath to the repo directory.

    '''
    # 2 parents up to step up through tests/<repo_dir>
    return pathlib.Path(__file__).parent.parent.absolute()

def examples_dir() -> pathlib.Path:
    '''
    Return the abspath to the examples directory as `pathlib.Path`.

    '''
    return repodir() / 'examples'

def pytest_addoption(parser):
    parser.addoption(
        "--ll", action="store", dest='loglevel',
        default='ERROR', help="logging level to set when testing"
    )

    parser.addoption(
        "--spawn-backend", action="store", dest='spawn_backend',
        default='trio',
        help="Processing spawning backend to use for test run",
    )

def pytest_configure(config):
    backend = config.option.spawn_backend
    tractor._spawn.try_set_start_method(backend)

@pytest.fixture(scope='session', autouse=True)
def loglevel(request):
    orig = tractor.log._default_loglevel
    level = tractor.log._default_loglevel = request.config.option.loglevel
    tractor.log.get_console_log(level)
    yield level
    tractor.log._default_loglevel = orig

@pytest.fixture(scope='session')
def spawn_backend(request) -> str:
    return request.config.option.spawn_backend

_ci_env: bool = os.environ.get('CI', False)

@pytest.fixture(scope='session')
def ci_env() -> bool:
    """Detect CI envoirment.
    """
    return _ci_env

@pytest.fixture(scope='session')
def arb_addr():
    return _arb_addr

def pytest_generate_tests(metafunc):
    spawn_backend = metafunc.config.option.spawn_backend

    if not spawn_backend:
        # XXX some weird windows bug with `pytest`?
        spawn_backend = 'trio'

    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
    assert spawn_backend in (
        'mp_spawn',
        'mp_forkserver',
        'trio',
    )

    # NOTE: used to be used to dyanmically parametrize tests for when
    # you just passed --spawn-backend=`mp` on the cli, but now we expect
    # that cli input to be manually specified, BUT, maybe we'll do
    # something like this again in the future?
    if 'start_method' in metafunc.fixturenames:
        metafunc.parametrize("start_method", [spawn_backend], scope='module')

def sig_prog(proc, sig):
    "Kill the actor-process with ``sig``."
    proc.send_signal(sig)
    time.sleep(0.1)
    if not proc.poll():
        # TODO: why sometimes does SIGINT not work on teardown?
        # seems to happen only when trace logging enabled?
        proc.send_signal(_KILL_SIGNAL)
    ret = proc.wait()
    assert ret

@pytest.fixture
def daemon(
    loglevel: str,
    testdir,
    arb_addr: tuple[str, int],
):
    '''
    Run a daemon actor as a "remote arbiter".

    '''
    if loglevel in ('trace', 'debug'):
        # too much logging will lock up the subproc (smh)
        loglevel = 'info'

    cmdargs = [
        sys.executable, '-c',
        "import tractor; tractor.run_daemon([], registry_addr={}, loglevel={})"
        .format(
            arb_addr,
            "'{}'".format(loglevel) if loglevel else None)
    ]
    kwargs = dict()
    if platform.system() == 'Windows':
        # without this, tests hang on windows forever
        kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP

    proc = testdir.popen(
        cmdargs,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        **kwargs,
    )
    assert not proc.returncode
    time.sleep(_PROC_SPAWN_WAIT)
    yield proc
    sig_prog(proc, _INT_SIGNAL)


"""
http 功能演示

将网页发送给浏览器展示

"""

from socket import *

# 搭建tcp网路

def main():
    sock= socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(('0.0.0.0', 8000))
    sock.listen(3)
    print('Listen the port 8000...')
    while True:
        connfd, addr = sock.accept()
        handle(connfd)

def handle(conn):
    print('Request from', conn.getpeername())
    request = conn.recv(4096)
    if not request:
        return
    request_line = request.splitlines()[0].decode()
    info = request_line.split(' ')[1]
    if info == '/':
        f = open("index.html", 'r', encoding='UTF-8')
        response = "HTTP/1.1 200 OK\r\n"
        response += "Content-Type: text/html\r\n"
        response += '\r\n'
        response += f.read()
    else:
        response = "HTTP/1.1 404 Not Found \r\n"
        response += "Content-Type: text/html\r\n"
        response += '\r\n'
        response += "
Sorry...
"
    conn.send(response.encode())

if __name__ == '__main__':
    main()


import pygame
import cv2 as cv
import numpy as np
import imutils
from ball import Ball
from cart import Cart
from game import PygameGame

def rescaleFrame(frame, scale):
    width = int(frame.shape[1] * scale)
    height = int(frame.shape[0] * scale)
    dimensions = (width, height)
    return cv.resize(frame, dimensions, interpolation=cv.INTER_AREA)

def detect_and_draw_balls(frame, lower_range, upper_range, color):
    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)

    # Detect balls
    mask = cv.inRange(hsv, lower_range, upper_range)
    mask = cv.erode(mask, None, iterations=2)
    mask = cv.dilate(mask, None, iterations=2)
    contours = cv.findContours(mask, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
    contours = imutils.grab_contours(contours)

    # Draw rectangles around balls
    for c in contours:
        # Calculate the area and circularity of the contour
        area = cv.contourArea(c)
        perimeter = cv.arcLength(c, True)
        circularity = 4 * np.pi * area / (perimeter * perimeter)

        # Ignore small or non-circular contours
        if area > 500 and circularity > 0.8:
            # Get the bounding box of the contour
            (x, y, w, h) = cv.boundingRect(c)

            # Calculate the center of the ball
            center_x = x + w // 2
            center_y = y + h // 2

            # Draw a rectangle around the contour
            cv.rectangle(frame, (x, y), (x + w, y + h), color, 2)

            # Display the coordinates of the center
            text = f"({center_x}, {center_y})"
            cv.putText(frame, text, (x, y - 10), cv.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)

    return frame

capture = cv.VideoCapture("balls.mp4")

# Set the color ranges for the ping pong balls
lower_ranges = {
    "yellow": np.array([20, 100, 100]),
    "green": np.array([60, 100, 50]),
    "blue": np.array([100, 100, 50]),
    "red1": np.array([0, 100, 50]),
    "red2": np.array([170, 100, 50])
}

upper_ranges = {
    "yellow": np.array([40, 255, 255]),
    "green": np.array([100, 255, 255]),
    "blue": np.array([130, 255, 255]),
    "red1": np.array([10, 255, 255]),
    "red2": np.array([180, 255, 255])
}

colors = {
    "yellow": (0, 255, 255),
    "green": (0, 255, 0),
    "blue": (255, 0, 0),
    "red1": (0, 0, 255),
    "red2": (0, 0, 255)
}

while True:
    isTrue, frame = capture.read()
    if not isTrue:
        break
    frame_resized = rescaleFrame(frame, 0.5)

    # Detect and draw balls of different colors
    for color in colors:
        frame_resized = detect_and_draw_balls(frame_resized, lower_ranges[color], upper_ranges[color], colors[color])

    # Show the modified frame
    cv.imshow("Frame", frame_resized)

    # Exit if the 'd' key is pressed
    if cv.waitKey(20) & 0xFF == ord('d'):
        break

capture.release()
cv.destroyAllWindows()


import concurrent.futures
import json
import math
import random
import re
import zlib

"""
    Implements parallel, partial HTTP requests for fetching Common Crawl content.
"""
class CommonCrawlDataRetriever(object):
    """
        :param http_concurrency: HTTP concurrent requests.
    """
    def __init__(
        self,
        http_concurrency
    ):
        self.http_concurrency = http_concurrency

        print("COMMON CRAWL DATA: ctor: concurrency: {}".format(self.http_concurrency))

    """
        :param indices: RDD of Common Crawl indices.
        :param http_retriever: the object that we will delegate HTTP retrieval to.
        :param transformer: the object that we will delegate HTTP response transformation to.

        :return: list of lists of Common Crawl content.
    """
    def get(
        self,
        indices,
        http_retriever,
        transformer
    ):
        print("COMMON CRAWL DATA: get: got {} indices to fetch".format(len(indices)))

        results = []

        with concurrent.futures.ThreadPoolExecutor(max_workers = self.http_concurrency) as executor:
            # Start the load operations and mark each future with its URL
            futures = {
                executor.submit(
                    http_retriever.get,
                    "https://commoncrawl.s3.amazonaws.com/" + index["warc_filename"],
                    params = None,
                    headers = {"Range": "bytes={}-{}".format(int(index["warc_record_offset"]), int(index["warc_record_offset"]) + int(index["warc_record_length"]))},
                    # Quasi-exponential random sleep to avoid congesting HTTP requests.
                    # Reason for doing is to avoid HTTP 503s.
                    sleep = int(random.uniform(0, math.log(math.pow(len(indices), 3))))
                ): index for index in indices
            }

            for future in concurrent.futures.as_completed(futures):
                try:
                    # Get the Common Crawl index that this future maps to so as to retrieve
                    # fields "url" and "urlkey".
                    index = futures[future]

                    print("COMMON CRAWL DATA: get: got response for {}".format(index["url_surtkey"]))

                    # Partial GZipped response (binary).
                    response = future.result()

                    # Retrieve GZip content.
                    response_content = zlib.decompress(response.content, zlib.MAX_WBITS | 16)

                    # Process response text.
                    # Strip HTML and extra whitespace characters from response.
                    # It is important to decode text using the encoding identifier provided with the Common Crawl index.
                    response_content = " ".join(re.split(r"\W+", transformer.get(response_content.decode(index["content_charset"]))))

                    response_json = {
                        "urlkey": index["url_surtkey"],
                        "url": index["url"],
                        "content": response_content
                    }

                    # Create list of lists so that we can handle it in a uniform way on the head node.
                    results.append([response_json])

                except Exception as e:
                    print("{}: {}".format("COMMON CRAWL DATA: get: ERROR", e))

        print("COMMON CRAWL DATA: get: got {} total results".format(len(results)))

        return results


import wget
import argparse
import os
import zipfile

def geturl(line):
    return line.strip().split(" ")[-1]

def getname(url):
    return url.split("/")[3] + ".zip"

# extend with actually downloading monolingual opus data - opus get does not do that currently.

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-f", "--file", help="file with ro opus zip locations as given by opus_get list")
    parser.add_argument("-o", "--output",
                        default="./raw/opus",
                        help="output dir where archives will be downloaded")
    args = parser.parse_args()

    if not os.path.exists(args.output):
        os.makedirs(args.output)

    archives = [
        wget.download(geturl(l),
                      out=os.path.join(args.output, getname(geturl(l))))
        for l in open(args.file).readlines()]

    for a in archives:
        with zipfile.ZipFile(a, 'r') as zip_ref:
            zip_ref.extractall(args.output)
        os.remove(a)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from logger import get_logger

logger = get_logger(__name__)

class Type(object):
    def __init__(self, str):
        self.str = str

class Chain(object):
    def __init__(self, name):
        self._path = f'{{ {name}}}'
        self._no_add = False # has add {},so not add
        self._start = self._path.index("}")#next insert point

    def __getattr__(self, path):
        if(self._no_add):
            t = f' {path} '
        else:
            t = f'{{ {path} }}'
        idx = self._path.index("}",self._start)
        self._path = self._path[:idx] + t + self._path[idx:]

        if(self._no_add):
            self._start += len(t)
        else:
            self._path.index("}", self._start)
            #now add {} ,so next insert point is before new {} 's }

        self._no_add = False
        return self

    def _insert_chain(self, chain):
        t = chain.str_without_bracket()
        # as subchain,it does not require the outermost {}
        return t

    def get(self, *args):
        idx = self._path.index("}",self._start)
        if self._no_add:
            t = ""
            for i in args:
                if isinstance(i, Chain):
                    t += self._insert_chain(i)
                    continue
                t += f" {i} "
        else:
            t = " {"
            for i in args:
                if isinstance(i, Chain):
                    t += self._insert_chain(i)
                    continue
                t += f" {i} "
            t += " } "

        head = self._path[:idx]
        tail = self._path[idx:]
        self._path =  head + t + tail
        #print(f"t = {t}")
        #print(f"{head}  < this > {tail}")
        j = self._start
        if(self._no_add):
            self._start += len(t)
        else:
            delta = t.rindex("}")
            self._start += delta
            #self._start = self._path.index("}", self._start)
        head = self._path[:self._start]
        tail = self._path[self._start:]
        #print(f" next {head}  < this > {tail}")

        self._no_add = True
        return self

    def __call__(self, **kargs):
        t = "("
        for i, j in kargs.items():
            if isinstance(j, Type):
                t += f'{i}:{j.str},'
            elif isinstance(j, str):
                t += f'{i}:"{j}",'
            else:
                t += f'{i}:{j},'
        t = t[:-1] + ")"

        idx = self._path.index("}",self._start)
        self._path = self._path[:idx] + t + self._path[idx:]
        self._start += len(t)
        self._no_add = False
        return self

    def on(self, type):
        if(self._no_add):
            t = f" ... on {type} "
        else:
            t = f"{{ ... on {type} }}"

        idx = self._path.index("}",self._start)
        self._path = self._path[:idx] + t + self._path[idx:]
        #self._start += len(t) - 1
        if(self._no_add):
            self._start += len(t)
        else:
            delta = t.rindex("}")
            self._start += delta
            #self._path.index("}", self._start + 1)
        self._no_add = False
        return self

    def __str__(self):
        return(self._path)

    def to_dict(self):
        # print(self._path)
        return {"query":self._path}

    def str(self):
        return self._path

    def str_without_bracket(self):
        t1 = self._path.index("{")
        t2 = self._path.rindex("}",self._start)
        #print(f"no is {self._path[t1 + 1:t2]}")
        return self._path[t1 + 1:t2 ]

def test():
    #c = Chain("search")
    #logger.debug(c(last = 10).get("repositoryCount").get("test")\
          #.get(Chain("star")(last = 4).get("test3")).on("type").attr)
    #chain = Chain("repositoryOwner")\
            #(login = "3rf")\
            #.on("User")\
            #.get(Chain("organizations")\
                 #(first = 10)\
                 #.nodes.get("f"))\
    #.get("first")

    #logger.info(Chain("first")(first = 3).to_dict())
    #logger.info(Chain("first")(first = 3).get("test").nodes.to_dict())
    #logger.info(Chain("first")(first = 3).nodes.get("f").to_dict())
    c = Chain("test")\
        .on("Repository")\
        .get(Chain("... on User")\
             .get("totalCount"))\
        .get(Chain("stargazers")\
             (last = 1).\
             get("totalCount"))
    logger.info(c.to_dict())
    #logger.info(Chain("test").get("test1").str_without_bracket())
    #logger.info(Chain("first")(first = 3).get(Chain("test").get("test1")).nodes.get("f").to_dict())

    #logger.info(Chain("first")(first = 3).nodes.get("f").get("g").to_dict())
    #logger.info(chain.to_dict())

if __name__ == "__main__":
    test()


'''

OOPS - Real-world

    Encapsulation -  Important concept of OOPS which mainly deals
                        with hiding of the data and the methods

    -> Hide the internal working of the object
    -> Expose/Provides the methods to use the object

Laptop -> ram, cpu, os, brand, keyboard
        -> how does the keyboard => q is pressed => it sends a signal to motherboard (0&1) =>
                    displayed on the screen or monitor
        q key is pressed => convert the q => 0&1 => send the signal => application => shortcut
        -> press the key =>

Car -> press the acc. -> Speed goes up (methods exposed)
        Internal working: Wire -> sends more fuel to the engine -> burn the fuel in engine -> speeds up the car
                                    calling few methods
                        Hidden from the object  (Private methods)

In Python:
    public methods => no special declaration

        def <methodName>(<parameters>):
            pass

            => Can be used anywhere

    private method => provide two underscores at the starting of the function name

        def __<methodName>(<parameters>):
            pass

                => Not the child can use it
                => Not allowed outside the class

    protected method => provide one underscore at the starting of the function name
        def _<methodName>(<parameters):
            pass

                => Child class can use it from base class
                => Not outside the class
'''

class Car:

    __gear = 0

    def __init__(self, windows = 4):
        self.__a = 0
        self.max_speed = 0
        self.windows = windows

    def reset(self):

        self.__gear = 0
        self.max_speed = 0

    def print_state(self):
        print(f"Current gear: {self.__gear}, MaxSpeed: {self.max_speed}, Windows: {chevy.windows}")

    def press_acc(self):

        if self.__gear == 0:
            self.max_speed = 0

        if self.__gear > 4:
            self.__gear = 5

        self.__gear = self.__gear + 1
        self.max_speed = self.__gear * 30

        #self.__fuel_burnt = self.__fuel_burnt - (self.gear*2)

    def press_break(self):

        if self.__gear == 0:
            self.max_speed = 0

        if self.__gear < 0:
            self.__gear = 0

        self.__gear = self.__gear - 1
        self.max_speed = self.__gear * 30

    def __internal_gear(self):
        print("Internal gear changes")

chevy = Car()           # Created the object

chevy.press_acc()       #
chevy.print_state()

chevy.print_state()
chevy.print_state()
chevy.print_state()

chevy.press_acc()
chevy.print_state()

#chevy.__internal_gear()

'''
chevy.press_break()
print(f"Current gear: {chevy.gear}, MaxSpeed: {chevy.max_speed}")

chevy.press_acc()
print(f"Current gear: {chevy.gear}, MaxSpeed: {chevy.max_speed}")

chevy.press_break()
print(f"Current gear: {chevy.gear}, MaxSpeed: {chevy.max_speed}")

Design a car class, which has the attributes, speed, gear

    Maths => average of marks
                    sum, length
                    avg = sum(list)/len(list)

    Create a class named Maths, average, mean, meadian

            sum
            avg = sum/len
            perc =  what/total * 100
            median = sum /len

            20% of 100 => 20/100 * 100
            what% of <total>

    Umbrella => Open & Close => button pressed =>

    class Umrella:

        def __init__():
            pass

        lock => hook that holds the wire => frees up the lock =>  wire is expand => certain point with force => open the umbrella
        def press_button():

            self.__open_lock()
            self.__expand_wire()

'''


import pandas as pd
import re
import numpy as np

def cleaning(df):

    # Drop rows where date and year have NaN values
    df=df.dropna(subset=['Date','Year'])

    # Define a pattern from the date column and extract date values to a new column
    pattern_date="\d{2}\-\w{3}\-\d{4}"
    #pattern_month="\w{3}"

    #compile the pattern into regex
    reg_date=re.compile(pattern_date)

    df["Date_clean"]=df.apply(lambda row: str(re.findall(pattern_date,row["Date"]))[2:-2] if reg_date.search(str(row["Date"])) else row["Year"], axis=1)
    #create a month column
    df['Month']=df['Date_clean'].apply(lambda x: x.split('-')[1] if type(x) == str else np.nan)

    # Year as int
    df['Year']=df['Year'].astype(int)

    # remove the unnecessary columns: Case Number, pdf, href formula, href, Case Number.1, Case Number.2, original order, 22 and 23
    df=df.drop(['Case Number','pdf', 'href formula', 'href', 'Case Number.1', 'Case Number.2','original order','Unnamed: 22','Unnamed: 23'],axis=1)

    # remove values where Year and Date_clean are 0
    df=df[(df['Year']!=0)&(df['Date_clean']!=0)]

    # replace Date with Date_clean
    df['Date']=df['Date_clean']

    # Drop Date_clean
    df=df.drop(['Date_clean'],axis=1)

    # replace 0 years with years from the date
    df['Year']=df.apply(lambda row: str(row['Date'])[-4:] if row['Year']==0 else row['Year'], axis=1)

    #Strip the column names of spaces
    df=df.rename(columns=lambda x: x.strip())

    # Change column names - make them shorter
    df=df.rename(columns={'Fatal (Y/N)':'Fatal','Investigator or Source':'Source'})

    # Clean the values from Fatal and Sex
    df['Fatal']=df['Fatal'].str.strip().str.upper()
    df['Sex']=df['Sex'].str.strip().str.upper()
    # Filter out the values where Fatal or Sex is not clear
    df=df[((df['Fatal']=='N') | (df['Fatal']=='Y')) & ((df['Sex']=='M') | (df['Sex']=='F'))]

    # Define a function that will classify shark names
    shark_list=['white shark', 'tiger shark', 'bull shark', 'wobbegong shark', 'blacktip shark', 'blue shark']

    def clean_spec(element,list,remain): # remain is the value that is given if no match is found, 'same' leaves the same value
        sk=[i.capitalize() for i in list if i in str(element).lower()]
        if sk==[]:
            if remain=='same':
                sk=element
            else:
                sk=remain
        else:
            sk=str(sk[0])
        return sk

    #run the species clean function on the relevant column
    df['Species']=df['Species'].apply(clean_spec,args=(shark_list,'Shark'))

    # Run the clean_spec function on the specified activity list
    activities=['kite surfing', 'windsurfing', 'surfing', 'free diving', 'diving', 'snorkeling', 'fishing', 'wading', 'swimming', 'bathing', 'boarding', 'standing', 'kayking', 'surf']
    df['Activity']=df['Activity'].apply(clean_spec, args=(activities,'same'))

    df['Activity'].replace(to_replace='Surf', value='Surfing', inplace=True)
    df['Activity'].replace(to_replace='Wading', value='Fishing', inplace=True)
    df['Activity'].replace(to_replace='Bathing', value='Swimming', inplace=True)
    df['Activity'].replace(to_replace='Boarding', value='Surfing', inplace=True)

    # Clean up the country list
    # Put all the country names in lower letters and capitalize
    # Replace Usa with USA
    df['Country']=df['Country'].str.split('/').str[0].str.strip()
    df['Country']=df['Country'].str.lower().str.capitalize()
    df['Country'].replace(to_replace='Usa', value='USA', inplace=True)

    # Clean up the age column
    def clean_age(age):
        if isinstance(age, int):
            if age>100:
                age=100
        elif re.findall('\d{1,2}',age)!=[]:
            age=int(re.findall('\d{1,2}',age)[0])
        else:
            age=0
        return age

    df['Age']=df['Age'].fillna(0)
    df['Age']=df['Age'].apply(clean_age)

    return df

def year_stats(df2):
    # Create index of years and calculate the fatality rates for each year
    years=[]
    fatality_rate=[]
    fatality_r_male=[]
    fatality_r_female=[]

    for i in range(int(df2['Year'].min()),int(df2['Year'].max())+1):
        years.append(i)
        if len(df2[df2['Year']==i])==0:
            fatality_rate.append(0)
        else:
            fatality_rate.append(len(df2[(df2['Year']==i)&(df2['Fatal']=='Y')])/len(df2[df2['Year']==i])*100)

        if len(df2[(df2['Year']==i)&(df2['Sex']=='M')])==0:
            fatality_r_male.append(0)
        else:
            fatality_r_male.append(len(df2[(df2['Year']==i)&(df2['Sex']=='M')&(df2['Fatal']=='Y')])/len(df2[(df2['Year']==i)&(df2['Sex']=='M')])*100)

        if len(df2[(df2['Year']==i)&(df2['Sex']=='F')])==0:
            fatality_r_female.append(0)
        else:
            fatality_r_female.append(len(df2[(df2['Year']==i)&(df2['Sex']=='F')&(df2['Fatal']=='Y')])/len(df2[(df2['Year']==i)&(df2['Sex']=='F')])*100)

    # build a dictionary which will be used to build a dataframe
    dict=list(zip(years,fatality_rate))
    df_y=pd.DataFrame(dict,columns=['Year','Fatality rate'])
    df_y['Fatality rate male']=fatality_r_male
    df_y['Fatality rate female']=fatality_r_female

    # calculate the 5 year averages
    df_y['Fatality rate 5y_avg']=df_y['Fatality rate'].rolling(5).mean()
    df_y['Fatality rate male 5y_avg']=df_y['Fatality rate male'].rolling(5).mean()
    df_y['Fatality rate female 5y_avg']=df_y['Fatality rate female'].rolling(5).mean()

    return df_y

# create a dataframe for countries
def country_stats(df2, top):
    top_countries=df2['Country'].value_counts()[:top]
    top_df=pd.DataFrame(top_countries,columns=['Country'])
    top_countries=list(top_countries.index)
    top_df.rename(columns={'Country':'Cases'}, inplace=True)

    fatal_y=[]
    fatal_n=[]
    males=[]
    females=[]
    males_fatal=[]
    females_fatal=[]
    for i in top_countries:
        fatal_y.append(df2[(df2['Country']==i)&(df2['Fatal']=='Y')]['Fatal'].count())
        fatal_n.append(df2[(df2['Country']==i)&(df2['Fatal']=='N')]['Fatal'].count())
        males.append(df2[(df2['Country']==i)&(df2['Sex']=='M')]['Fatal'].count())
        females.append(df2[(df2['Country']==i)&(df2['Sex']=='F')]['Fatal'].count())
        males_fatal.append(df2[(df2['Country']==i)&(df2['Sex']=='M')&(df2['Fatal']=='Y')]['Fatal'].count())
        females_fatal.append(df2[(df2['Country']==i)&(df2['Sex']=='F')&(df2['Fatal']=='Y')]['Fatal'].count())

    top_df['Fatal']=fatal_y
    top_df['Not fatal']=fatal_n
    top_df['Males']=males
    top_df['Females']=females
    top_df['Males fatal']=males_fatal
    top_df['Females fatal']=females_fatal

    top_df['Fatality rate']=top_df['Fatal']/top_df['Cases']
    top_df['Fatality rate males']=top_df['Males fatal']/top_df['Males']
    top_df['Fatality rate females']=top_df['Females fatal']/top_df['Females']

    return top_df

# Create activities dataframe

def activity_stats(df2,top):
    top_activities=df2['Activity'].value_counts()[:10]
    top_a_df=pd.DataFrame(top_activities,columns=['Activity'])
    top_a_df.rename(columns={'Activity':'Cases'}, inplace=True)
    top_activities=list(top_activities.index)

    fatal_y=[]
    fatal_n=[]
    males=[]
    females=[]
    males_fatal=[]
    females_fatal=[]
    for i in top_activities:
        fatal_y.append(df2[(df2['Activity']==i)&(df2['Fatal']=='Y')]['Fatal'].count())
        fatal_n.append(df2[(df2['Activity']==i)&(df2['Fatal']=='N')]['Fatal'].count())
        males.append(df2[(df2['Activity']==i)&(df2['Sex']=='M')]['Fatal'].count())
        females.append(df2[(df2['Activity']==i)&(df2['Sex']=='F')]['Fatal'].count())
        males_fatal.append(df2[(df2['Activity']==i)&(df2['Sex']=='M')&(df2['Fatal']=='Y')]['Fatal'].count())
        females_fatal.append(df2[(df2['Activity']==i)&(df2['Sex']=='F')&(df2['Fatal']=='Y')]['Fatal'].count())

    top_a_df['Fatal']=fatal_y
    top_a_df['Not fatal']=fatal_n
    top_a_df['Males']=males
    top_a_df['Females']=females
    top_a_df['Males fatal']=males_fatal
    top_a_df['Females fatal']=females_fatal

    top_a_df['Fatality rate']=top_a_df['Fatal']/top_a_df['Cases']
    top_a_df['Fatality rate males']=top_a_df['Males fatal']/top_a_df['Males']
    top_a_df['Fatality rate females']=top_a_df['Females fatal']/top_a_df['Females']

    return top_a_df

# create a dataframe specific for the USA and based on the unemployment rate data
def usa_database(unrate,df_usa):
    years=unrate['Year'].tolist()

    # into the unrate database calculate the total number of cases, fatal cases, Surfing, Fishing, Swimming, Diving, and calculate the fatality rates

    cases=[]
    fatal_y=[]
    surfing=[]
    fishing=[]
    swimming=[]
    diving=[]
    surfing_fatal=[]
    fishing_fatal=[]
    swimming_fatal=[]
    diving_fatal=[]
    for i in years:
        cases.append(df_usa[(df_usa['Year']==i)]['Fatal'].count())
        fatal_y.append(df_usa[(df_usa['Year']==i)&(df_usa['Fatal']=='Y')]['Fatal'].count())
        surfing.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Surfing')]['Fatal'].count())
        surfing_fatal.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Surfing')&(df_usa['Fatal']=='Y')]['Fatal'].count())
        fishing.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Fishing')]['Fatal'].count())
        fishing_fatal.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Fishing')&(df_usa['Fatal']=='Y')]['Fatal'].count())
        swimming.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Swimming')]['Fatal'].count())
        swimming_fatal.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Swimming')&(df_usa['Fatal']=='Y')]['Fatal'].count())
        diving.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Diving')]['Fatal'].count())
        diving_fatal.append(df_usa[(df_usa['Year']==i)&(df_usa['Activity']=='Diving')&(df_usa['Fatal']=='Y')]['Fatal'].count())

    unrate['Cases']=cases
    unrate['Fatal_y']=fatal_y
    unrate['Surfing']=surfing
    unrate['Surfing fatal']=surfing_fatal
    unrate['Fishing']=fishing
    unrate['Fishing fatal']=fishing_fatal
    unrate['Swimming']=swimming
    unrate['Swimming fatal']=swimming_fatal
    unrate['Diving']=diving
    unrate['Diving fatal']=diving_fatal

    unrate['UNRATE_log']=np.log(unrate['UNRATE'])
    unrate['Cases_log']=np.log(unrate['Cases'])
    unrate['Surfing_log']=np.log(unrate['Surfing'])
    unrate['Fishing_log']=np.log(unrate['Fishing'])
    unrate['Swimming_log']=np.log(unrate['Swimming'])
    unrate['Diving_log']=np.log(unrate['Diving'])

    return unrate


import requests
import json
import os
from bs4 import BeautifulSoup
# from dotenv import load_dotenv
# load_dotenv()
# from linebot.models import TextSendMessage
# from linebot import (
#     LineBotApi, WebhookHandler
# )

line_notify_url = "https://notify-api.line.me/api/notify"
line_notify_token = os.getenv('LINE_NOTIFY_TOKEN')

# class SlackNotification:
#     def __init__(self, context, slack_webhook):
#         self.context = context
#         self.slack_webhook = slack_webhook
#     def push(self):
#         slack_data = {
#             "username": "NotificationBot",
#             "icon_emoji": ":satellite:",
#             "channel" : "#random",
#             "attachments": [
#                 {
#                     "color": "#a633ee",
#                     "fields": [
#                         {
#                             "title": "New Incoming Message :zap:",
#                             "value": self.context,
#                             "short": "false",
#                         }
#                     ]
#                 }
#             ]
#         }
#         headers = {'Content-Type': "application/json"}
#         response = requests.post(self.slack_webhook, data=json.dumps(slack_data), headers=headers)
#         if response.status_code != 200:
#             raise Exception(response.status_code, response.text)

class LineNotification:
    def __init__(self, context, Channel_Access_Token=None, ):
        self.context = context
        # self.line_bot_api = LineBotApi(Channel_Access_Token)
    # def push_line_bot(self, target):
    #     self.line_bot_api.push_message(target, TextSendMessage(text=self.context))

    def push_line_notify(self):
        headers = {
            "Authorization": f"Bearer {line_notify_token}",
            "Content-Type": "application/x-www-form-urlencoded"
        }
        payload = {"message": self.context}
        response = requests.post(line_notify_url, headers=headers, data=payload)

        # response = requests.post(line_notify_url, headers=headers, data=payload)
        print(response)
class WebCrawlerUSA:
    def __init__(self):
        self.rs = requests.session()
        self.urls = [
            ('道瓊指數','https://invest.cnyes.com/index/GI/DJI'), # DJI
            ('S&P 500','https://invest.cnyes.com/index/GI/INX'), # SPX
            ('費城半導體','https://invest.cnyes.com/index/GI/SOX'),# 費城半導體
            ('那斯達克綜合指數','https://invest.cnyes.com/index/GI/IXIC'),# NASDAQ
        ]
        self.result = []

    def fetch(self):
        for url in self.urls:
            res = self.rs.get(url[1], verify=False)
            soup = BeautifulSoup(res.text, 'html.parser')
            info_date = soup.select('._zFXfK')[0].text
            info_date = info_date.split(' ')[0]
            info_price = soup.select('.jsx-2214436525.info-price')[0].text
            info_net = soup.select('.jsx-2214436525.change-net')[0].text
            info_percent = soup.select('.jsx-2214436525.change-percent')[0].text
            if '+' in info_net:
                info = '{}▲  {}▲'.format(info_net, info_percent)
                info = info.replace('+', '')
            else:
                info = '{}▼  {}▼'.format(info_net, info_percent)
                info = info.replace('-', '')
            self.result.append('{}-{}\n{}\n{}'.format(info_date, url[0], info_price, info))

    def push(self, slack_webhook=None, line_token=None):
        result = '\n'+'\n\n'.join(self.result)
        try:
            msg = LineNotification(result)
            msg.push_line_notify()
        except Exception as e:
            print(e)

if __name__ == '__main__':
    crawler = WebCrawlerUSA()
    crawler.fetch()
    crawler.push()

    # crawler.push(os.getenv('SLACK_WEBHOOK'), os.getenv('CHANNEL_ACCESS_TOKEN'))


#!/usr/bin/python3

import os;
import tensorflow as tf;
from models import WGAN;

def save_model():

  wgan = WGAN();
  optimizerG = tf.keras.optimizers.Adam(2e-4);
  optimizerD = tf.keras.optimizers.Adam(2e-4);
  checkpoint = tf.train.Checkpoint(model = wgan, optimizerG = optimizerG, optimizerD = optimizerD);
  checkpoint.restore(tf.train.latest_checkpoint('checkpoints'));
  if False == os.path.exists('models'): os.mkdir('models');
  wgan.G.save(os.path.join('models', 'G.h5'));
  wgan.D.save(os.path.join('models', 'D.h5'));

if __name__ == "__main__":

  assert True == tf.executing_eagerly();
  save_model();


import json

from reformer.modeling import ReformerModel as PDReformerModel
from transformers.models.reformer.modeling_reformer import (
    ReformerModel as PTReformerModel,
)
from transformers import ReformerConfig
import paddle
from collections import OrderedDict

paddle.set_device("cpu")

def load_json(file_path):
    with open(str(file_path), "r", encoding="utf8") as f:
        data = json.load(f)
    return data

def to_paddle(x):
    return paddle.to_tensor(x.numpy())

def generate_random_2_layer():
    pdjson = load_json("weights/random_2_layer/model_config.json")
    pd_model = PDReformerModel(**pdjson)
    c = ReformerConfig.from_json_file("weights/random_2_layer/config.json")
    pt_model = PTReformerModel(c)
    pd_model.eval()
    pt_model.eval()
    d = OrderedDict()
    for k, v in pt_model.state_dict().items():
        if "_mask_value_" in k:
            continue
        if v.ndim == 2 and "embedding" not in k:
            v = v.transpose(1, 0)
        d[k] = to_paddle(v)
    pd_model.set_dict(d)
    pd_model.save_pretrained("weights/random_2_layer")
    pt_model.save_pretrained("weights/random_2_layer")

def generate_random_12_layer():
    pdjson = load_json("weights/random_12_layer/model_config.json")
    pd_model = PDReformerModel(**pdjson)
    c = ReformerConfig.from_json_file("weights/random_12_layer/config.json")
    pt_model = PTReformerModel(c)
    pd_model.eval()
    pt_model.eval()
    d = OrderedDict()
    for k, v in pt_model.state_dict().items():
        if "_mask_value_" in k:
            continue
        if v.ndim == 2 and "embedding" not in k:
            v = v.transpose(1, 0)
        d[k] = to_paddle(v)
    pd_model.set_dict(d)

    pd_model.save_pretrained("weights/random_12_layer")
    pt_model.save_pretrained("weights/random_12_layer")

if __name__ == "__main__":
    generate_random_2_layer()
    generate_random_12_layer()


"""
Created on Friday 12th March 2021

@author: Aditya Asopa, Bhalla Lab, NCBS
"""

import sys
import os
import pathlib
import importlib

from eidynamics             import ephys_classes
from eidynamics             import data_quality_checks
from eidynamics.errors      import *
from eidynamics.plot_maker  import dataframe_to_plots
from eidynamics.utils       import show_experiment_table, reset_and_print, parse_other_experiment_param_file

def parse_cell(cell_directory, load_cell=True, save_pickle=True, add_cell_to_database=False, all_cell_response_db='', export_training_set=True, save_plots=True, user='Adi'):
    cell_directory = pathlib.Path(cell_directory)
    cell_response_file, cell_pickle_file, cellFile = "", "", ""
    cellID = cell_directory.stem
    cell = None
    # _ = show_experiment_table(cell_directory)

    if load_cell:
        cell_pickle_file = cell_directory / str(cellID + ".pkl")
        try:
            print("#___ Loading cell from: ", cell_pickle_file)
            cell = ephys_classes.Neuron.loadCell(cell_pickle_file)
        except:
            print(f"#___ Error in loading cell from {cell_pickle_file}. Creating new cell.")
            cell_pickle_file = ''
            cell = None

    try:
        file_ext = "rec.abf" if user=='Adi' else '.abf'
        rec_files = list(cell_directory.glob('*' + file_ext))
        # print all rec files one by one
        print("Following are the recordings in the directory: ")
        print(rec_files)

        for i, rec_file in enumerate(rec_files):
            msg = "Now parsing: " + rec_file.name
            reset_and_print(i, len(rec_files), clear=False, message=msg)
            cell, cell_pickle_file, cell_response_file = parse_recording(rec_file, cell_file=cell_pickle_file, user=user)
        print(f"Following is the summary of all experiments on the cell {cellID} ")
        _ = cell.summarize_experiments()

        print('##__ Making Dataframe.')
        cell.make_dataframe() # type: ignore

        print('###_ Running cell stability checks')
        data_quality_checks.run_qc(cell, cell_directory)
        '''
        if add_cell_to_database:
            if all_cell_response_db == '':
                # create a new database file
                all_cell_response_db = cell_directory / "all_cells_response.xlsx"

            print("#### Adding cell to database file: ", all_cell_response_db)
            cell.add_cell_to_xl_db(all_cell_response_db) # type: ignore
        '''
        if export_training_set:
            print("#### Saving traces for training for each protocol")
            cell.save_full_dataset(cell_directory) # type: ignore

        if save_pickle:
            print("#### Saving pickle and excel files")
            cellFile            = cell_directory / str(str(cell.cellID) + ".pkl") # type: ignore
            cellFile_csv        = cell_directory / str(str(cell.cellID) + ".xlsx") # type: ignore
            ephys_classes.Neuron.saveCell(cell, cellFile)
            cell.response.to_excel(cellFile_csv) # type: ignore
        else:
            cellFile = ''

    except UnboundLocalError as err:
        print(err)
        print("Check if there are '_rec' labeled .abf files in the directory.")

    if save_plots:
        dataframe_to_plots(cell_pickle_file, ploty="PeakRes",  gridRow="NumSquares",
                                             plotby="EI",        clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="PeakRes",  gridRow="NumSquares",
                                             plotby="PatternID", clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="PeakRes",  gridRow="PatternID",
                                             plotby="Repeat",    clipSpikes=True)

        dataframe_to_plots(cell_pickle_file, ploty="PeakTime", gridRow="NumSquares",
                                             plotby="EI",        clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="PeakTime", gridRow="NumSquares",
                                             plotby="PatternID", clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="PeakTime", gridRow="PatternID",
                                             plotby="Repeat",    clipSpikes=True)

        dataframe_to_plots(cell_pickle_file, ploty="AUC",      gridRow="NumSquares",
                                             plotby="EI",        clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="AUC",      gridRow="NumSquares",
                                             plotby="PatternID", clipSpikes=True)
        dataframe_to_plots(cell_pickle_file, ploty="AUC",      gridRow="PatternID",
                                             plotby="Repeat",    clipSpikes=True)

    os.remove(cell_pickle_file)  # remove temporary pickle and excel file
    os.remove(cell_response_file)
    return cell, cellFile

def parse_recording(recording_file, load_cell=True, cell_file=None, user='Adi'):
    datafile      = pathlib.Path(recording_file)
    exptDir       = datafile.parent
    exptFile      = datafile.name
    fileID        = exptFile[:15]

    parameterFilePath = ''
    paramfileName = ''
    if user=='Adi':
        parameterFilePath = exptDir / str(fileID + "_experiment_parameters.py")
        paramfileName = parameterFilePath.stem
        parameterFilePath = str(parameterFilePath.parent) # str() to convert pathlib path --> plain string
    elif user=='Sulu':
        parameterFilePath = exptDir / str("PPFExpt_protocol_params_March06_23_" + datafile.stem + ".py")

    print("ParameterFilePath: ", parameterFilePath)

    # Import Experiment Variables
    try:
        print("Looking for experiment parameters locally")
        sys.path.append(parameterFilePath)

        if user=='Adi':
            exptParams = importlib.import_module(paramfileName, parameterFilePath)
        elif user=='Sulu':
            exptParams = parse_other_experiment_param_file(parameterFilePath, user=user)

        datafile_according_to_epfile = exptParams.datafile
        if not datafile_according_to_epfile == exptFile:
            raise FileMismatchError()
        print('Experiment parameters loaded from: ', parameterFilePath)

    except (FileMismatchError, FileNotFoundError) as err:
        print(err)
        print("No special instructions, using default variables.")
        import eidynamics.experiment_parameters_default as exptParams
        save_trial  = False
        print('Default Experiment Parameters loaded.\n'
              'Experiment will not be added to the cell pickle file,\n'
              'only excel file of cell response will be created.')
    except Exception as err:
        print(err)
        print("Experiment Parameters error. Quitting!")
        sys.exit()

    # Import stimulation coordinates
    try:
        coordfileName   = exptParams.polygonProtocol
        if not coordfileName:
            raise FileNotFoundError
        coordfile       = pathlib.Path.cwd() / "polygonProtocols" / coordfileName
        print('Local coord file loaded from: ', coordfile)
    except FileNotFoundError:
        print('No coord file found, probably there isn\'t one')
        coordfile       = ''
    except Exception as err:
        print(err)
        coordfile       = ''

    # Load or initialize a Neuron object
    if (cell_file != '') & (load_cell == True):
        try:
            print('Loading local cell data')
            cell = ephys_classes.Neuron.loadCell(cell_file)
        except:
            print('Error in loading from file. Creating new cell.')
            cell = ephys_classes.Neuron(exptParams)
    else:
        print('Creating new cell.')
        cell = ephys_classes.Neuron(exptParams)

    # Add current experiment to the cell object
    print(f'Adding experiment {fileID} to the cell.')
    cell.addExperiment(datafile=datafile, coordfile=coordfile, exptParams=exptParams)

    # Create temporary file to save
    cell_file_pickle     = exptDir / str(str(exptParams.cellID) + "_temp.pkl")
    cell_file_csv        = exptDir / str(str(exptParams.cellID) + "_temp.xlsx")

    # Save the neuron object
    ephys_classes.Neuron.saveCell(cell, cell_file_pickle)
    cell.response.to_excel(cell_file_csv)

    return cell, cell_file_pickle, cell_file_csv

if __name__ == "__main__":
    input_address = pathlib.Path(sys.argv[1])

    user = sys.argv[2]
    print(user, "Input address: ", input_address)
    if input_address.is_dir():
        parse_cell(input_address, load_cell=True, save_pickle=True,
                                    add_cell_to_database=True, export_training_set=True,
                                    save_plots=False, user=user)
    else:
        parse_recording(input_address, load_cell=True, cell_file='', user=user)
else:
    print("Data parsing program imported")


import json
from . import fields
from . import validate

try:
    import urllib2 as request
except ImportError:
    from urllib import request

ENABLED_HUMAN_VALUE_MAP = {
    0 : 'Disabled',
    1 : 'Enabled'
}

class Thermostat(object):
    """
    This class implements the most basic functionality of communicating with
    an actual thermostat.
    """
    MODEL = ''
    # The current API doesn't require this header, but it also doesn't hurt,
    # and it's the right thing to do.
    JSON_HEADER = {'Content-Type' : 'application/json'}

    def __init__(self, host, timeout=4):
        self.host = host
        self.timeout = timeout

    def get(self, relative_url):
        """
        :param relative_url:    The relative URL from the root of the website.

        :returns:   file-like object as returned by urllib[2,.request].urlopen
        """
        url = self._construct_url(relative_url)
        return request.urlopen(url, timeout=self.timeout)

    def post(self, relative_url, value):
        """
        :param relative_url:    The relative URL from the root of the website.
        :param value:           Value to set this attribute to

        :returns:   file-like object as returned by urllib[2,.request].urlopen
        """
        url = self._construct_url(relative_url)
        request_instance = request.Request(url, value, self.JSON_HEADER)
        return request.urlopen(request_instance, timeout=self.timeout)

    def _construct_url(self, relative_url):
        """
        :param relative_url:    The relative URL from the root of the website

        :returns:   Full URL, for example 'http://192.168.0.2/tstat'
        """
        return 'http://%s/%s' % (self.host, relative_url.lstrip('/'))

class CommonThermostat(Thermostat):
    """
    This class implements the common API features that are available and work
    across all models of thermostat.
    """
    def reboot(self):
        """reboots the thermostat"""
        response = self.post('/sys/command', json.dumps({'command' : 'reboot'}).encode('utf-8'))
        validate.validate_response(response)

    ### tstat subsystem ###
    tstat = fields.ReadOnlyField('/tstat', None, validate_response=validate.validate_tstat_response)
    model = fields.ReadOnlyField('/tstat/model', 'model')
    version = fields.Field('/tstat/version', 'version')
    temp = fields.ReadOnlyField('/tstat', 'temp')
    tmode = fields.Field('/tstat', 'tmode',
        human_value_map={
            0 : 'Off',
            1 : 'Heat',
            2 : 'Cool',
            3 : 'Auto'
    })
    fmode = fields.Field('/tstat', 'fmode',
        human_value_map={
            0 : 'Auto',
            1 : 'Auto/Circulate',
            2 : 'On'
    })
    override = fields.ReadOnlyField('/tstat', 'override',
        human_value_map=ENABLED_HUMAN_VALUE_MAP)
    hold = fields.Field('/tstat', 'hold', human_value_map=ENABLED_HUMAN_VALUE_MAP)
    led = fields.Field('/tstat/led', 'energy_led')
    t_heat = fields.Field('/tstat/ttemp', 't_heat', post_url='/tstat')
    t_cool = fields.Field('/tstat/ttemp', 't_cool', post_url='/tstat')
    it_heat = fields.Field('/tstat/ttemp', 't_heat', post_url='/tstat', post_name='it_heat')
    it_cool = fields.Field('/tstat/ttemp', 't_cool', post_url='/tstat', post_name='it_cool')

    tstate = fields.ReadOnlyField('/tstat', 'tstate',
        human_value_map={
            0 : 'Off',
            1 : 'Heat',
            2 : 'Cool'
    })
    fstate = fields.ReadOnlyField('/tstat', 'fstate',
        human_value_map={
            0 : 'Off',
            1 : 'On'
    })
    time = fields.Field('/tstat', 'time')
    pump = fields.ReadOnlyField('/tstat/hvac_settings', 'pump',
        human_value_map={
            1 : 'Normal',
            2 : 'Heat Pump'
    })
    aux_type = fields.ReadOnlyField('/tstat/hvac_settings', 'aux_type',
        human_value_map={
            1 : 'Gas',
            2 : 'Electric'
    })

    # LED status values: 1 = green, 2 = yellow, 4 = red
    energy_led = fields.WriteOnlyField('/tstat/led', 'energy_led', None)

    # This isn't documented. It might be postable, but I'm not going to try.
    power = fields.ReadOnlyField('/tstat/power', 'power')

    program_cool = fields.ReadOnlyField('/tstat/program/cool', None)
    program_heat = fields.ReadOnlyField('/tstat/program/heat', None)
    datalog = fields.ReadOnlyField('/tstat/datalog', None)

    # Remote temperature control; posting to rem_temp sets rem_mode to 1
    rem_mode = fields.Field('/tstat/remote_temp', 'rem_mode',
        human_value_map=ENABLED_HUMAN_VALUE_MAP)
    rem_temp = fields.WriteOnlyField('/tstat/remote_temp', 'rem_temp', None)

    ### sys subsystem ###
    sys = fields.ReadOnlyField('/sys', None)
    name = fields.Field('/sys/name', 'name')
    services = fields.ReadOnlyField('/sys/services', None)
    mode = fields.Field('/sys/mode', 'mode', human_value_map={
        0 : 'Provisioning',
        1: 'Normal'
    })
    network = fields.ReadOnlyField('/sys/network', None)
    security = fields.ReadOnlyField('/sys/network', 'security',
        human_value_map = {
            1 : 'WEP',
            3 : 'WPA',
            4 : 'WPA2 Personal'
    })

    ### cloud subsystem ###
    cloud = fields.ReadOnlyField('/cloud', None)

    ### methods ###
    def set_day_program(self, heat_cool, day, program):
        """
        Sets the program for a particular day. See the API docs for details,
        as it is a bit complicated.

        :param heat_cool:   Ether the string 'heat' or 'cool'
        :param day:         One of 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'
        :param program:     See thermostat API docs
        :type program:      dict
        """
        self.post('/tstat/program/%s/%s' % (heat_cool, day), json.dumps(program).encode('utf-8'))

class CT30(CommonThermostat):
    """
    Base model for CT30-based thermostats (including the 3M-50)
    """

    MODEL = 'CT30'
    hvac_code = fields.ReadOnlyField('/tstat/hvac_settings', 'hvac_code',
        human_value_map={
            1 : '1 stage heat, 1 stage cool',
            2 : '2 stage heat, 1 stage cool',
            3 : '2 stage heat, 2 stage cool',
            4 : '2 stage heat, 1 stage cool',
            5 : '2 stage heat, 2 stage cool',
            10 : '1 stage pump, 1 stage aux',
            11 : '1 stage pump, 1 stage aux',
            12 : '1 stage pump, no aux',
    })

class CT80(CommonThermostat):
    """
    Base model for CT80-based thermostats
    """

    MODEL = 'CT80'
    ### Program Mode (extended tstat subsystem)
    program_mode = fields.Field('/tstat', 'program_mode',
        human_value_map={
            0 : 'Program A',
            1 : 'Program B',
            2 : 'Vacation',
            3 : 'Holiday'
        })

    # These three stages attributes take place of the CT30's hvac_code
    heat_stages = fields.ReadOnlyField('/tstat/hvac_settings', 'heat_stages')
    cool_stages = fields.ReadOnlyField('/tstat/hvac_settings', 'cool_stages')
    aux_stages = fields.ReadOnlyField('/tstat/hvac_settings', 'aux_stages')

    ### (De)humidifier system ###
    humidity = fields.ReadOnlyField('/tstat/humidity', 'humidity')

    humidifier_mode = fields.Field('/tstat/humidifier', 'humidifier_mode',
        human_value_map = {
            0: 'Off',
            1: 'Run only with heat',
            2: 'Run any time (runs fan)',
        })

    humidifier_setpoint = fields.Field('/tstat/thumidity', 'thumidity')

class CT80RevB(CT80):
    """
    Base model for all Revision B versions of the CT80
    """
    MODEL = 'CT80 RevB'
    swing = fields.Field('/tstat/tswing', 'tswing')

    # Dehumidifier attributes
    dehumidifier_mode = fields.Field('/tstat/dehumidifier', 'mode',
        human_value_map = {
            0: 'Off',
            1: 'On with fan',
            2: 'On without fan',
    })
    dehumidifier_setpoint = fields.Field('/tstat/dehumidifier', 'setpoint')

    # External dehumidifier
    external_dehumidifier_mode = fields.Field('/tstat/ext_dehumidifier', 'mode',
        human_value_map = {
            0: 'Off',
            1: 'On with fan',
            2: 'On without fan',
    })
    external_dehumidifier_setpoint = fields.Field('/tstat/ext_dehumidifier', 'setpoint')

    # Note: the night light is tricky and will return the last-set intensity even if it's off!
    night_light = fields.Field('/tstat/night_light', 'intensity',
        human_value_map = {
            0: 'Off',
            1: '25%',
            2: '50%',
            3: '75%',
            4: '100%',
        })

    # Note: lock_mode 3 can only be changed remotely
    lock_mode = fields.Field('/tstat/lock', 'lock_mode',
        human_value_map={
            0 : 'Lock disabled',
            1 : 'Partial lock',
            2 : 'Full lock',
            3 : 'Utility lock'
    })

    simple_mode = fields.Field('/tstat/simple_mode', 'simple_mode',
        human_value_map={
            1 : 'Normal mode',
            2 : 'Simple mode'
    })

# Specific model classes
class CT50(CT30):
    MODEL = 'CT50'

class CT30v175(CT30):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT30 V1.75'

class CT30v192(CT30):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT30 V1.92'

class CT30v194(CT30):
    """
    Defines API features that differ for this specific model from
    CommonThermostat4
    """
    MODEL = 'CT30 V1.94'

class CT30v199(CT30):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT30 V1.99'

class CT50v109(CT50):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT50 V1.09'

class CT50v188(CT50):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT50 V1.88'

class CT50v192(CT30):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT50 V1.92'

class CT50v194(CT50):
    """
    Defines API features that differ for this specific model from
    CommonThermostat
    """
    MODEL = 'CT50 V1.94'

class CT80RevB1v100(CT80RevB):
    MODEL = 'CT80 Rev B1 V1.00'

class CT80RevB2v100(CT80RevB):
    MODEL = 'CT80 Rev B2 V1.00'

class CT80RevB2v103(CT80RevB):
    MODEL = 'CT80 Rev B2 V1.03'

class CT80RevB2v109(CT80RevB):
    MODEL = 'CT80 Rev B2 V1.09'


import requests
import logging

from bs4 import BeautifulSoup

from torrer.constants import TIMEOUT_AMT

logger = logging.getLogger(__name__)

def search(pb_url: str, query: str) -> str:
    """
    Searches pirates bay for the top result and looks for the top result

    Args:
        pb_url (str): pirates bay url
            The pirates bay url to use as there are many and the main one I use is down
        query (str): The search query for the pirates bay

    Returns:
        str: The link to the top result of the search
    """
    search_url: str = f"{pb_url}/search/{query}"
    logger.info(f"Search url is {search_url}")

    r: requests.Response = requests.get(search_url, timeout=TIMEOUT_AMT)
    r.raise_for_status()

    logger.info(f"Parsing {search_url} for top result")
    html: BeautifulSoup = BeautifulSoup(r.text, features="lxml")
    top_result_element = html.find_all("a", class_="detLink")[
        0
    ]  # 0 to grab the first index
    top_result_link: str = top_result_element["href"]

    logger.info(f"Top result is {top_result_link}")
    return top_result_link

def parse_search_result(torrent_page: str) -> str:
    """
    Gets the magnet link from an individual torrent page

    Args:
        torrent_page (str): The individual torrent page to download
            the magnet link from. Is an url

    Returns:
        str: The magnet link
    """
    r: requests.Response = requests.get(torrent_page, timeout=TIMEOUT_AMT)
    r.raise_for_status()

    html: BeautifulSoup = BeautifulSoup(r.text, features="lxml")

    logger.info("Finding the magnet link")
    download_class = html.find("div", {"class": "download"})
    a_divs = download_class.find_all("a")[0]
    magnet_link = a_divs["href"]

    logger.info(f"Magnet link: {magnet_link}")
    return magnet_link


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse

import json
import numpy as np

import torch
import time
import os
from six.moves import cPickle
import opts
import misc.utils as utils
from misc import eval_utils
from misc.dataloader import *
from misc.dataloader.dataloader import *
from misc.dataloader.dataloader_pool import *
from models import *

NUM_THREADS = 2  # int(os.environ['OMP_NUM_THREADS'])

# Input arguments and options
parser = argparse.ArgumentParser()
parser.add_argument('--model_name', type=str, default='debug',help='')
parser.add_argument('--caption_model', type=str, default='debug',help='')
parser.add_argument('--start_from', type=str, default='save/09021100.lm_stack_cap_2finestages', help='None')
parser.add_argument('--data_type', type=int, default=1)  # 'use pool to load pre-extracted features from dir'
# Model settings
parser.add_argument('--exchange_vocab', type=int, default=0)
# Input paths
parser.add_argument('--disp_coarse_to_fine', type=int, default=0)
parser.add_argument('--dump_output', type=int, default=0)
parser.add_argument('--shuffle', type=int, default=0)#load shuffled data from h5 or dir
parser.add_argument('--pool_loader', type=int, default=0)#use pool to load pre-extracted features from dir
parser.add_argument('--pre_ft', type=int, default=1)#pre extracted feature input
# Basic options
parser.add_argument('--batch_size', type=int, default=50)#if > 0 then overrule, otherwise load from checkpoint.
parser.add_argument('--val_images_use', type=int, default=5000)#how many images to use when periodically evaluating the loss? (-1 = all)
parser.add_argument('--language_eval', type=int, default=0)
parser.add_argument('--dump_images', type=int, default=0) #Dump images into vis/imgs folder for vis? (1=yes,0=no)
parser.add_argument('--dump_json', type=int, default=0) #Dump json with predictions into vis folder? (1=yes,0=no)
parser.add_argument('--dump_path', type=int, default=0) #Write image paths along with predictions into vis json? (1=yes,0=no)
# Sampling options
parser.add_argument('--sample_max', type=int, default=1) #1 = sample argmax words. 0 = sample from distributions.
parser.add_argument('--beam_size', type=int, default=5)
parser.add_argument('--temperature', type=float, default=1.0)
# For evaluation on a folder of images:
parser.add_argument('--server', type=int, default=0)#if running on MSCOCO challenge
parser.add_argument('--server_best', type=int, default=1)#if running on MSCOCO challenge
parser.add_argument('--split', type=str, default='test')#if running on MSCOCO images, which split to use: val|test|train
parser.add_argument('--coco_json', type=str, default='data/mscoco/annotations/image_info_test2014.json')#if nonempty then use this file in DataLoaderRaw (see docs there)
parser.add_argument('--image_folder', type=str, default='data/mscoco/test2014')#If this is nonempty then will predict on the images in this folder path
parser.add_argument('--image_root', type=str, default='') #In case the image paths have to be preprended with a root path to an image folder
# For evaluation on a folder of h5 file:
parser.add_argument('--input_json', type=str, default='data/mscoco/cocotalk_karpathy_0816.json')
parser.add_argument('--input_im_h5', type=str, default='data/mscoco/cocotalk_karpathy_images.h5')
parser.add_argument('--input_fc_h5', type=str, default='data/mscoco/cocotalk_karpathy_fc_0816.h5')
parser.add_argument('--input_att_h5', type=str, default='data/mscoco/cocotalk_karpathy_att_0816.h5')
parser.add_argument('--input_label_h5', type=str, default='data/mscoco/cocotalk_karpathy_label_0816.h5')
parser.add_argument('--input_attrs_h5', type=str, default='data/mscoco/cocotalk_karpathy_label_semantic_words_mil_det.h5')
parser.add_argument('--input_fc_dir', type=str, default='data/mscoco/cocotalk_karpathy_resnet101_fc')
parser.add_argument('--input_att_dir', type=str, default='data/mscoco/cocotalk_karpathy_resnet101_att')

# misc
parser.add_argument('--model_id', type=str, default='')#an id identifying this run/job. used in cross-val and appended when writing progress files
parser.add_argument('--id', type=str, default='evalscript')#an id identifying this run/job. used only if language_eval = 1 for appending to intermediate files

args = parser.parse_args()
last_name = os.path.basename(args.start_from)
last_time = last_name[0:8]

# Load infos
def load_infos(args):
    model_name = os.path.basename(args.start_from)
    with open(os.path.join(args.start_from, 'infos.pkl')) as f:
        infos = cPickle.load(f)
    # override and collect parameters
    opt = infos['opt']

    for k in vars(args).keys():
        if k in vars(opt):
            vars(opt).update({k: vars(args)[k]})  # copy over options from model
        else:
            vars(opt)[k] = vars(args)[k]
    if opt.data_type ==1:
        opt.input_json = 'data/mscoco/cocotalk_karpathy_0816.json'
        opt.input_label_h5 = 'data/mscoco/cocotalk_karpathy_label_0816.h5'

    if opt.server:
        opt.dump_path = 0
        opt.dump_json = 1
        opt.disp_coarse_to_fine = 0
        opt.dump_output = 0
        opt.batch_size = 10
        if opt.split=='test':
            opt.coco_json = 'data/mscoco/annotations/image_info_test2014.json'
            opt.image_folder = 'data/mscoco/test2014'
            opt.val_images_use = 40775
        else:
            opt.coco_json = 'data/mscoco/annotations/captions_val2014.json'
            opt.image_folder = 'data/mscoco/val2014'
            opt.val_images_use = 40504
    else:
        opt.image_folder = ''

    model, cnn_model, crit = load_models(opt, infos)

    return cnn_model, model, crit, opt, infos

def eval(cnn_model, model, crit, opt, infos, args):
    visualization = False
    if visualization:
        eval_utils.visualize_embeddings(20, model.embed, infos['vocab'])

    # Create the Data Loader instance
    if len(opt.image_folder) == 0:
        loader = DataLoader_pool(opt) if args.pool_loader else DataLoader(opt)
        loader.ix_to_word = infos['vocab']
        loss, split_predictions, lang_stats = eval_utils.eval_split(opt, infos, cnn_model, model, crit, loader, vars(opt))
        print('loss: ', loss)
        if lang_stats:
            print(lang_stats)
    else:
        from misc.dataloader.dataloaderraw import *
        loader = DataLoaderRaw(opt)
        loader.ix_to_word = infos['vocab']
        split_predictions = eval_utils.eval_split(opt, infos, cnn_model, model, crit, loader, vars(opt))

    dump_json = getattr(args, 'dump_json', 0)
    if dump_json == 1:
        # dump the json
        if opt.server:
            json.dump(split_predictions, open('captions_' + opt.split + '2014_' + last_time + '_results.json', 'w'))
        else:
            json.dump(split_predictions, open('vis/vis.json', 'w'))

if __name__ == "__main__":
    cnn_model, model, crit, opt, infos = load_infos(args)
    eval(cnn_model, model, crit, opt, infos, args)


class Solution:
    def topKFrequent(self, nums: List[int], k: int) -> List[int]:
        from collections import Counter
        c = Counter(nums)

        cnt = 0
        ans = []
        for cm in c.most_common():
            ans.append(cm[0])
            cnt += 1
            if cnt == k:
                break

        return ans


import sys
from typing import Iterator
sys.path.append('src/udf/python')  # noqa

from risingwave.udf import udf, udtf, UdfServer

@udf(input_types=[], result_type='INT')
def int_42() -> int:
    return 42

@udf(input_types=['INT', 'INT'], result_type='INT')
def gcd(x: int, y: int) -> int:
    while y != 0:
        (x, y) = (y, x % y)
    return x

@udf(input_types=['INT', 'INT', 'INT'], result_type='INT')
def gcd3(x: int, y: int, z: int) -> int:
    return gcd(gcd(x, y), z)

@udtf(input_types='INT', result_types='INT')
def series(n: int) -> Iterator[int]:
    for i in range(n):
        yield i

@udtf(input_types=['INT'], result_types=['INT', 'VARCHAR'])
def series2(n: int) -> Iterator[tuple[int, str]]:
    for i in range(n):
        yield i, str(i)

if __name__ == '__main__':
    server = UdfServer()
    server.add_function(int_42)
    server.add_function(gcd)
    server.add_function(gcd3)
    server.add_function(series)
    server.add_function(series2)
    server.serve()


# Дан список чисел. Определите, сколько в нем встречается различных чисел.
# Input: [1, 1, 2, 0, -1, 3, 4, 4]
# Output: 6

# Вариант_1

my_list = [1, 1, 2, 0, -1, 3, 4, 4]

count = 0
for i in range(len(my_list)):
    if (my_list[i]) not in my_list[:i]:
        count += 1
print(count)

# Вариант_2

my_list = [1, 1, 2, 0, -1, 3, 4, 4]
print(len(set(my_list)))

# Вариант_3

my_list = [1, 1, 2, 0, -1, 3, 4, 4]
your_list = []

for item in my_list:

    if item not in your_list:
        your_list.append(item)
print(f'Различных чисел в списке: {len(your_list)}')

# Вариант_4

# my_list = [1, 1, 2, 0, -1, 3, 4, 4]
# already_seen = []

# for i in range(len(my_list)):
#     for j in range(len(already_seen)):
#         if my_list[i] == already_seen[j]:
#             break
#         else:
#             already_seen.append(my_list[i])
# print(len(already_seen))

#                                                              Срезы
# arr = [1, 2, 3, 4, 5]
# arr[1:3] -> [2, 3]
# arr[:3] -> [1, 2, 3]
# arr[2:] -> [3, 4, 5]
# arr[:] -> [1, 2, 3, 4, 5]
# arr[::2] -> [1, 3, 5]

# Дана последовательность из N целых чисел и число K. Необходимо сдвинуть всю последовательность (сдвиг - циклический) на K элементов вправо, K – положительное число.
# Input: [1, 2, 3, 4, 5] k = 3
# Output: [3, 4, 5, 1, 2]

# Вариант_1

my_list = [1, 2, 3, 4, 5]
q = 7
k = q % len(my_list)
print(my_list[k + 1:] + my_list[:k + 1])

# Вариант_2

# my_list = [1, 2, 3, 4, 5]
# your_list = []
# q = 3
# j = 0
# for i in range(len(my_list)):
#     if i + k <= len(my_list):
#         your_list.append(my_list[i + q - 1])
#     else:
#         your_list.append(my_list[j])
# print(your_list)

# Вариант_3

my_list = [1, 2, 3, 4, 5]
q = 7
k = q % len(my_list)
for i in range(k):
    temp = my_list.pop()
    my_list.insert(0, temp)
print(my_list)

#                                           Словари

my_dict = {
    142342: 'Стол',
    534534: 'Диван',
    149342: {
        'name': 'Кресло',
        'count': 10,
        'color': 'Зеленое',
        'info': [1, 4, 2]
    }
}

for key, val in my_dict.items():
    print(key, '---', val)

# Напишите программу для печати всех уникальных значений в словаре.
# Input: [{"V": "S001"}, {"V": "S002"}, {"VI": "S001"}, {"VI": "S005"}, {"VII": "S005"}, {"V":"S009"}, {"VIII":"S007"}]
# Output: {'S005', 'S002', 'S007', 'S001', 'S009'}

list_items = [{"V": "S001"}, {"V": "S002"}, {"VI": "S001"}, {
    "VI": "S005"}, {"VII": "S005"}, {"V": "S009"}, {"VIII": "S007"}]

new_list = []
for i in range(len(list_items)):
    for value in list_items[i].values():
        if value not in new_list:
            new_list.append(value)
print(new_list)

# Дан массив, состоящий из целых чисел. Напишите программу, которая подсчитает количество элементов массива, больших предыдущего (элемента с предыдущим номером)
# Input: [0, -1, 5, 2, 3]
# Output: 2
# Пояснение: (-1 < 5, 2 < 3)

list_1 = [0, -1, 5, 2, 3]
count = 0
for i in range(1, len(list_1)):
    if list_1[i] > list_1[i-1]:
        count += 1
print(count)


import sys
import pygame

# Keyboard events
def check_keydown_events(event, settings, screen, ship,
    bullets):
    if event.key == pygame.K_q:
        sys.exit()

    elif event.key == pygame.K_UP:
        ship.moving_up = True

    elif event.key == pygame.K_DOWN:
        ship.moving_down = True

    elif event.key == pygame.K_SPACE:
        for bullet in bullets:
            if bullet.need_reload == False:
                bullet.shooting_bullet = True

    elif event.key == pygame.K_r:
        bullets.empty()

def check_keyup_events(event, settings, screen, ship):
    if event.key == pygame.K_UP:
        ship.moving_up = False
    elif event.key == pygame.K_DOWN:
        ship.moving_down = False

def check_events(settings, screen, ship, bullets, stats, play_button):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_events(event, settings, screen, ship,
                bullets)
        elif event.type == pygame.KEYUP:
            check_keyup_events(event, settings, screen, ship)
        elif event.type == pygame.MOUSEBUTTONDOWN:
            mouse_x, mouse_y = pygame.mouse.get_pos()
            check_play_button(settings, screen, stats, play_button, ship,
                bullets, mouse_x, mouse_y)

def check_bullet_target_collisions(settings, screen, bullets,
        target, stats):
    if pygame.sprite.spritecollideany(target, bullets):
        stats.target_hit = True
        stats.reset_tries()
        bullets.empty()

def start_game(settings, screen, stats, ship, bullets):
    """Start the game"""
    pygame.mouse.set_visible(False)
    stats.reset_tries()
    stats.game_running = True

def check_play_button(settings, screen, stats, play_button, ship, bullets, mouse_x,
     mouse_y):
     """ Start a new game when the player clicks play."""
     button_clicked = play_button.rect.collidepoint(mouse_x, mouse_y)
     if button_clicked and not stats.game_running:
         start_game(settings, screen, stats, ship, bullets)


from database import db
from process.globals import get_config

def get_impressions(campaign=None, banner=None, **ignore):
    config = get_config()
    query = db.Query()
    query.columns.append("SUM(count) AS count")
    query.tables.append("{impressions_db}bannerimpressions".format(impressions_db=config.impressions_prefix))
    if campaign:
        query.where.append("campaign = %(campaign)s")
        query.params['campaign'] = campaign
    if banner:
        query.where.append("banner = %(banner)s")
        query.params['banner'] = banner

    result = list(db.get_db().execute(query))
    row = result.pop()

    return row['count']


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

class BinaryClassifier(nn.Module):
    def __init__(self):
        super(BinaryClassifier, self).__init__()
        self.block1 = self.conv_block(c_in=3, c_out=256, dropout=0.1, kernel_size=5, stride=1, padding=2)
        self.block2 = self.conv_block(c_in=256, c_out=128, dropout=0.1, kernel_size=3, stride=1, padding=1)
        self.block3 = self.conv_block(c_in=128, c_out=64, dropout=0.1, kernel_size=3, stride=1, padding=1)
        self.lastcnn = nn.Conv2d(in_channels=64, out_channels=2, kernel_size=56, stride=1, padding=0)
        self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2)

    def forward(self, x):
        x = self.block1(x)
        x = self.maxpool(x)
        x = self.block2(x)
        x = self.block3(x)
        x = self.maxpool(x)
        x = self.lastcnn(x)
        return x

    def conv_block(self, c_in, c_out, dropout,  **kwargs):
        seq_block = nn.Sequential(
            nn.Conv2d(in_channels=c_in, out_channels=c_out, **kwargs),
            nn.BatchNorm2d(num_features=c_out),
            nn.ReLU(),
            nn.Dropout2d(p=dropout)
        )
        return seq_block

# VGG
class Vgg19(torch.nn.Module):
    def __init__(self, num_classes):
        super(Vgg19, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1),
            nn.BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 64, 3, padding=1),
            nn.BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True)
        )

        self.conv2 = nn.Sequential(
            nn.MaxPool2d(2, stride=2, ceil_mode=True),  # 1/2
            nn.Conv2d(64, 128, 3, padding=1),
            nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, 3, padding=1),
            nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True)
        )

        self.conv3 = nn.Sequential(
            nn.MaxPool2d(2, stride=2, ceil_mode=True),  # 1/4
            nn.Conv2d(128, 256, 3, padding=1),
            nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, 3, padding=1),
            nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, 3, padding=1),
            nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, 3, padding=1),
            nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
        )

        self.conv4 = nn.Sequential(
            nn.MaxPool2d(2, stride=2, ceil_mode=True),  # 1/8
            nn.Conv2d(256, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True)
        )

        self.conv5 = nn.Sequential(
            nn.MaxPool2d(2, stride=2, ceil_mode=True),  # 1/16
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, padding=1),
            nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, stride=2, ceil_mode=False)
        )

        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))

        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout()
        )

        self.last_fc = nn.Sequential(nn.Linear(4096, num_classes))

        self._initialize_weights()

    def _initialize_weights(self):
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                    if m.bias is not None:
                        nn.init.constant_(m.bias, 0)
                elif isinstance(m, nn.BatchNorm2d):
                    nn.init.constant_(m.weight, 1)
                    nn.init.constant_(m.bias, 0)
                elif isinstance(m, nn.Linear):
                    nn.init.normal_(m.weight, 0, 0.01)
                    nn.init.constant_(m.bias, 0)

    def forward(self, x):
        h = x
        h = self.conv1(h)
        h = self.conv2(h)
        h = self.conv3(h)
        h = self.conv4(h)
        h = self.conv5(h)

        pool = self.avgpool(h)
        flatten = torch.flatten(pool, 1)
        classifier = self.classifier(flatten)
        classes = self.last_fc(classifier)

        return classes


from os import name
from scrollable_frame import ScrollableFrame
# # importing regex and random libraries
import re
import random
import tkinter as tk

class ChatWindow(tk.Toplevel):
    # potential negative responses
    negative_responses = ("no", "nope", "nah", "naw", "not a chance", "sorry")
    # keywords for exiting the conversation
    exit_commands = ("quit", "pause", "exit", "goodbye", "bye", "later")
    # random starter questions
    random_questions = (
        "Why are you here? ",
        "Are there many humans like you? ",
        "What do you consume for sustenance? ",
        "Is there intelligent life on this planet? ",
        "Does Earth have a leader? ",
        "What planets have you visited? ",
        "What technology do you have on this planet? "
    )

    def __init__(self, master):
        super().__init__(master)

        self.protocol('WM_DELETE_WINDOW', master.destroy)

        self.alienbabble = {'describe_planet_intent': r'.*\s*your planet.*',
                            'answer_why_intent': r'why\sare.*',
                            'cubed_intent': r'.*cube.*(\d+)'
                            }

        self.resizable(False, False)
        self.geometry('300x400')
        self.transient(master)

        self.rowconfigure(0, weight=1)

        self.convo = ScrollableFrame(self, width=400)
        self.convo.grid(row=0, column=0, sticky='nsew')

        self.bar = tk.Frame(self)

        self.btnpressed = tk.BooleanVar(self, name='btn')
        self.msg = tk.Entry(self.bar, width=24, font=(None, 12))

        self.btn = tk.Button(self.bar, text="SEND",
                             width=4, command=self.send_msg)

        self.msg.pack(side=tk.LEFT)
        self.btn.pack(side=tk.LEFT)
        self.bar.grid(row=1, column=0, sticky='ew')

        self.greeted = False
        self.greet()

    def send_msg(self):
        message = self.msg.get()
        tk.Message(self.convo.window, text=message, bg="#eee",
                   width=256).grid(sticky='e', pady=5)
        self.btnpressed.set(True)
        self.chat(message)

    def greet(self):
        tk.Message(self.convo.window, width=256,
                   text="Hello there, what's your name?", bg="#eee").grid(sticky='w')

        self.wait_variable('btn')

        will_help = "Hi {}, I'm Etcetera. I'm not from this planet. Will you help me learn about your planet? ".format(
            self.msg.get())

        tk.Message(self.convo.window, width=256,
                   text=will_help, bg="#eee").grid(sticky='w')

        self.wait_variable('btn')

        if self.msg.get() in ChatWindow.negative_responses:
            tk.Message(self.convo.window, width=256,
                       text="Okay, have a nice earth day!", bg="#eee").grid(sticky='w')
            self.after(1000, self.master.destroy)
            return

        tk.Message(self.convo.window, width=256,
                   text=random.choice(self.random_questions), bg="#eee").grid(sticky='w')
        self.greeted = True

    def make_exit(self, reply):
        if reply in ChatWindow.exit_commands:
            tk.Message(self.convo.window, width=256,
                       text="Okay, have a nice earth day!", bg="#eee").grid(sticky='w')
            self.after(10, self.master.destroy)

        return False

    def chat(self, message):
        if not self.greeted:
            return
        reply = message

        if not self.make_exit(reply):
            tk.Message(self.convo.window, width=256,
                       text=self.match_reply(reply), bg="#eee").grid(sticky='w')

    def match_reply(self, reply):
        for key, value in self.alienbabble.items():
            intent = key
            regex_pattern = value
            found_match = re.match(regex_pattern, reply)
            if found_match and intent == "describe_planet_intent":
                return self.describe_planet_intent()
            elif found_match and intent == "answer_why_intent":
                return self.answer_why_intent()
            elif found_match and intent == "cubed_intent":
                return self.cubed_intent(found_match.groups()[0])
            else:
                return self.no_match_intent()

    def describe_planet_intent(self):
        responses = ("My planet is a utopia of diverse organisms and species.",
                     "I am from Opidipus, the capital of the Wayward Galaxies. ")
        return random.choice(responses)
    # Define .answer_why_intent():

    def answer_why_intent(self):
        responses = ("I come in peace", "I am here to collect data on your planet and its inhabitants",
                     "I heard the coffee is good", "Just checking, shut up!")
        return random.choice(responses)

    # Define .cubed_intent():
    def cubed_intent(self, number):
        return "Cubed Number is {}, is there anything else I can help you with?".format(int(number)**3)

    # Define .no_match_intent():
    def no_match_intent(self):
        responses = ("Please tell me more.", "Tell me more!", "Why do you say that?", "I see. Can you elaborate",
                     "Interesting, can you tell me something more on that", "I see, how do you think!", "Why?", "How do you think I feel when you say that?")
        return random.choice(responses)

if __name__ == "__main__":
    win = ChatWindow()
    win.mainloop()


import time
from tkinter import*
root = Tk()
canvas = Canvas(root, width='1340', height='700', bg='white')
canvas.pack()
time.sleep(0.03)
canvas.update()

canvas.create_rectangle(0, 0, 670, 350, outline='black', fill='Light Goldenrod', width=2)
canvas.create_rectangle(670, 0, 1340, 350, outline='black', fill='black', width=2)
canvas.create_rectangle(0, 350, 670, 700, outline='black', fill='black', width=2)
canvas.create_rectangle(670, 350, 1340, 700, outline='black', fill='white', width=2)

lastpic = PhotoImage(file=r"C:\Greenfox\Ferenc-Hartmann\week3\day5\lastpic.png")
item = canvas.create_image(1000, 550, image=lastpic)

root.mainloop()