#!/usr/bin/env python3

import glob
import gzip
import os
import re
import subprocess
import sys
import tempfile
import threading
import time

# You should change these to your liking
n_threads   = 8
datadirs    = "/tmp/rdcost/data/"
resultdir   = "/tmp/rdcost/coeff_buckets"

gzargs      = ["gzip", "-d"]
filtargs    = ["./frcosts_matrix"]
octargs     = ["octave-cli", "invert_matrix.m"]
filt2args   = ["./ols_2ndpart"]

class MultiPipeManager:
    pipe_fn_template  = "%02i.txt"

    def __init__(self, odpath, dest_qps):
        self.odpath = odpath
        self.dest_qps = dest_qps

        self.pipe_fns  = []
        for qp in dest_qps:
            pipe_fn  = os.path.join(self.odpath, self.pipe_fn_template % qp)
            self.pipe_fns.append(pipe_fn)

    def __enter__(self):
        os.makedirs(self.odpath, exist_ok=True)
        for pipe_fn in self.pipe_fns:
            try:
                os.unlink(pipe_fn)
            except FileNotFoundError:
                pass
            os.mkfifo(pipe_fn)
        return self

    def __exit__(self, *_):
        for pipe_fn in self.pipe_fns:
            os.unlink(pipe_fn)

    def items(self):
        for pipe_fn in self.pipe_fns:
            yield pipe_fn

class MTSafeIterable:
    def __init__(self, iterable):
        self.lock = threading.Lock()
        self.iterable = iterable

    def __iter__(self):
        return self

    def __next__(self):
        with self.lock:
            return next(self.iterable)

def read_in_blocks(f):
    BLOCK_SZ = 65536
    while True:
        block = f.read(BLOCK_SZ)
        if (len(block) == 0):
            break
        else:
            yield block

def exhaust_gzs(sink_f, gzs):
    for gz in gzs:
        with gzip.open(gz, "rb") as f:
            if (gz == "/tmp/rdcost/data/RaceHorses_416x240_30.yuv-qp22/20.txt.gz"):
                print("kjeh")
            print("  Doing %s ..." % gz)
            for block in read_in_blocks(f):
                sink_f.write(block)
                sink_f.flush()

def run_job(jobname, input_gzs):
    resultpath = os.path.join(resultdir, "%s.result" % jobname)
    print("Running job %s" % jobname)

    with tempfile.NamedTemporaryFile() as tf:
        filt = subprocess.Popen(filtargs, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        octa = subprocess.Popen(octargs, stdin=filt.stdout, stdout=tf)

        try:
            exhaust_gzs(filt.stdin, input_gzs)
        except OSError as e:
            print("OSError %s" % e, file=sys.stderr)
            raise

        filt.stdin.close()
        filt.wait()
        octa.wait()

        if (filt.returncode != 0):
            print("First stage failed: %s" % jobname, file=sys.stderr)
            assert(0)

        with open(resultpath, "w") as rf:
            f2a = filt2args + [tf.name]
            f2 = subprocess.Popen(f2a, stdin=subprocess.PIPE, stdout=rf)
            exhaust_gzs(f2.stdin, input_gzs)
            f2.communicate()
            if (filt.returncode != 0):
                print("Second stage failed: %s" % jobname, file=sys.stderr)
                assert(0)

    print("Job %s done" % jobname)

def threadfunc(joblist):
    for jobname, job in joblist:
        run_job(jobname, job)

def scan_datadirs(path):
    seq_names = set()
    for dirent in os.scandir(path):
        if (not dirent.is_dir()):
            continue
        match = re.search("^([A-Za-z0-9_]+\.yuv)-qp[0-9]{1,2}$", dirent.name)
        if (not match is None):
            seq_name = match.groups()[0]
            seq_names.add(seq_name)

    for seq_name in seq_names:
        seq_glob = os.path.join(path, seq_name + "-qp*/")

        for qp in range(51):
            job_name = seq_name + "-qp%02i" % qp
            qp_fn = "%02i.txt.gz" % qp
            yield job_name, glob.glob(os.path.join(seq_glob, qp_fn))

def main():
    for d in (datadirs, resultdir):
        os.makedirs(d, exist_ok=True)

    jobs = scan_datadirs(datadirs)
    joblist = MTSafeIterable(iter(jobs))

    threads = [threading.Thread(target=threadfunc, args=(joblist,)) for _ in range(n_threads)]
    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

if (__name__ == "__main__"):
    main()


"""Script to test the acquisition functions."""
from __future__ import print_function
from __future__ import absolute_import

import os
import unittest
import numpy as np

from ase.ga.data import DataConnection

from catlearn.api.ase_data_setup import get_unique, get_train
from catlearn.featurize.setup import FeatureGenerator
from catlearn.regression import GaussianProcess
from catlearn.active_learning.acquisition_functions import (
    rank, classify, probability_density)
from catlearn.active_learning.algorithm import ActiveLearning

wkdir = os.getcwd()

train_size, test_size = 30, 30

def classifier(atoms):
    """Simple function to classify atoms objects."""
    return atoms.get_chemical_formula()

class TestAcquisition(unittest.TestCase):
    """Test out the various acquisition routines."""

    def get_data(self):
        """Generate features from atoms objects."""
        # Connect database generated by a GA search.
        gadb = DataConnection('{}/data/gadb.db'.format(wkdir))

        # Get all relaxed candidates from the db file.
        print('Getting candidates from the database')
        all_cand = gadb.get_all_relaxed_candidates(use_extinct=False)

        # Setup the test and training datasets.
        testset = get_unique(atoms=all_cand, size=test_size, key='raw_score')

        trainset = get_train(atoms=all_cand, size=train_size,
                             taken=testset['taken'], key='raw_score')

        # Clear out some old saved data.
        for i in trainset['atoms']:
            del i.info['data']['nnmat']

        # Initiate the fingerprint generators with relevant input variables.
        print('Getting the fingerprints')
        f = FeatureGenerator()

        train_features = f.return_vec(
            trainset['atoms'], [f.nearestneighbour_vec])
        test_features = f.return_vec(
            testset['atoms'], [f.nearestneighbour_vec])

        train_targets = []
        for a in trainset['atoms']:
            train_targets.append(a.info['key_value_pairs']['raw_score'])
        test_targets = []
        for a in testset['atoms']:
            test_targets.append(a.info['key_value_pairs']['raw_score'])

        return train_features, train_targets, trainset['atoms'], \
            test_features, test_targets, testset['atoms']

    def test_acquisition(self):
        """Test acquisition functions."""
        train_features, train_targets, train_atoms, test_features, \
            test_targets, test_atoms = self.get_data()
        # Test prediction routine with gaussian kernel.
        kdict = [{'type': 'gaussian', 'width': 1., 'scaling': 1.}]
        gp = GaussianProcess(
            train_fp=train_features, train_target=train_targets,
            kernel_list=kdict, regularization=1e-3,
            optimize_hyperparameters=True, scale_data=True)
        pred = gp.predict(
            test_fp=test_features, test_target=test_targets,
            get_validation_error=True, get_training_error=True,
            uncertainty=True)

        print('gaussian prediction (rmse):',
              pred['validation_error']['rmse_average'])

        acq = rank(
            targets=train_targets, predictions=pred['prediction'],
            uncertainty=pred['uncertainty'], train_features=train_features,
            test_features=test_features, metrics=[
                'cdf', 'optimistic', 'pdf', 'UCB', 'EI', 'PI'])
        self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
        self.assertTrue(len(acq['pdf']) == len(pred['prediction']))
        self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
        self.assertTrue(len(acq['EI']) == len(pred['prediction']))
        self.assertTrue(len(acq['PI']) == len(pred['prediction']))

        acq = classify(
            classifier, train_atoms, test_atoms, targets=train_targets,
            predictions=pred['prediction'], uncertainty=pred['uncertainty'],
            train_features=train_features, test_features=test_features,
            metrics=['optimistic', 'pdf', 'UCB', 'EI', 'PI'])
        self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
        self.assertTrue(len(acq['pdf']) == len(pred['prediction']))
        self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
        self.assertTrue(len(acq['EI']) == len(pred['prediction']))
        self.assertTrue(len(acq['PI']) == len(pred['prediction']))

    def test_surrogate_model(self):
        train_features, train_targets, train_atoms, test_features, \
            test_targets, test_atoms = self.get_data()

        sg0 = ActiveLearning(_surrogate_model,
                             train_features, train_targets)
        batch_size = 10
        sg0.test_acquisition(batch_size=batch_size)
        to_acquire, _ = sg0.acquire(test_features, batch_size=batch_size)

        self.assertTrue(len(to_acquire) == batch_size)

        sg1 = ActiveLearning(_surrogate_model,
                             train_features, train_targets)
        batch_size = 3
        to_acquire, _ = sg1.acquire(test_features, batch_size=batch_size)

        self.assertTrue(len(to_acquire) == batch_size)

        sg2 = ActiveLearning(_surrogate_model,
                             train_features, train_targets)
        batch_size = 1
        to_acquire, _ = sg2.acquire(test_features, batch_size=batch_size)

        self.assertTrue(len(to_acquire) == batch_size)

        sg0.ensemble_test(size=2, batch_size=batch_size)

def _surrogate_model(train_features, train_targets,
                     test_features=None, test_targets=None):
    kdict = [{'type': 'gaussian', 'width': 0.5}]
    gp = GaussianProcess(
        train_fp=train_features, train_target=train_targets,
        kernel_list=kdict, regularization=1e-3,
        optimize_hyperparameters=False, scale_data=True)
    if test_targets is None:
        get_validation_error = False
    else:
        get_validation_error = True
    score = gp.predict(
        test_fp=test_features, test_target=test_targets,
        get_validation_error=get_validation_error,
        get_training_error=False,
        uncertainty=True)
    to_acquire = np.argsort(probability_density(0.,
                                                score['prediction'],
                                                score['uncertainty']))
    return to_acquire, score

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
# *************************
# 11 De Octubre del 2018
# Emulador Procesador Z80
# Jorge Ivan Torres
# Daniel Caita
# *************************

import Funcions
from Z80CPU import dicFunciones, registros, operar
import EnlzCarg
import os
import ALZ80

def start_ins(ins):
    #print (ins)
    toUser = list()
    itx = list()
    l = ALZ80.lector(ins)
    #print(itx, len(itx))
    #print(l)
    ALZ80.Lexema(l, itx)
    #print ("ITX",itx)
    LToCpu = EnlzCarg.enlazador(dicFunciones, registros, itx)
    #print(LToCpu)
    if LToCpu[0] == 0:
        toUser.append(0)
        toUser.append(LToCpu[1])
        return toUser
    else:
        if len(LToCpu) == 3:
            operar(LToCpu[1],LToCpu[2],'')
        elif len(LToCpu) == 2:
            operar(LToCpu[1],'','')
        else:
            #print("paso por aquí")
            operar(LToCpu[1],LToCpu[2],LToCpu[3])


from typing import Optional, List
from fastapi import FastAPI, Cookie, HTTPException, status
import random
from fastapi.param_functions import Query
import requests

import recommend

app = FastAPI(
    docs_url='/recommendapi/docs',
    openapi_url='/recommendapi/openapi.json',
)

@app.get('/recommendapi/')
async def index():
    return {'message': 'Hello, recommendapi!'}

@app.get(
    '/recommendapi/project',
    description='recommend project',
    status_code=status.HTTP_200_OK,
    responses={
        status.HTTP_200_OK: {
            'model': List[int],
            'description': 'Successful Response (list of project-id)',
        },
        status.HTTP_401_UNAUTHORIZED: {
            'description': 'Not logged in (cookie token is required)'
        },
        status.HTTP_404_NOT_FOUND: {
            'description': 'User not found.',
        },
        status.HTTP_500_INTERNAL_SERVER_ERROR: {
            'description': 'Internal server error'
        }
    },
)
async def recommend_projects_with_usertoken(
    token: Optional[str] = Cookie(None),
    limit: Optional[int] = Query(None), offset: Optional[int] = Query(None)
):
    if token is None:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)

    r = recommend.RecommendWithUsertoken(token)
    r.gather_projects()
    r.calc_point()

    sorted_projects = sorted(r.points.items(), key=lambda x: x[1],
                             reverse=True)

    result = []
    for p_id, _ in sorted_projects:
        # random project
        if random.randint(0, 9) % 3 == 0:
            resp = requests.get(
                'http://projectapi:8000/projectapi/project/random'
            )
            if resp.status_code != 200:
                raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)
            rand_id = resp.json()[0]
            if rand_id not in [x for x, _ in sorted_projects]:
                result.append(rand_id)

        result.append(p_id)

    if offset is None:
        return result[:limit]

    if limit is None:
        return result[offset:]

    return result[offset:offset+limit]

@app.get(
    '/recommendapi/project/no-user',
    description='Recommend projects without user-token',
    status_code=status.HTTP_200_OK,
)
async def recommend_projects_without_usertoken(
    limit: Optional[int] = Query(None), offset: Optional[int] = Query(None)
):
    endpoint = 'http://projectapi:8000/projectapi/project/all'
    sorted_by_like_resp = requests.get(
        f'{endpoint}?sort_by=LIKE&reverse=false'
    )
    if sorted_by_like_resp.status_code != 200:
        raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)
    sorted_by_like: List[int] = sorted_by_like_resp.json()

    sorted_by_date_resp = requests.get(
        f'{endpoint}?sort_by=DATETIME&reverse=false'
    )
    if sorted_by_date_resp.status_code != 200:
        raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)
    sorted_by_date: List[int] = sorted_by_date_resp.json()

    result: List[int] = []
    if len(sorted_by_date) != len(sorted_by_like):
        raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)

    len_sorted_by_date = len(sorted_by_date)
    while len(result) < len_sorted_by_date:
        c = random.choice(['like', 'like', 'date'])  # like:date = 2:1
        print(c)
        if len(sorted_by_like) > 0 and c == 'like':
            p = sorted_by_like.pop()
            if p not in result:
                result.append(p)
        elif len(sorted_by_date) > 0 and c == 'date':
            p = sorted_by_date.pop()
            if p not in result:
                result.append(p)
        else:
            continue

    if offset is None:
        return result[:limit]

    if limit is None:
        return result[offset:]

    return result[offset:offset+limit]


# -*- coding: utf-8 -*-
"""I/O of hierarchical configurations in various formats."""

import os.path

import jsonschema
import six

import ubimaior.formats
import ubimaior.mappings

class ConfigSettings(ubimaior.mappings.MutableMapping):
    """Manages the settings for hierarchical configurations."""

    #: Settings that are currently permitted
    valid_settings = ("scopes", "format", "schema")

    def __init__(self):
        self._settings = {"scopes": None, "format": None, "schema": None}

    def __getitem__(self, key):
        self._validate_key(key)
        return self._settings[key]

    def __setitem__(self, key, value):
        self._validate_key(key)
        self._validate_value(key, value)
        self._settings[key] = value

    def __delitem__(self, key):
        self._validate_key(key)
        del self._settings[key]

    def __iter__(self):
        return iter(self._settings)

    def __len__(self):
        return len(self._settings)

    @staticmethod
    def _validate_value(key, value):
        """Validate the arguments written as defaults.

        Args:
            key (str): default setting
            value: new value to use for the setting

        """
        if key == "scopes":
            if not isinstance(value, list):
                msg = '"scopes" must be a list of tuples [current type is {0}]'
                raise TypeError(msg.format(type(value).__name__))

            if not all(isinstance(x, tuple) and len(x) == 2 for x in value):
                msg = 'item in "scopes" must be tuples [(scope_name, dir)]'
                raise TypeError(msg)

        if key == "format":
            if not isinstance(value, six.string_types):
                msg = '"format" must be a valid string'
                raise TypeError(msg)

            if value not in ubimaior.formats.FORMATTERS:
                msg = "unknown format type. Allowed values are {0}"
                raise ValueError(msg.format(", ".join(ubimaior.formats.FORMATTERS)))

    def _validate_key(self, key):
        if key not in self.valid_settings:
            msg = "allowed settings are {0}".format(", ".join(self._settings.keys()))
            raise KeyError(msg)

    def __getattr__(self, name):
        if name not in self.valid_settings:
            msg = "{0} object has no attribute {1}"
            raise AttributeError(msg.format(type(self).__name__, name))
        return self._settings[name]

#: Default settings for the module
DEFAULTS = ConfigSettings()

#: Schema for the global configuration file
_UBIMAIOR_CFG_SCHEMA = {
    "type": "object",
    "additionalProperties": True,
    "properties": {
        "format": {
            "description": "The format used for the configuration file",
            "enum": list(ubimaior.formats.FORMATTERS),
        },
        "scopes": {
            "description": "The hierarchical scopes of the configuration as"
            " a list of (name, path) tuples",
            "type": "array",
            "minItems": 1,
            "uniqueItems": True,
            "items": {
                "type": "array",
                "minItems": 2,
                "maxItems": 2,
                "items": {"type": "string"},
            },
        },
        "schema": {
            "description": "Path to the schema file used to validate the hierarchy",
            "type": "string",
        },
    },
    "required": ["format", "scopes"],
}

def setup_from_file(configuration_file):
    """Sets ubimaior global defaults from a configuration file.

    Args:
        configuration_file (str): path to the configuration file
    """
    # Check if the configuration file exists
    if not os.path.exists(configuration_file):
        msg = 'configuration file "{0}" does not exist'
        raise IOError(msg.format(configuration_file))

    configuration_file = os.path.abspath(configuration_file)
    configuration_dir = os.path.dirname(configuration_file)

    def make_abs(path):
        if os.path.isabs(path):
            return path
        return os.path.abspath(os.path.join(configuration_dir, path))

    # Retrieve the correct formatter to load the configuration
    _, fmt = os.path.splitext(configuration_file)
    formatter = ubimaior.formats.FORMATTERS[fmt.strip(".")]

    # Load the settings and return them
    with open(configuration_file) as cfg_stream:
        configuration = formatter.load(cfg_stream)
        jsonschema.validate(configuration, _UBIMAIOR_CFG_SCHEMA)

    # Ensure that scopes is a list of tuples
    scopes = [(name, make_abs(d)) for name, d in configuration["scopes"]]
    set_default_scopes(scopes)

    set_default_format(configuration["format"])

    if "schema" in configuration:
        set_default_schema(make_abs(configuration["schema"]))

def search_file_in_path(filename, start_dir=None):
    """Searches for a file starting from the directory passed as argument and proceeding
    up to parent directories, until root. Raises if the file is not found.

    Args:
        filename (str): name of the file to look for
        start_dir (path): directory where to start looking for the file

    Returns:
        Absolute path to the file

    Raises:
        IOError: if the file is not found.
    """
    # If the file is given with an absolute path just check it exists (or raise)
    if os.path.isabs(filename):
        if os.path.exists(filename) and os.path.isfile(filename):
            return filename
        raise IOError("file not found [{0}]".format(filename))

    # Otherwise search in start_dir and proceed up to parent until root is reached
    start_dir = start_dir or os.getcwd()
    start_dir = os.path.realpath(os.path.abspath(start_dir))

    while True:
        abs_filename = os.path.join(start_dir, filename)
        if os.path.exists(abs_filename) and os.path.isfile(abs_filename):
            return abs_filename

        start_dir, is_root = (
            os.path.dirname(start_dir),
            os.path.dirname(start_dir) == start_dir,
        )
        if is_root:
            raise IOError("file not found [{0}]".format(filename))

def set_default_scopes(scopes):
    """Sets the default scopes to look for configurations.

    Args:
        scopes (list of tuples): the new scopes to be used as a default

    Raises:
        TypeError: when ``scopes`` is not a list of two elements tuples

    """
    DEFAULTS["scopes"] = scopes

def set_default_format(fmt):
    """Sets the default format to encode/decode configuration files

    Args:
        fmt (str): a supported format for configuration files

    Raises:
        TypeError: if format is not a string
        ValueError: if the requested format is not among the
            supported ones
    """
    DEFAULTS["format"] = fmt

def set_default_schema(schema):
    """Sets the default schema to validate configuration

    Args:
        schema (dict): a valid jsonschema schema

    """
    DEFAULTS["schema"] = schema

def validate(cfg_object, schema):
    """Validates a configuration object against a schema.

    Args:
        cfg_object: configuration object to be validated
        schema (dict or path): either a schema object already in
            memory or a path where to read one. If it is a file
            all the registered formats are accepted (JSON, YAML, etc.)

    Raises:
        jsonschema.ValidationError: if the configuration object doesn't conform
            to the schema
        jsonschema.SchemaError: if the schema does not conform to the jsonschema
            specifications
        ValueError: if ``schema`` is a file and either the file does not exist or
            its format is not recognized
    """
    # If schema is a string, assume it's a file containing the schema
    if isinstance(schema, six.string_types):
        # If it is not a valid file raise an appropriate error
        if not os.path.isfile(schema):
            msg = '"{0}" does not exist or is not a file'
            # TODO: in python > 3 a FileNotFoundError would be more appropriate
            raise ValueError(msg.format(schema))

        schema_format = os.path.splitext(schema)[1].lstrip(".")

        # If the format of the file is not recognized, raise an error too
        if schema_format not in ubimaior.formats.FORMATTERS:
            msg = '"{0}" is not a valid format [Allowed formats are: {1}]'
            msg = msg.format(schema_format, ", ".join(list(ubimaior.formats.FORMATTERS)))
            raise ValueError(msg)

        schema_formatter = ubimaior.formats.FORMATTERS[schema_format]
        schema_file = schema
        with open(schema_file) as schema_f:
            schema = schema_formatter.load(schema_f)

    flattened_obj = cfg_object.as_dict()
    jsonschema.validate(flattened_obj, schema)

def _is_empty(item):
    # If it is not a dictionary cast to bool
    if not isinstance(item, ubimaior.mappings.MutableMapping):
        return not bool(item)

    # If it is a dictionary recurse
    if not item:
        return True
    return all(_is_empty(v) for v in item.values())

def load(config_name, scopes=None, config_format=None, schema=None):
    """Loads a hierarchical configuration into an object.

    Leaving None in any of the optional arguments below means to read its
    value from the default settings.

    Args:
        config_name (str): name of the configuration files to load
        scopes (list or None): list of ``(name, directory)`` tuples that
            represent the scopes of the hierarchical configuration
        config_format (str or None): format of the configuration files
        schema (dict or None): configuration parameters for schema

    Returns:
        Merged configuration object
    """

    # Retrieve default values
    settings = retrieve_settings(scopes, config_format, schema)

    # Construct the filename of the configuration
    config_filename = config_name + "." + settings.format

    # Retrieve the reader of the files
    formatter = ubimaior.formats.FORMATTERS[settings.format]

    mappings = []
    for scope_name, directory in settings.scopes:
        current = os.path.join(directory, config_filename)

        # Check if the file we are looking for exists
        if not os.path.exists(current) and not os.path.isfile(current):
            mappings.append((scope_name, {}))
            continue

        # If so load the content and append it to the hierarchy
        with open(current) as partial_cfg:
            mappings.append((scope_name, formatter.load(partial_cfg)))

    obj = ubimaior.mappings.OverridableMapping(mappings)

    # Validate the object against the supplied schema
    if schema:
        validate(obj, schema)

    return obj

def dump(cfg, config_name, scopes=None, config_format=None, schema=None):
    """Loads a hierarchical configuration into an object.

    Leaving None in any of the optional arguments below means to read its
    value from the default settings.

    Args:
        cfg: hierarchical configuration object to be dumped
        config_name (str): name of the configuration files where to dump
            the object
        scopes (list or None): list of ``(name, directory)`` tuples that
            represent the scopes of the hierarchical configuration
        config_format (str or None): format of the configuration files
        schema (dict or None): configuration parameters for schema

    Raises:
        ValueError: if ``cfg`` scopes do not match ``scopes``
    """
    # Retrieve default values
    settings = retrieve_settings(scopes, config_format, schema)

    # Construct the filename of the configuration
    config_filename = config_name + "." + settings.format

    # Retrieve the writer
    formatter = ubimaior.formats.FORMATTERS[settings.format]

    # Check that the configuration object is valid and consistent with settings
    _validate_cfg_consistency(cfg, settings)

    # Validate the object against the supplied schema
    if schema:
        validate(cfg, schema)

    # Dump the scopes to file
    scopes_d = dict(settings.scopes)
    for scope_name, obj in cfg.mappings.items():
        # If the object is empty, then continue to dump the hierarchy
        if _is_empty(obj):
            continue

        directory = scopes_d[scope_name]
        current = os.path.join(directory, config_filename)
        with open(current, "w") as partial_cfg:
            formatter.dump(obj, partial_cfg)

def retrieve_settings(scopes=None, config_format=None, schema=None):
    """Retrieves the settings to be used when dumping or loading a
    hierarchical configuration.

    Returns:
        ConfigSettings: object containing the current settings
    """
    settings = ConfigSettings()
    settings["scopes"] = scopes or DEFAULTS["scopes"]
    settings["format"] = config_format or DEFAULTS["format"]
    # TODO: still to be implemented
    settings["schema"] = schema or DEFAULTS["schema"]
    return settings

def _validate_cfg_consistency(cfg, settings):
    """Checks that ``cfg`` can be dumped and is consistent with settings.

    Args:
        cfg: configuration object
        settings: settings of the hierarchical configuration

    Raises:
        ValueError: if any inconsistency is found between ``cfg`` and
            ``settings``
    """
    # Check that we don't have modifications in scratch still to be merged
    if not _is_empty(cfg.mappings["_scratch_"]):
        msg = "cannot dump an object with modifications in scratch"
        raise ValueError(msg)
    # Check that the current object matches the scope that will be used
    scopes_in_object = [x for x, _ in cfg.mappings.items() if x != "_scratch_"]
    scopes_in_settings = [x for x, _ in settings.scopes]
    if scopes_in_object != scopes_in_settings:
        msg = "scopes in the object do not match with scopes in settings"
        raise ValueError(msg)


""" Common functions for controllers
implement commonly used functions here
"""
import os
from view import terminal_view

def add(table, record):
    """
    Add new record to table

    Args:
        table (list): table to add new record to
        record (list): new record

    Returns:
        list: Table with a new record
    """
    # your code
    table = table + [record]
    return table

def remove(table, id_):
    """
    Remove a record with a given id from the table.

    Args:
        table (list): table to remove a record from
        id_ (str): id of a record to be removed

    Returns:
        list: Table without specified record.
    """
    for a in range(0, len(table)):
        if id_ == table[a][0]:
            table = table[:a] + table[a+1:]
            return table

def update(table, id_, record):
    """
    Updates specified record in the table.

    Args:
        table: list in which record should be updated
        id_ (str): id of a record to update
        record (list): updated record

    Returns:
        list: table with updated record
    """
    for a in range(0, len(table)):
        if id_ == table[a][0]:
            table[a] = record
    return table

def all_print_table(title_list, file):
    file_name = get_input("Choose a file: ")
    if file_name == "":
        file_name = file
    table = get_table_from_file(file_name)
    terminal_view.print_table(table, title_list)
    waiting()
    os.system("clear")

def all_add(title_list, file):
    file_name = get_input("Choose a file: ")
    if file_name == "":
        file_name = file
    table = get_table_from_file(file_name)
    terminal_view.print_table(table, title_list)
    record = terminal_view.get_inputs(title_list, "Enter data: ")
    table = add(table, record)
    write_table_to_file(file_name, table)
    os.system("clear")
    terminal_view.gprint('*** Record has been added ***')
    waiting()
    os.system("clear")

def all_remove(title_list, file):
    file_name = get_input("Choose a file: ")
    if file_name == "":
        file_name = file
    table = get_table_from_file(file_name)
    terminal_view.print_table(table, title_list)
    id_ = get_input("Enter id of a record you want to remove: ")
    table = remove(table, id_)
    write_table_to_file(file_name, table)
    os.system("clear")
    terminal_view.gprint('*** Record has been removed ***')
    waiting()
    os.system("clear")

def all_updates(title_list, file):
    file_name = get_input("Choose a file: ")
    if file_name == "":
        file_name = file
    table = get_table_from_file(file_name)
    terminal_view.print_table(table, title_list)
    id_ = get_input("Enter id to update: ")
    record = terminal_view.get_inputs(title_list, "Enter data: ")
    table = update(table, id_, record)
    write_table_to_file(file_name, table)
    os.system("clear")
    terminal_view.gprint('*** Record has been updated ***')
    waiting()
    os.system("clear")

def write_table_to_file(file_name, table):
    """
    Writes list of lists into a csv file.

    Args:
        file_name (str): name of file to write to
        table (list): list of lists to write to a file

    Returns:
         None
    """
    with open(file_name, "w") as file:
        for record in table:
            row = ';'.join(record)
            file.write(row + "\n")

def waiting():
    input()

def get_table_from_file(file_name):
    """
    Reads csv file and returns it as a list of lists.
    Lines are rows columns are separated by ";"

    Args:
        file_name (str): name of file to read

    Returns:
         list: List of lists read from a file.
    """
    with open(file_name, "r") as file:
        lines = file.readlines()
    table = [element.replace("\n", "").split(";") for element in lines]
    return table

def get_input(title):
    inp = input(title)
    return inp

def get_file():
    file_exists = True
    while file_exists:
        try:
            inp = input("Choose file: ")
            if inp == "":
                return inp
            with open(inp, "r") as file:
                pass
            return inp
        except FileNotFoundError:
            print("There is no such file")

def get_double_file(title):
    file_exists = True
    while file_exists:
        try:
            inp = input(title)
            if inp == "":
                return inp
            with open(inp, "r") as file:
                pass
            return inp
        except FileNotFoundError:
            print("There is no such file")


import json
import random
from tqdm import tqdm
import torch
import numpy as np
from transformers import AutoModel, AutoTokenizer, AutoModelForQuestionAnswering, AdamW, get_linear_schedule_with_warmup
import torch.nn.functional as F

# Задаем путь к файлу с данными для обучения
train_file_path = 'train.json'

# Загружаем данные из файла
with open(train_file_path, 'r', encoding="utf-8") as f:
    data = json.load(f)

# Задаем параметры обучения
epochs = 3
batch_size = 8
learning_rate = 5e-5
adam_epsilon = 1e-8

# Загружаем предобученный токенизатор и модель
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelForQuestionAnswering.from_pretrained("bert-base-uncased")

# Задаем оптимизатор и расписание обучения
optimizer = AdamW(model.parameters(), lr=learning_rate, eps=adam_epsilon)
total_steps = len(data) * epochs // batch_size
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)

# Создаем функцию для формирования батчей данных
def create_batches(data, batch_size):
    batches = []
    batch = []
    for item in data:
        batch.append(item)
        if len(batch) == batch_size:
            batches.append(batch)
            batch = []
    if batch:
        batches.append(batch)
    return batches

# Создаем функцию для обучения модели на одном батче данных
def train_batch(batch, tokenizer, model, optimizer):
    input_ids = []
    attention_masks = []
    start_positions = []
    end_positions = []

    for item in batch:
        text = item['text']
        label = item['label']
        extracted_part_text = item['extracted_part']['text'][0]
        answer_start = item['extracted_part']['answer_start'][0]
        answer_end = item['extracted_part']['answer_end'][0]

        # Формируем входные данные для модели
        encoded_dict = tokenizer(text, extracted_part_text, return_tensors='pt', padding=True, truncation='only_first')
        input_ids.append(encoded_dict['input_ids'])
        attention_masks.append(encoded_dict['attention_mask'])
        start_positions.append(torch.tensor([answer_start]))
        end_positions.append(torch.tensor([answer_end]))

    # Преобразуем списки в тензоры
    input_ids = torch.cat(input_ids, dim=0)
    attention_masks = torch.cat(attention_masks, dim=0)
    start_positions = torch.cat(start_positions, dim=0)
    end_positions = torch.cat(end_positions, dim=0)

    # Передаем входные данные в модель и получаем предсказания
    model.train()
    outputs = model(input_ids, attention_mask=attention_masks, start_positions=start_positions, end_positions=end_positions)

    # Вычисляем функцию потерь и выполняем обратное распространение ошибки
    loss = outputs.loss
    loss.backward()

    # Обновляем параметры модели
    optimizer.step()
    scheduler.step()
    model.zero_grad()

    return loss.item()

def train(data, tokenizer, model, optimizer, device, batch_size=16, num_epochs=5):
    train_loss = []
    for epoch in range(num_epochs):
        print(f"EPOCH {epoch + 1}/{num_epochs}")
        epoch_loss = 0
        epoch_steps = 0
        model.train()
        for i in tqdm(range(0, len(data), batch_size)):
            batch = data[i:i+batch_size]
            texts = [d["text"] for d in batch]
            labels = [d["label"] for d in batch]
            inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt").to(device)
            labels = torch.tensor([0 if label == "обеспечение исполнения контракта" else 1 for label in labels]).to(device)
            #outputs = model(**inputs)[0]
            labels = labels.unsqueeze(1)
            outputs = model(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'])[0]

            loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
            epoch_loss += loss.item()
            epoch_steps += 1
        epoch_loss /= epoch_steps
        print(f"LOSS: {epoch_loss}")
        train_loss.append(epoch_loss)
    return train_loss

# Тренировка модели
model_name = "distilbert-base-uncased"
model = AutoModelForQuestionAnswering.from_pretrained("bert-base-uncased")
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
train_loss = train(data, tokenizer, model, optimizer, device)

torch.save(model.state_dict(), 'model.pt')


import pyblish.api

from openpype.client import get_subset_by_name, get_asset_by_name
import openpype.lib.usdlib as usdlib

class CollectUsdBootstrap(pyblish.api.InstancePlugin):
    """Collect special Asset/Shot bootstrap instances if those are needed.

    Some specific subsets are intended to be part of the default structure
    of an "Asset" or "Shot" in our USD pipeline. For example, for an Asset
    we layer a Model and Shade USD file over each other and expose that in
    a Asset USD file, ready to use.

    On the first publish of any of the components of a Asset or Shot the
    missing pieces are bootstrapped and generated in the pipeline too. This
    means that on the very first publish of your model the Asset USD file
    will exist too.

    """

    order = pyblish.api.CollectorOrder + 0.35
    label = "Collect USD Bootstrap"
    hosts = ["houdini"]
    families = ["usd", "usd.layered"]

    def process(self, instance):

        # Detect whether the current subset is a subset in a pipeline
        def get_bootstrap(instance):
            instance_subset = instance.data["subset"]
            for name, layers in usdlib.PIPELINE.items():
                if instance_subset in set(layers):
                    return name  # e.g. "asset"
                    break
            else:
                return

        bootstrap = get_bootstrap(instance)
        if bootstrap:
            self.add_bootstrap(instance, bootstrap)

        # Check if any of the dependencies requires a bootstrap
        for dependency in instance.data.get("publishDependencies", list()):
            bootstrap = get_bootstrap(dependency)
            if bootstrap:
                self.add_bootstrap(dependency, bootstrap)

    def add_bootstrap(self, instance, bootstrap):

        self.log.debug("Add bootstrap for: %s" % bootstrap)

        project_name = instance.context.data["projectName"]
        asset = get_asset_by_name(project_name, instance.data["asset"])
        assert asset, "Asset must exist: %s" % asset

        # Check which are not about to be created and don't exist yet
        required = {"shot": ["usdShot"], "asset": ["usdAsset"]}.get(bootstrap)

        require_all_layers = instance.data.get("requireAllLayers", False)
        if require_all_layers:
            # USD files load fine in usdview and Houdini even when layered or
            # referenced files do not exist. So by default we don't require
            # the layers to exist.
            layers = usdlib.PIPELINE.get(bootstrap)
            if layers:
                required += list(layers)

        self.log.debug("Checking required bootstrap: %s" % required)
        for subset in required:
            if self._subset_exists(project_name, instance, subset, asset):
                continue

            self.log.debug(
                "Creating {0} USD bootstrap: {1} {2}".format(
                    bootstrap, asset["name"], subset
                )
            )

            new = instance.context.create_instance(subset)
            new.data["subset"] = subset
            new.data["label"] = "{0} ({1})".format(subset, asset["name"])
            new.data["family"] = "usd.bootstrap"
            new.data["comment"] = "Automated bootstrap USD file."
            new.data["publishFamilies"] = ["usd"]

            # Do not allow the user to toggle this instance
            new.data["optional"] = False

            # Copy some data from the instance for which we bootstrap
            for key in ["asset"]:
                new.data[key] = instance.data[key]

    def _subset_exists(self, project_name, instance, subset, asset):
        """Return whether subset exists in current context or in database."""
        # Allow it to be created during this publish session
        context = instance.context
        for inst in context:
            if (
                inst.data["subset"] == subset
                and inst.data["asset"] == asset["name"]
            ):
                return True

        # Or, if they already exist in the database we can
        # skip them too.
        if get_subset_by_name(
            project_name, subset, asset["_id"], fields=["_id"]
        ):
            return True
        return False


from typing import (
    ClassVar,
    Dict,
    Optional,
    Tuple,
)
from typing_extensions import (
    Self,  # Python 3.11 adds native support
)

from collections.abc import (
    Iterable,
)

import attrs
import cattrs

import models._converters as convs_registry
import api_utils
import models._constants as constants

class NonExistingEntityRequested(Exception):

    """Raised when the entity with the given id does not exist in db."""

ID_FIELD = attrs.field(
        default=None,
        metadata={constants.FieldSpecifiers.ID_COLUMN: True},
)
main_fk_field = attrs.field(metadata={constants.FieldSpecifiers.MAIN_FK_COLUMN: True})

@attrs.define(kw_only=True)
class _BaseModel:

    # This is present only for type annotations and auto complete in id's.
    # The actual converter instance
    # is added dynamically by the decorator at runtime.
    to_json_converter: ClassVar[cattrs.Converter]

    @classmethod
    def id_field_name(cls):
        return cls.field_name_for_truthy_metadata(constants.FieldSpecifiers.ID_COLUMN)

    @classmethod
    def field_name_for_truthy_metadata(cls, metadata_key: constants.FieldSpecifiers) -> str:
        for field_name, field_atrs in attrs.fields_dict(cls).items():
            if field_atrs.metadata.get(metadata_key, False):
                return field_name
        else:
            raise RuntimeError(
                (
                    f"Failed to find field with metadata {metadata_key}, "
                    f"for class {cls}"
                )
            )

    @property
    def id(self) -> Optional[int]:
        """Return id of this record.
        If this is returns `None` the record has not been inserted into database yet.
        """

    def set_id(self, new_val: Optional[int]) -> None:
        setattr(
            self,
            self.field_name_for_truthy_metadata(constants.FieldSpecifiers.ID_COLUMN),
            new_val
        )

    db_table_name: ClassVar[str]
    get_endpoint: ClassVar[str]
    get_single_end_point: ClassVar[str]

    @staticmethod
    def records_from_end_point(end_point_name):
        yield from api_utils.get_data(end_point_name)["item"]

    @classmethod
    def from_json_info(cls, json_info) -> Self:
        return convs_registry.from_json_conv.structure_attrs_fromdict(
            json_info, cls
        )

    @classmethod
    def from_endpoint(cls) -> Iterable[Self]:
        for record in cls.records_from_end_point(cls.get_endpoint):
            yield cls.from_json_info(record)

    @classmethod
    def from_end_point_by_id(cls, entity_id: int) -> Self:
        res = api_utils.get_data(
            cls.get_single_end_point,
            entity_id=str(entity_id)
        )["item"]
        if res is not None:
            return cls.from_json_info(res)
        raise NonExistingEntityRequested

    @classmethod
    def from_normalized_record(cls, record) -> Self:
        return cls(**record)

    def cols_to_attrs(self):

        def should_include(attr, val) -> bool:
            return attr.metadata.get(constants.FieldSpecifiers.ID_COLUMN, True)

        return attrs.asdict(
            inst=self,
            filter=should_include,
            recurse=False
        )

    def cols_for_insert(self) -> Dict:
        res = self.to_json_converter.unstructure(self)
        return res

    def insert_into_db(self) -> None:
        if self.id is not None:
            raise RuntimeError("Model is already in the database.")
        current_instance_vars = self.cols_for_insert()
        self.set_id(api_utils.post_data(self.add_endpoint, current_instance_vars))

    def update_db_record(self, new_values: Dict) -> Self:
        record_with_updated_vals = attrs.evolve(self, **new_values)
        updated_fields = record_with_updated_vals.to_json_converter.unstructure(record_with_updated_vals)
        api_utils.edit_data(self.edit_endpoint, str(self.id), updated_fields)
        return record_with_updated_vals

    def delete_db_record(self) -> None:
        api_utils.delete_data(self.delete_endpoint, {self.id_field_name(): self.id})

@attrs.define(kw_only=True)
class _Owned_model(_BaseModel):

    @property
    def owner(self) -> _BaseModel:
        """Return the entity representing main foreign key for this model"""
        return getattr(self, self.fk_field_name())

    @classmethod
    def fk_field_name(cls):
        return cls.field_name_for_truthy_metadata(constants.FieldSpecifiers.MAIN_FK_COLUMN)

    @owner.setter
    def owner(self, new_val):
        setattr(
            self,
            self.fk_field_name(),
            new_val
        )

    @staticmethod
    def data_from_end_point(end_point_name, end_point_id):
        yield from api_utils.get_data(
            end_point_name,
            entity_id=end_point_id
        )["item"]

    @classmethod
    def from_endpoint(cls, owner: _BaseModel):
        for record in cls.data_from_end_point(
            end_point_name=cls.get_endpoint,
            end_point_id=str(owner.id)
        ):
            yield cls.from_json_info(record)


from ast import arg
import cmd, sys
from tracemalloc import start
import pynames
import inspect

class GeneratingShell(cmd.Cmd):
    intro = 'Welcome to the name generation shell.   Type help or ? to list commands.\n'
    prompt = '(generator) '
    language = 'NATIVE'
    genders = ['MALE', 'FEMALE']
    languages = ['RU', 'EN', 'NATIVE']
    generators = []
    for generator in pynames.get_all_generators():
        name = generator.__name__
        subclass_position = name.find('NamesGenerator') + name.find('FullnameGenerator') + 1
        subclass_name = ''
        if subclass_position > -1:
            subclass_name = name[0:subclass_position]
        class_name = generator.__module__.split('.')[-1]
        if (subclass_name.lower() == class_name):
            generators.append({'class': class_name, 'subclass': '', 'generator': generator.__module__ + '.' + generator.__name__})
        else:
            generators.append({'class': class_name, 'subclass': subclass_name, 'generator': generator.__module__ + '.' + generator.__name__})

    def do_language(self, arg):
        'Set generating language: language <language>'
        args = arg.split()
        if (args[0] in self.languages):
            self.language = args[0]
    def complete_language(self, text, line, begidx, endidx):
        return [l for l in self.languages if l.startswith(text)]

    def do_generate(self, arg):
        'Generate name: generate <class> <subclass> <gender> or generate <class> <subclass>'
        args = arg.split()
        gender = 'ALL'
        if (args[-1].upper() in self.genders):
            gender = args[-1].upper()
        if (len(args) == 1) or (len(args) == 2 and gender != 'ALL'):
            if gender == 'ALL':
                gender = 'MALE'
            for gen in self.generators:
                if (gen['class'] == args[0]):
                    print(eval(gen['generator'] + '().get_name_simple(pynames.GENDER.' + gender + ', pynames.LANGUAGE.' + self.language + ')'))
                    return
        if (len(args) == 2) or (len(args) == 3 and gender != 'ALL'):
            if gender == 'ALL':
                gender = 'MALE'
            for gen in self.generators:
                if (gen['class'] == args[0]) and (gen['subclass'] == args[1]):
                    print(eval(gen['generator'] + '().get_name_simple(pynames.GENDER.' + gender + ', pynames.LANGUAGE.' + self.language + ')'))
                    return
    def complete_generate(self, text, line, begin, end):
        command = line.split(' ')
        if (len(command) == 2):
            return [l['class'] for l in self.generators if l['class'].startswith(text)]
        if (len(command) == 3):
            main_generator = [l for l in self.generators if l['class'] == command[1]][0]
            if main_generator['subclass'] == '':
                return [g for g in self.genders if g.startswith(command[2])]
            else:
                return [l['subclass'] for l in self.generators if (l['class'] == command[1]) and (l['subclass'].startswith(command[2]))]
        if (len(command) == 4):
            return [g for g in self.genders if g.startswith(command[3])]

    def do_info(self, arg):
        'Print info about class/subclass: info <class> <subclass> or info <class> <subclass> <gender> or info <class> <subclass> language'
        args = arg.split()
        gender = 'ALL'
        if (args[-1].upper() in self.genders):
            gender = args[-1].upper()
        if (len(args) == 1) or (len(args) == 2 and gender != 'ALL'):
            for gen in self.generators:
                if (gen['class'] == args[0]):
                    print(eval(gen['generator'] + '().get_names_number(pynames.GENDER.' + gender + ')'))
                    return
        if (len(args) == 2) or (len(args) == 3 and gender != 'ALL'):
            for gen in self.generators:
                if (gen['class'] == args[0]) and (gen['subclass'] == args[1]):
                    print(eval(gen['generator'] + '().get_names_number(pynames.GENDER.' + gender + ')'))
                    return
        if (args[-1] == 'language'):
            if (len(args) == 2):
                for gen in self.generators:
                    if (gen['class'] == args[0]):
                        print(' '.join(eval(gen['generator'] + '().languages')))
            else:
                for gen in self.generators:
                    if (gen['class'] == args[0]) and (gen['subclass'] == args[1]):
                        print(' '.join(eval(gen['generator'] + '().languages')))
    def complete_info(self, text, line, begin, end):
        command = line.split(' ')
        if (len(command) == 2):
            return [l['class'] for l in self.generators if l['class'].startswith(text)]
        if (len(command) == 3):
            main_generator = [l for l in self.generators if l['class'] == command[1]][0]
            if main_generator['subclass'] == '':
                completion_list = [g for g in self.genders if g.startswith(command[2])]
                if ("language".startswith(command[2])):
                    completion_list.append("language")
                return completion_list
            else:
                return [l['subclass'] for l in self.generators if (l['class'] == command[1]) and (l['subclass'].startswith(command[2]))]
        if (len(command) == 4):
            completion_list = [g for g in self.genders if g.startswith(command[3])]
            if ("language".startswith(command[3])):
                completion_list.append("language")
            return completion_list

    def do_exit(self, arg):
        'Exit name generation:  exit'
        return True

if __name__ == '__main__':
    GeneratingShell().cmdloop()


# Import required libraries
import os
import pickle
import random

import cv2
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm

# Level 2 - display information about errors only
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras import datasets, layers, models

# Data needed for training process
MAIN_PATH = os.path.dirname(os.path.realpath(__file__))
DATADIR = f'{MAIN_PATH}\\random_data'
CATEGORIES = ["20kmh", "30kmh", "50kmh", "60kmh", "70kmh", "80kmh", "100kmh", "120kmh", "droga_z_pierwszenstwem",
              "dzikie_zwierzeta", "gololedz", "koniec_80kmh", "koniec_zakazow", "koniec_zakazu_wyprzedzania",
              "koniec_zakazu_wyprzedzania_ciezarowe", "nakaz_lewo", "nakaz_lewo_prosto", "nakaz_na_lewo",
              "nakaz_na_prawo", "nakaz_prawo", "nakaz_prawo_prosto", "nakaz_prosto", "ostre_zakrety",
              "ostry_zakret_lewo", "ostry_zakret_prawo", "pierwszenstwo_przejazdu", "przejscie_dla_pieszych",
              "roboty_drogowe", "ruch_okrezny", "sliska_jezdnia", "stop", "swiatla", "ustap_pierwszenstwa", "uwaga",
              "uwaga_dzieci", "uwaga_rower", "wyboje", "zakaz_ciezarowe", "zakaz_ruchu", "zakaz_wjazdu",
              "zakaz_wyprzedzania", "zakaz_wyprzedzania_ciezarowe", "zwezenie_prawo"]

for category in CATEGORIES:
    path = os.path.join(DATADIR, category)
    for img in os.listdir(path):
        img_array = cv2.imread(os.path.join(path, img))
        break
    break

training_data = []

# Prepare data as a list of OpenCV elements
def create_training_data():
    for category in CATEGORIES:

        path = os.path.join(DATADIR, category)
        class_num = CATEGORIES.index(category)

        for img in tqdm(os.listdir(path)):
            try:
                img_array = cv2.imread(os.path.join(path, img))
                training_data.append([img_array, class_num])
            except Exception as e:
                pass

# Next steps in data preparing process
create_training_data()

print(len(training_data))

random.shuffle(training_data)

for sample in training_data[:10]:
    print(sample[1])

X = []
y = []

for features, label in training_data:
    X.append(features)
    y.append(label)

print(X[0].reshape(-1, 100, 100, 3))

X = np.array(X)
y = np.array(y)

# Model specification
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=4, restore_best_weights=True)
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
                        input_shape=(100, 100, 3)))  # , initializer='he'))#, input_shape=(100, 100, 3)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(256, (3, 3), activation='relu'))

model.add(layers.Flatten())
model.add(Dropout(0.1))
model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(43))  # , activation="softmax"))

model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

# Start of training process
history = model.fit(X, y, epochs=30, callbacks=[callback], validation_split=0.25)

# Save generated model
keras_model_path = f'{MAIN_PATH}\\model'
model.save(keras_model_path)


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from layers import *
from data import voc, coco
from torch.nn.parameter import Parameter
import os

#换做se模块，增加2层fpn融合，将deconv改为upsample，效果为0.760和baseline一样
#换做se模块，增加2层fpn融合，将尺度不变层改为1*1卷积实现，并且去掉smooth层，效果变差 0.743
######################################################
# 【通道显著性模块】 CBAM模块
class ChannelAttention(nn.Module):
    def __init__(self, inplanes, ratio=16):
        super(ChannelAttention, self).__init__()
        # 特征图先经过最大池化和平均池化 结果是1*1*通道数的tensor【最大池化，平均池化】
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.max_pool = nn.AdaptiveMaxPool2d(1)
        # 在经过全连接层先降低维度再升高维度，进行特征融合【MLP】
        self.fc1 = nn.Conv2d(inplanes, inplanes // 16, 1, bias=False)
        self.relu1 = nn.ReLU()
        self.fc2 = nn.Conv2d(inplanes // 16, inplanes, 1, bias=False)
        # 【激活层】
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x))))
        max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x))))
        out = avg_out + max_out  # 相加之后每个像素点的位置元素相加
        return self.sigmoid(out)

# 【空间显著性模块】
class SpatialAttention(nn.Module):
    def __init__(self, kernel_size=7):
        super(SpatialAttention, self).__init__()
        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'  # 这里设定kernal_size必须是3,7
        padding = 3 if kernel_size == 7 else 1

        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        avg_out = torch.mean(x, dim=1, keepdim=True)
        max_out, _ = torch.max(x, dim=1, keepdim=True)  # 会返回结果元素的值 和 对应的位置index
        x = torch.cat([avg_out, max_out], dim=1)
        x = self.conv1(x)
        return self.sigmoid(x)

# 【Bottleneck将特征图先经过 通道显著性模块，再经过 空间显著性模块】
class Bottleneck(nn.Module):  # 将通道显著性和空间显著性模块相连接
    def __init__(self, inplanes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.ca = ChannelAttention(inplanes)
        self.sa = SpatialAttention()
        self.stride = stride
        self.relu = nn.ReLU(inplace=True)

    def forward(self, x):
        save = x  # 先将原本的特征图保存下来
        out = self.ca(x) * x  # 先经过通道显著性模块
        out = self.sa(out) * out  # 再经过空间显著性模块
        out += save  ###相加
        out = self.relu(out)  # 最后再经过relu激活函数
        return out  # 输出结果尺寸不变，但是通道数变成了【planes * 4】这就是残差模块

######################################################
#SE 模块
class SEModule(nn.Module):
    def __init__(self, channel, reduction=16):
        super(SEModule, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Sequential(
            nn.Linear(channel, channel // reduction, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(channel // reduction, channel, bias=False),
            nn.Sigmoid()
        )

    def forward(self, x):
        b, c, _, _ = x.size()
        y = self.avg_pool(x).view(b, c)
        y = self.fc(y).view(b, c, 1, 1)
        return x * y.expand_as(x)

#####################################################
#ECA模块
class ECAModule(nn.Module):
    """Constructs a ECA module.
    Args:
        channel: Number of channels of the input feature map
        k_size: Adaptive selection of kernel size
    """
    def __init__(self, channel, k_size=3):
        super(ECAModule, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.conv = nn.Conv1d(1, 1, kernel_size=k_size, padding=(k_size - 1) // 2, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        # x: input features with shape [b, c, h, w]
        b, c, h, w = x.size()

        # feature descriptor on the global spatial information
        y = self.avg_pool(x)

        # Two different branches of ECA module
        y = self.conv(y.squeeze(-1).transpose(-1, -2)).transpose(-1, -2).unsqueeze(-1)

        # Multi-scale information fusion
        y = self.sigmoid(y)

        return x * y.expand_as(x)

#####################################################
##Upsample---->deconvd

class Upsample(nn.Module):
    """ nn.Upsample is deprecated

    The input dimensions are interpreted in the form:
    `mini-batch x channels x [optional depth] x [optional height] x width`.
    The modes available for upsampling are: `nearest`, `linear` (3D-only),
    `bilinear` (4D-only), `trilinear` (5D-only)
    Args:
        input (Tensor): the input tensor
        size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]):
            output spatial size.
        scale_factor (int): multiplier for spatial size. Has to be an integer.
        mode (string): algorithm used for upsampling:
            'nearest' | 'linear' | 'bilinear' | 'trilinear'. Default: 'nearest'
        align_corners (bool, optional): if True, the corner pixels of the input
            and output tensors are aligned, and thus preserving the values at
            those pixels. This only has effect when :attr:`mode` is `linear`,
            `bilinear`, or `trilinear`. Default: False
    warning::
        With ``align_corners = True``, the linearly interpolating modes
        (`linear`, `bilinear`, and `trilinear`) don't proportionally align the
        output and input pixels, and thus the output values can depend on the
        input size. This was the default behavior for these modes up to version
        0.3.1. Since then, the default behavior is ``align_corners = False``.
        See :class:`~torch.nn.Upsample` for concrete examples on how this
        affects the outputs.
    """

    def __init__(self, size, scale_factor=None, mode="nearest"): #bilinear，nearest
        super(Upsample, self).__init__()
        self.size = size
        self.scale_factor = scale_factor
        self.mode = mode

    def forward(self, x):
        x = F.interpolate(x, size=self.size, scale_factor=self.scale_factor, mode=self.mode)
        return x

#############################【SSD中融合特征显著性模块CBAM】######################
class SSD(nn.Module):
    """Single Shot Multibox Architecture
    The network is composed of a base VGG network followed by the
    added multibox conv layers.  Each multibox layer branches into
        1) conv2d for class conf scores
        2) conv2d for localization predictions
        3) associated priorbox layer to produce default bounding
           boxes specific to the layer's feature map size.
    See: https://arxiv.org/pdf/1512.02325.pdf for more details.

    Args:
        phase: (string) Can be "test" or "train"
        size: input image size
        base: VGG16 layers for input, size of either 300 or 500
        extras: extra layers that feed to multibox loc and conf layers
        head: "multibox head" consists of loc and conf conv layers
    """

    def __init__(self, phase, size, base, extras, head, num_classes):
        super(SSD, self).__init__()
        self.phase = phase
        self.num_classes = num_classes
        self.cfg = voc
        self.priorbox = PriorBox(self.cfg)
        self.priors = Variable(self.priorbox.forward(), volatile=True)
        self.size = size

        # SSD network
        # 经过修改的vgg网络
        self.vgg = nn.ModuleList(base)
        # Layer learns to scale the l2 normalized features from conv4_3
        self.L2Norm = L2Norm(512, 20)
        self.extras = nn.ModuleList(extras)

        self.loc = nn.ModuleList(head[0])
        self.conf = nn.ModuleList(head[1])

        if phase == 'test':
            self.softmax = nn.Softmax(dim=-1)
            self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)

    # =====se_fpn新增==================
        # pool2到conv4_3  扩张卷积，尺度少一半
        self.DilationConv_128_128 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, padding=2, dilation=2,
                                              stride=2)
        # conv4_3到conv4_3  尺度不变
        self.conv_512_256 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=1, stride=1)
        # fc7 到 conv4_3    反卷积上采样，尺度大一倍
        #self.DeConv_1024_128 = nn.ConvTranspose2d(in_channels=1024, out_channels=128, kernel_size=2, stride=2)
        self.upsample_1024_1024 = Upsample(38)
        self.conv_1024_128 = nn.Conv2d(in_channels=1024, out_channels=128, kernel_size=1, stride=1)

        # conv4_3 到FC7  扩张卷积，尺度少一半
        self.DilationConv_512_256 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=3, padding=2, dilation=2,
                                              stride=2)
        # FC7到FC7 尺度不变
        self.conv_1024_512 = nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=1, stride=1)
        # conv8_2 到 FC7    反卷积上采样，尺度大一倍  10->19
        #self.DeConv_512_128 = nn.ConvTranspose2d(in_channels=512, out_channels=128, kernel_size=3, stride=2, padding=1)
        self.upsample_512_512 = Upsample(19)
        self.conv_512_256_fc7 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=1, stride=1)

        # conv5_3到conv8_2,扩张卷积，尺度少一半
        self.DilationConv_512_128_2 = nn.Conv2d(in_channels=512, out_channels=128, kernel_size=3, padding=2, dilation=2,
                                                stride=2)
        # conv8_2到conv8_2 尺度不变
        self.conv_512_256_2 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=1, stride=1)
        # conv9_2到conv8_2
        #self.DeConv_256_128_2 = nn.ConvTranspose2d(in_channels=256, out_channels=128, kernel_size=2, stride=2)
        self.upsample_256_256_2 = Upsample(10)
        self.conv_256_128_2 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=1, stride=1)

        # conv9_2到conv9_2 尺度不变
        self.conv_256_128_9_2 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=3, padding=1, stride=1)
        self.upsample_256_256_10_2 = Upsample(5)
        self.conv_256_128_9_2 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=1,stride=1)

        #conv10_2到conv10_2 尺度不变
        self.conv_256_128_10_2 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=3, padding=1, stride=1)
        self.upsample_256_256_11_2 = Upsample(3)
        self.conv_256_128_10_2 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=1, stride=1)

        # 平滑层
        # self.smooth = nn.Conv2d(512, 512, kernel_size=3, padding=1, stride=1)
        # self.smooth2 = nn.Conv2d(256, 256, kernel_size=3, padding=1, stride=1)
        # 通道数BN层的参数是输出通道数out_channels
        self.bn = nn.BatchNorm2d(128)
        self.bn1 = nn.BatchNorm2d(256)

        # CBAM模块【6个特征层：512 512 512 256 256 256 】
        # self.CBAM1 = Bottleneck(512)
        # self.CBAM2 = Bottleneck(512)
        # self.CBAM3 = Bottleneck(512)
        # self.CBAM4 = Bottleneck(256)
        # self.CBAM5 = Bottleneck(256)
        # self.CBAM6 = Bottleneck(256)

        # SE模块【6个特征层：512 512 512 256 256 256 】
        self.SE1 = SEModule(512)
        self.SE2 = SEModule(1024)
        self.SE3 = SEModule(512)
        self.SE4 = SEModule(256)
        self.SE5 = SEModule(256)
        self.SE6 = SEModule(256)

        # ECA模块【6个特征层：512 512 512 256 256 256 】
        # self.ECA1 = ECAModule(512)
        # self.ECA2 = ECAModule(512)
        # self.ECA3 = ECAModule(512)
        # self.ECA4 = ECAModule(256)
        # self.ECA5 = ECAModule(256)
        # self.ECA6 = ECAModule(256)

    def forward(self, x):
        """Applies network layers and ops on input image(s) x.

        Args:
            x: input image or batch of images. Shape: [batch,3,300,300].

        Return:
            Depending on phase:
            test:
                Variable(tensor) of output class label predictions,
                confidence score, and corresponding location predictions for
                each object detected. Shape: [batch,topk,7]

            train:
                list of concat outputs from:
                    1: confidence layers, Shape: [batch*num_priors,num_classes]
                    2: localization layers, Shape: [batch,num_priors*4]
                    3: priorbox layers, Shape: [2,num_priors*4]
        """
        sources = list()
        loc = list()
        conf = list()

        # 原论文中vgg的conv4_3，relu之后加入L2 Normalization正则化，然后保存feature map
        # apply vgg up to conv4_3 relu
        # 将vgg层的feature map保存
        # k的范围为0-22
        # =========开始保存 所需的所有中间信息

        # 保存pool2（pool下标从1开始）的结果
        # 经过maxpool，所以不需要L2Norm正则化
        for k in range(10):
            x = self.vgg[k](x)
        sources.append(x)

        # apply vgg up to conv4_3 relu
        for k in range(10, 23):
            x = self.vgg[k](x)

        s = self.L2Norm(x)
        sources.append(s)

        # apply vgg up to conv5_3 relu
        for k in range(23, 30):
            x = self.vgg[k](x)

        s = self.L2Norm(x)
        sources.append(s)

        # apply vgg up to fc7
        for k in range(30, len(self.vgg)):
            x = self.vgg[k](x)
        sources.append(x)

        #apply extra layers and cache source layer outputs
        for k, v in enumerate(self.extras):
            x = F.relu(v(x), inplace=True)
            if k % 2 == 1:
                sources.append(x)

        # 此时sources保存了所有中间结果，论文中的pool2、conv4_3、conv5_3、fc7、conv8_2、conv9_2、conv10_2、conv11_2
        # sources_final保存各层融合之后的最终结果
        sources_final = list()
        # con4_3层融合结果  self.bn1(self.conv1(x)) 在通道维度上融合
        # conv4_fp = torch.cat((F.relu(self.bn(self.DilationConv_128_128(sources[0])), inplace=True),
        #                       F.relu(self.conv_512_256(sources[1]), inplace=True),
        #                       F.relu(self.DeConv_1024_128(sources[3]), inplace=True)), 1)

        conv4_fp = torch.cat((F.relu(self.bn(self.DilationConv_128_128(sources[0])), inplace=True),
                              F.relu(self.conv_512_256(sources[1]), inplace=True),
                              F.relu(self.conv_1024_128(self.upsample_1024_1024(sources[3])), inplace=True)), 1)

        conv4_fp = F.relu(conv4_fp, inplace=True)
        sources_final.append(self.SE1(conv4_fp))
        # FC7层融合结果
        # fc7_fp = torch.cat((F.relu(self.bn(self.DilationConv_512_128(sources[1])), inplace=True),
        #                     F.relu(self.conv_1024_256(sources[3]), inplace=True),
        #                     F.relu(self.DeConv_512_128(sources[4]), inplace=True)), 1)

        fc7_fp = torch.cat((F.relu(self.bn1(self.DilationConv_512_256(sources[1])), inplace=True),
                            F.relu(self.conv_1024_512(sources[3]), inplace=True),
                            F.relu(self.conv_512_256_fc7(self.upsample_512_512(sources[4])), inplace=True)), 1)

        # sources_final.append(F.relu(self.smooth(fc7_fp) , inplace=True))
        fc7_fp = F.relu(fc7_fp, inplace=True)
        sources_final.append(self.SE2(fc7_fp))
        # conv8_2层融合结果
        # conv8_fp = torch.cat((F.relu(self.bn(self.DilationConv_512_128_2(sources[2])), inplace=True),
        #                       F.relu(self.conv_512_256_2(sources[4]), inplace=True),
        #                       F.relu(self.DeConv_256_128_2(sources[5]), inplace=True)), 1)

        conv8_fp = torch.cat((F.relu(self.bn(self.DilationConv_512_128_2(sources[2])), inplace=True),
                              F.relu(self.conv_512_256_2(sources[4]), inplace=True),
                              F.relu(self.conv_256_128_2(self.upsample_256_256_2(sources[5])), inplace=True)), 1)
        # sources_final.append(F.relu(self.smooth(conv8_fp) , inplace=True))
        conv8_fp = F.relu(conv8_fp, inplace=True)
        sources_final.append(self.SE3(conv8_fp))

        #conv9_2层融合
        conv9_fp = torch.cat((F.relu(self.conv_256_128_9_2 (sources[5]), inplace=True),
                              F.relu(self.conv_256_128_9_2(self.upsample_256_256_10_2(sources[6])), inplace=True)), 1)

        conv9_fp = F.relu(conv9_fp, inplace=True)
        sources_final.append(self.SE4(conv9_fp))

        #conv10_2层融合
        conv10_fp = torch.cat((F.relu(self.conv_256_128_10_2 (sources[6]), inplace=True),
                              F.relu(self.conv_256_128_10_2(self.upsample_256_256_11_2(sources[7])), inplace=True)), 1)
        conv9_fp = F.relu(conv10_fp, inplace=True)
        sources_final.append(self.SE5(conv10_fp))

        # conv11_2
        sources_final.append(self.SE6(sources[7]))

        # apply multibox head to source layers
        for (x, l, c) in zip(sources_final, self.loc, self.conf):
            loc.append(l(x).permute(0, 2, 3, 1).contiguous())
            conf.append(c(x).permute(0, 2, 3, 1).contiguous())

        loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
        conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
        if self.phase == "test":
            output = self.detect(
                loc.view(loc.size(0), -1, 4),                  #loc preds
                self.softmax(conf.view(conf.size(0), -1,
                             self.num_classes)),               #conf preds
                self.priors.type(type(x.data))                 #default boxes
            )
        else:
            output = (
                loc.view(loc.size(0), -1, 4),
                conf.view(conf.size(0), -1, self.num_classes),
                self.priors
            )
        return output

    def load_weights(self, base_file):
        _, ext = os.path.splitext(base_file)
        if ext == '.pkl' or '.pth':
            print('Loading weights into state dict...')
            self.load_state_dict(torch.load(base_file,
                                 map_location=lambda storage, loc: storage))
            print('Finished!')
        else:
            print('Sorry only .pth and .pkl files supported.')

# This function is derived from torchvision VGG make_layers()
# https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py
def vgg(cfg, i, batch_norm=False):
    layers = []
    in_channels = i
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        elif v == 'C':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2, ceil_mode=True)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    pool5 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
    conv6 = nn.Conv2d(512, 1024, kernel_size=3, padding=6, dilation=6)
    conv7 = nn.Conv2d(1024, 1024, kernel_size=1)
    layers += [pool5, conv6,
               nn.ReLU(inplace=True), conv7, nn.ReLU(inplace=True)]
    return layers

def add_extras(cfg, i, batch_norm=False):
    # Extra layers added to VGG for feature scaling
    layers = []
    in_channels = i
    flag = False
    for k, v in enumerate(cfg):
        if in_channels != 'S':
            if v == 'S':
                layers += [nn.Conv2d(in_channels, cfg[k + 1],
                           kernel_size=(1, 3)[flag], stride=2, padding=1)]
            else:
                layers += [nn.Conv2d(in_channels, v, kernel_size=(1, 3)[flag])]
            flag = not flag
        in_channels = v
    return layers

def multibox(vgg, extra_layers, cfg, num_classes):
    loc_layers = []
    conf_layers = []
    # 传入的修改过的vgg网络用于预测的网络是21层以及倒数第二层
    vgg_source = [21, -2]
    for k, v in enumerate(vgg_source):
        # 按照fp-ssd论文，将1024改为512通道
        if k == 1:
            in_channels = 1024
        else:
            in_channels = vgg[v].out_channels

        loc_layers += [nn.Conv2d(in_channels,
                                 cfg[k] * 4, kernel_size=3, padding=1)]
        conf_layers += [nn.Conv2d(in_channels,
                        cfg[k] * num_classes, kernel_size=3, padding=1)]
    # [x::y] 从下标x开始，每隔y取值
    # 论文中新增层也是每隔一个层添加一个预测层
    # 将新增的额外层中的预测层也添加上   start=2：下标起始位置
    for k, v in enumerate(extra_layers[1::2], 2):
        loc_layers += [nn.Conv2d(v.out_channels, cfg[k]
                                 * 4, kernel_size=3, padding=1)]
        conf_layers += [nn.Conv2d(v.out_channels, cfg[k]
                                  * num_classes, kernel_size=3, padding=1)]
    return vgg, extra_layers, (loc_layers, conf_layers)

base = {
    '300': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'C', 512, 512, 512, 'M',
            512, 512, 512],
    '512': [],
}
extras = {
    '300': [256, 'S', 512, 128, 'S', 256, 128, 256, 128, 256],
    '512': [],
}
mbox = {
    '300': [4, 6, 6, 6, 4, 4],  # number of boxes per feature map location
    '512': [],
}

def build_ssd(phase, size=300, num_classes=2):
    if phase != "test" and phase != "train":
        print("ERROR: Phase: " + phase + " not recognized")
        return
    if size != 300:
        print("ERROR: You specified size " + repr(size) + ". However, " +
              "currently only SSD300 (size=300) is supported!")
        return
    base_, extras_, head_ = multibox(vgg(base[str(size)], 3),
                                     add_extras(extras[str(size)], 1024),
                                     mbox[str(size)], num_classes)
    return SSD(phase, size, base_, extras_, head_, num_classes)


import allure
from model.contact import Contact
import random

def test_delete_some_contact(app, db, check_ui):
    # добавление проверки: если нет ни одного контакта, то перед удалением создаем
    if len(db.get_contact_list()) == 0:
        app.contact.create(Contact(firstname="John"))
    with allure.step("Given non empty contact list"):
        old_contacts = db.get_contact_list()
    with allure.step("Given a random contact from non empty contact list"):
        contact = random.choice(old_contacts)
    with allure.step("When I delete the contact {} from the list".format(contact)):
        app.contact.delete_contact_by_id(contact.id)
    with allure.step("Then the new contact list is equal to the old contact list without deleted contact"):
        new_contacts = db.get_contact_list()
        old_contacts.remove(contact)
        assert old_contacts == new_contacts
        if check_ui:
            assert sorted(new_contacts, key=Contact.id_or_max) == sorted(app.contact.get_contact_list(), key=Contact.id_or_max)


from models.mhop_retriever import MhopRetriever

import faiss
import numpy as np
import torch
from tqdm import tqdm
from transformers import AutoConfig, AutoTokenizer
import json
import logging
import argparse
from .utils.utils import (load_saved, move_to_cuda)

parser = argparse.ArgumentParser()
parser.add_argument('--topk', type=int, default=2, help="topk paths")
parser.add_argument('--num-workers', type=int, default=10)
parser.add_argument('--max-q-len', type=int, default=70)
parser.add_argument('--max-c-len', type=int, default=300)
parser.add_argument('--max-q-sp-len', type=int, default=350)
parser.add_argument('--model-name', type=str, default='bert-base-uncased')
parser.add_argument('--gpu', action="store_true")
parser.add_argument('--shared-encoder', action="store_true")
parser.add_argument("--stop-drop", default=0, type=float)
args = parser.parse_args()

index_path = "index/abstracts_v0_fixed.npy"
corpus_path = "index/abstracts_id2doc.json"
model_path = "logs/08-05-2020/baseline_v0_fixed-seed16-bsz150-fp16True-lr2e-05-decay0.0-warm0.1-valbsz3000-sharedTrue-multi1-schemenone/checkpoint_best.pt"

print(f"Loading corpus and index...")
id2doc = json.load(open(corpus_path))
index_vectors = np.load(index_path).astype('float32')

index = faiss.IndexFlatIP(768)
index.add(index_vectors)
res = faiss.StandardGpuResources()
index = faiss.index_cpu_to_gpu(res, 1, index)

print(f"Loading retrieval model...")
bert_config = AutoConfig.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = MhopRetriever(bert_config, args)
model = load_saved(model, args.model_path, exact=False)

cuda = torch.device('cuda')
model.to(cuda)
from apex import amp
model = amp.initialize(model, opt_level='O1')
model.eval()

while True:
    question = input("Type Question:")
    question = "the Danish musicians who died in 1931"
    batch_q_encodes = tokenizer.batch_encode_plus(["question"], max_length=args.max_q_len, pad_to_max_length=True, return_tensors="pt")
    batch_q_encodes = move_to_cuda(dict(batch_q_encodes))
    q_embeds = model.encode_q(batch_q_encodes["input_ids"], batch_q_encodes["attention_mask"], batch_q_encodes.get("token_type_ids", None))
    q_embeds_numpy = q_embeds.cpu().contiguous().numpy()
    D, I = index.search(q_embeds_numpy, 1)

    print(I)


from .. import db

class MonitoringPhoto(db.Model):
    """ Monitoring Photo Model for storing feedback photo related details """
    __tablename__ = "monitoring_photo"

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    buildings_id = db.Column(db.Integer, db.ForeignKey('buildings.id', ondelete='CASCADE'), nullable=False)
    buildings = db.relationship('Buildings', backref=db.backref('buildings_photo', lazy='dynamic'))
    timestamp = db.Column(db.DateTime, nullable=False)
    photo_file = db.Column(db.String(100), nullable=False)
    contributor = db.Column(db.String(50), nullable=False)
    verified_on = db.Column(db.DateTime, nullable=True)
    validator = db.Column(db.String(50), nullable=True)
    reason = db.Column(db.String(50), nullable=True)
    status_id = db.Column(db.Integer, db.ForeignKey('status.id', ondelete='CASCADE'), nullable=False)
    status = db.relationship('Status', backref=db.backref('status_photo', lazy='dynamic'))

    def as_dict(self):
       return {c.name: getattr(self, c.name) for c in self.__table__.columns if c.name != 'user_id'}


import streamlit as st
import numpy as np
import pandas as pd

#from st_aggrid import AgGrid

# import matplotlib.pyplot as plt
# import matplotlib
# matplotlib.use('Agg')       #서버에서, 화면에 표시하기 위해서 필요
# import seaborn as sns
# import altair as alt               ##https://altair-viz.github.io/
# import plotly.express as px

st.write("Hello, Streamlit!")
st.write("------------------------------------------------------------------")
st.write("                      Project 1. CSV Plotting                     ")
st.write("------------------------------------------------------------------")
st.write("                 1. 파일업로드                                    ")
st.write("                 2. CSV → Pandas df                              ")
st.write("                 3. df ploitting                                  ")
st.write("                 4. multiselect → df[’col’] 컬럼명             ")
st.write("                 5. 선택된 컬럼만 표시                            ")
st.write("------------------------------------------------------------------")

uploaded_file = st.file_uploader("Choose a file")

if uploaded_file is not None:
#      # To read file as bytes:
#      bytes_data = uploaded_file.getvalue()
#      st.write(bytes_data)

#      # To convert to a string based IO:
#      stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
#      st.write(stringio)

#      # To read file as string:
#      string_data = stringio.read()
#      st.write(string_data)

     # Can be used wherever a "file-like" object is accepted:
    df_test = pd.read_csv(uploaded_file)
#    st.write(dataframe)

    AgGrid(df_test)

    df_columns = df_test.columns
    st.write(df_columns)

    options = st.multiselect(
         'What are your columns',
         df_columns)

    # 슬라이스

    from datetime import datetime

    start_time = st.slider(
         "When do you start?",
        value=datetime(2020, 1, 1, 9, 30),
        format="YYYY")
    st.write("Start time:", start_time)

    if st.button("View Data"):
        st.write("Data Loading..")

        #AgGrid(dataframe)
        st.write(df_test[options])

        import matplotlib.pyplot as plt
        import numpy as np

        df_chart = df_test[options]
        st.write("----")

        fig, ax = plt.subplots()
        ax.hist(df_chart)

        st.pyplot(fig)
# checkbox_btn = st.checkbox('Checktbox Button')

# if checkbox_btn:
#       st.write('Great!')

# checkbox_btn2 = st.checkbox('Checktbox Button2', value=True)

# if checkbox_btn2:
#       st.write('Button2')

# values = st.slider('Select a range of values', 0.0, 100.0, (25.0, 75.0))
# st.write('Values:', values)


"""Test SQLAlchemy backend."""
import datetime
import decimal
import logging
import unittest

import src.sensor_net
import src.sensor_net.backend_driver.sqlalchemy
import src.sensor_net.backend_driver.sqlalchemy.schema

class TestSQLAlchemyBackendWorking(unittest.TestCase):
    """Tests SQLAlchemy backend."""
    def setUp(self) -> None:
        """
        Prepare the test case.

        -> Create a SQLite database
        -> Create tables
        -> Create SQLLite backend configuration
        -> Retrieve SQLAlchemy backend connected to SQLLite
        """
        logging.basicConfig(level=logging.DEBUG)

        self.driver = src.sensor_net.backend_driver.sqlalchemy.get_driver(
            "test_daemon",
            {"url": "sqlite:///:memory:"},
        )

        self.schema = src.sensor_net.backend_driver.sqlalchemy.schema.Schema()
        self.schema.setup(self.driver.engine)

    def test_write_success(self):
        """Test write to SQLAlchemy. Data should be written to the database."""
        self.driver.write(
            network_name="test_net",
            network_prefix="tst",
            sensor_address="0.0.0.0",
            data=[src.sensor_net.SensorData(
                "heartbeat",
                123,
                datetime.datetime(2022, 1, 2, 3, 4, 5)  # 2022-01-02, 03:04:05
            )],
        )

        with self.driver.engine.connect() as conn:
            results = conn.execute(self.driver.table.select()).all()
            self.assertEqual(len(results), 1, "There should be only one result.")
            self.assertEqual(
                results[0][:-1],  # Remove d_created_utc to avoid patching datetime.utcnow
                (
                    "test_daemon",
                    "tst0.0.0.0",
                    "test_net",
                    "heartbeat",
                    datetime.datetime(2022, 1, 2, 3, 4, 5),
                    decimal.Decimal(123.0),
                ),
                "Data are not as expected."
            )

class TestSQLAlchemyBackendIssues(unittest.TestCase):
    """Test with incomplete Database setup."""

    def setUp(self) -> None:
        """Backend driver."""
        self.driver = src.sensor_net.backend_driver.sqlalchemy.get_driver(
            "failed_tests",
            {"url": "sqlite:///:memory:"},
        )

    def test_write_failed(self):
        """Test write error accessing to SQLAlchemy. Should raise a BackendWriteError."""
        self.assertRaises(
            src.sensor_net.backend_driver.sqlalchemy.BackendWriteError,
            self.driver.write,
            network_name="test_net",
            network_prefix="tst",
            sensor_address="0.0.0.0",
            data=[src.sensor_net.SensorData(
                "heartbeat",
                123.456,
                datetime.datetime(2022, 1, 2, 3, 4, 5)  # 2022-01-02, 03:04:05
            )],
        )

if __name__ == '__main__':
    unittest.main()


#
# First lab using neural networks
# This training is intended to be my notes around training.
# I don't take any credit for making this lab.
#
#
# Goal: Train the MNIST dataset (handwritten data), and
# train an ML model using Tensorflow for predicting the value of a
# handwritten letter or number.
#
# Taken from: https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb

import tensorflow as tf

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

# Keras sequential model
# Stack layers, choose optimizer and loss function for training:

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10)
])

predictions = model(x_train[:1]).numpy()

# Review the predictions
# Each of  this represent a vector of "logits" or "log-odds" scores, one per class
print("Predictions for the trained moel:")
print(predictions)

# Convert these logits to probabilities
tf.nn.softmax(predictions).numpy()

loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

model.compile(optimizer='adam',
              loss=loss_fn,
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=5)

model.evaluate(x_test,  y_test, verbose=2)

probability_model = tf.keras.Sequential([
  model,
  tf.keras.layers.Softmax()
])

probability_model(x_test[:5])


import os
import pickle

import feedforward as NN
import gymnasium as gym
import memory as mem
import numpy as np
import torch
from Actor import Actor
from Agent import UnsupportedSpace, agent
from Critic import Critic_Q
from gymnasium import spaces

#device = torch.device('cpu')
#https://cloud.cs.uni-tuebingen.de/index.php/s/pm49B2xRpcNirry
#https://arxiv.org/pdf/2010.09163.pdf
#https://github.com/BY571/Soft-Actor-Critic-and-Extensions
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using device:', device)

class UnsupportedSpace(Exception):
    """Exception raised when the Sensor or Action space are not compatible
    """
    def __init__(self, message="Unsupported Space"):
        self.message = message
        super().__init__(self.message)

class SAC_Agent(agent):
    def __init__(self,observation_space,action_space, **userconfig):

        super().__init__(observation_space,action_space,**userconfig)

        self._config = {
            "start_steps":10000,
            "discount": 0.99,
            "buffer_size": int(1e6),
            "batch_size": 256,
            "lr_actor": float(3e-4),
            "lr_critic": float(1e-3),
            "lr_value": float(1e-3),
            "hidden_size_critic": [256,256],
            "hidden_size_actor": [256,256],
            "hidden_size_value": [256,256],
            "frequency_update_Q":1,
            "frequency_update_actor":1,
            "frequency_update_targets":1,
            "tau": 0.005,
            "update_target_every":1,
            "autotuned_temperature":True,
            "temperature":0.1,
            "numbers_critics":5,
            "use_smooth_L1":False,
            "play_hockey":True
            }

        self.device = device
        self._observation_space = observation_space
        self._obs_dim = self._observation_space.shape[0]
        self._action_space = action_space
        if self._config["play_hockey"]:
            self.action_dim = int(action_space.shape[0]/2)
            print(f"halfed the action space to {self.action_dim}")
        else:
            self.action_dim = action_space.shape[0]
            print(f"normal actionsspace of {self.action_dim}")
        self.discount = self._config["discount"]
        self.tau = self._config["tau"]
        self.train_iter = 0
        self.eval_mode = False
        self.start_steps = self._config["start_steps"]

        self.memory = mem.Memory(max_size=self._config["buffer_size"],state_dim=self._obs_dim,action_dim=self.action_dim,device=self.device)

        if self._config["autotuned_temperature"]:
            self.target_entropy = -torch.Tensor(self.action_dim).to(self.device)
            self.log_temperature = torch.zeros(1,requires_grad=True,device=self.device)
            self.temperature_optimizer = torch.optim.Adam([self.log_temperature],lr=self._config["lr_critic"])
        else:
            self.log_temperature = torch.Tensor(self._config["temperature"].log()).to(self.device)

        self.actor = Actor(self._obs_dim,self.action_dim,hidden_sizes=self._config["hidden_size_actor"],
                            learning_rate=self._config["lr_actor"],device=self.device)
        self.critic = Critic_Q(self._obs_dim,self.action_dim,self._config["lr_critic"],
                               hidden_sizes=self._config["hidden_size_critic"],
                               network_number=self._config["numbers_critics"],device=self.device)

        self.target = Critic_Q(self._obs_dim,self.action_dim,self._config["lr_critic"],
                            hidden_sizes=self._config["hidden_size_critic"],tau=self.tau,target=True,
                            network_number=self._config["numbers_critics"],device=self.device)

        self.target.soft_update(self.critic,tau=1)

        if self._config["play_hockey"]:
            self.action_scale = torch.FloatTensor((action_space.high[:self.action_dim] - action_space.low[:self.action_dim])/2).to(self.device)
            self.action_bias = torch.FloatTensor((action_space.high[:self.action_dim] + action_space.low[:self.action_dim])/2).to(self.device)
        else:
            if action_space is not None:
                self.action_scale = torch.FloatTensor((action_space.high - action_space.low)/2).to(self.device)
                self.action_bias = torch.FloatTensor((action_space.high + action_space.low)/2).to(self.device)
            else:
                self.action_scale = torch.tensor(1.).to(self.device)
                self.action_bias = torch.tensor(0.).to(self.device)

    def store_transition(self,transition):
        self.memory.add_transition(transition)

    def get_networks_states(self):
        return (self.actor.state_dict(),self.critic.get_network_states(),self.target.get_network_states())

    def load_network_states(self,state):
        self.actor.load_state_dict(state[0])
        self.critic.load_network_states(state[1])
        self.target.load_network_states(state[2])

    def load_network_states_from_file(self,filename,trained_on_cuda=False):
        if trained_on_cuda:
            self.load_network_states(torch.load(filename, map_location=torch.device('cpu')))
        else:
            self.load_network_states(torch.load(filename))

    def reset(self):
        raise NotImplementedError("U might wanna reset the policy network for faster learning as in DR3")

    def eval(self):
        self.eval_mode = True
        print("Agent now in evaluation Mode")

    def train(self):
        self.eval_mode = False
        print("Agent now in training mode")

    def rescale_action(self,action):
        return action[0]*self.action_scale + self.action_bias

    def act(self,state):
        state = torch.FloatTensor(state).to(self.device)[None,:]
        if self.eval_mode:
            action = self.actor.get_action(state)
        else:
            if self.start_steps> self.memory.size:
                action = self.actor.random_action()
            else:
                action, _ = self.actor.get_action_and_log_probs(state)
        action = self.rescale_action(action)
        return action.cpu().detach().numpy()

    def remote_act(self,state):
        state = torch.FloatTensor(state).to(self.device)[None,:]
        action = self.actor.get_action(state)
        acion = self.rescale_action(action)
        return action.cpu().detach().numpy()

    def get_Q_value(self,state,action):
        Q_val = self.critic.get_min_Q_value(state,action)
        return Q_val

    def get_target_Q_value(self,state,action):
        #same as get_Q_value but from target networks
        target_Q_val = self.target.get_min_Q_value(state,action)
        return target_Q_val

    def update_Q_functions(self,s0,action,done,rew,s1):
        with torch.no_grad():
            a_next , log_prob_next = self.actor.get_action_and_log_probs(s1,reparameterize=False)
            min_Q_next = self.get_target_Q_value(s1,a_next)
            #get V estimate
            target_value = min_Q_next - self.log_temperature.exp() * log_prob_next
            y = (rew + self.discount * (1 - done)*target_value)

        q_loss = self.critic.update_critics(state=s0,action=action,target=y)
        return q_loss

    def update_policy(self,s0):
        action, log_prob = self.actor.get_action_and_log_probs(s0,reparameterize=True)
        actor_Q = self.get_Q_value(s0,action)
        actor_loss = (-actor_Q+self.log_temperature.exp().detach()*log_prob).mean(axis=0)
        self.actor.optimizer.zero_grad()
        actor_loss.backward()
        self.actor.optimizer.step()
        return actor_loss.item(), log_prob

    def update_temperature(self,log_probs):
        self.temperature_optimizer.zero_grad()
        temperature_loss  =  -(self.log_temperature.exp() *
                               (log_probs+ self.target_entropy).detach()).mean()
        temperature_loss.backward()
        self.temperature_optimizer.step()
        return temperature_loss.item()

    def train(self,iter_fit=32):
        q_losses = []
        policy_losses = []
        temperature_losses=[]

        self.train_iter +=1

        for i in range(iter_fit):
            #print(i)
            if self.memory.size > self._config["batch_size"]:
                #Sample Batches from the replay Buffer
                data=self.memory.sample(batch=self._config["batch_size"])
                s0,a,rew,s1,done=data

                ######Start SAC train loop#######
                #updateQ
                if i % self._config["frequency_update_Q"] == 0:
                    q_loss = self.update_Q_functions(s0,a,done,rew,s1)
                    q_losses.append(q_loss)

                #update policy
                if i % self._config["frequency_update_actor"] == 0:
                    actor_loss,log_prob = self.update_policy(s0)
                    policy_losses.append(actor_loss)

                #Update temperature
                if self._config["autotuned_temperature"]:
                    temperature_loss = self.update_temperature(log_prob)
                else:
                    temperature_loss = torch.tensor(0.)
                temperature_losses.append(temperature_loss)

                #Update targets networks
                if i % self._config["frequency_update_targets"] == 0:
                    self.target.soft_update(self.critic)

        return q_losses,policy_losses,temperature_losses