import pygame
import sys
from moviepy.editor import VideoFileClip
from PIL import Image

pygame.init()

music_background = pygame.mixer.music.load("assets/LostCompanionTomboFry.mp3")
pygame.mixer.music.play()
pygame.mixer.music.set_volume(0.2)

lar = 550
hut = 700

screen = pygame.display.set_mode((lar, hut))
pygame.display.set_caption("Menu")

gif_path = "assets/bg.gif"
clip = VideoFileClip(gif_path)
fps = clip.fps

frames = []
for t in range(0, int(clip.duration * fps)):
    frame = clip.get_frame(t / fps)
    pil_image = Image.fromarray((frame * 255).astype('uint8'))
    pil_image = pil_image.resize((lar, hut))
    pygame_image = pygame.image.fromstring(pil_image.tobytes(), pil_image.size, pil_image.mode)
    frames.append(pygame_image)

# Carregar recursos do menu antecipadamente
fonte = pygame.font.Font(None, 30)
texto_play = fonte.render("Play", True, (0, 0, 0))
texto_quit = fonte.render("Quit", True, (0, 0, 0))
Title = fonte.render("Pythongoras-Game", True, (255, 255, 255))

def mostrar_menu():
    frame_index = 0
    clock = pygame.time.Clock()

    while True:
        for evento in pygame.event.get():
            if evento.type == pygame.QUIT:
                pygame.quit()
                sys.exit()
            elif evento.type == pygame.MOUSEBUTTONDOWN:
                if batom_play.collidepoint(evento.pos):
                    pygame.time.delay(100)
                    iniciar_jogo()
                elif batom_quit.collidepoint(evento.pos):
                    pygame.quit()
                    sys.exit()

        screen.blit(frames[frame_index], (0, 0))

        batom_Title = pygame.Rect(190, 100 + 50, 150, 50)
        pos_text_Title = Title.get_rect(center=batom_Title.center)
        screen.blit(Title, pos_text_Title)

        batom_play = pygame.Rect(lar/2 - 75, hut/2 + 50, 150, 50)
        pygame.draw.rect(screen, (255, 255, 255), batom_play)
        pos_text_play = texto_play.get_rect(center=batom_play.center)
        screen.blit(texto_play, pos_text_play)

        if batom_play.collidepoint(pygame.mouse.get_pos()):
            pygame.draw.rect(screen, (200, 200, 200), batom_play)

        batom_quit = pygame.Rect(lar/2 - 75, hut/2 + 140, 150, 50)
        pygame.draw.rect(screen, (255, 255, 255), batom_quit)
        pos_text_quit = texto_quit.get_rect(center=batom_quit.center)
        screen.blit(texto_quit, pos_text_quit)

        if batom_quit.collidepoint(pygame.mouse.get_pos()):
            pygame.draw.rect(screen, (200, 200, 200), batom_quit)

        pygame.display.flip()

        frame_index = (frame_index + 1) % len(frames)

        clock.tick(fps)

def iniciar_jogo():
    print("O jogo começou!")
    import Chose

mostrar_menu()


import asyncio
import collections
import contextlib
import datetime
import functools
import io
import multiprocessing
import multiprocessing.pool
import os
import signal
import tempfile

from aiohttp import web
import marshmallow

from oslo_config import cfg
from oslo_log import log

LOG = log.getLogger(__name__)

CONF = cfg.CONF

UploadedFile = collections.namedtuple(
    "UploadedFile", ("name", "filename", "content_type", "original_filename")
)
"""Class to hold uploaded field metadata when passed to model's methods

.. py:attribute:: name

   Name of the argument where this file is being sent.

.. py:attribute:: filename

   Complete file path to the temporary file in the filesystem,

.. py:attribute:: content_type

   Content-type of the uploaded file

.. py:attribute:: original_filename

   Filename of the original file being uploaded.
"""

ReturnedFile = collections.namedtuple(
    "ReturnedFile", ("name", "filename", "content_type", "original_filename")
)
"""Class to pass the files returned from predict in a pickable way

.. py:attribute:: name

   Name of the argument where this file is being sent.

.. py:attribute:: filename

   Complete file path to the temporary file in the filesystem,

.. py:attribute:: content_type

   Content-type of the uploaded file

.. py:attribute:: original_filename

   Filename of the original file being uploaded.
"""

# set defaults to None, mainly for compatibility (vkoz)
UploadedFile.__new__.__defaults__ = (None, None, None, None)
ReturnedFile.__new__.__defaults__ = (None, None, None, None)

class ModelWrapper(object):
    """Class that will wrap the loaded models before exposing them.

    Whenever a model is loaded it will be wrapped with this class to create a
    wrapper object that will handle the calls to the model's methods so as to
    handle non-existent method exceptions.

    :param name: Model name
    :param model: Model object
    :raises HTTPInternalServerError: in case that a model has defined
        a response schema that is not JSON schema valid (DRAFT 4)
    """

    def __init__(self, name, model_obj, app=None):
        self.name = name
        self.model_obj = model_obj
        self._app = app

        self._loop = asyncio.get_event_loop()

        self._workers = CONF.workers
        self._executor = self._init_executor()

        if self._app is not None:
            self._setup_cleanup()

        schema = getattr(self.model_obj, "schema", None)

        if isinstance(schema, dict):
            try:
                schema = marshmallow.Schema.from_dict(
                    schema, name="ModelPredictionResponse"
                )
                self.has_schema = True
            except Exception as e:
                LOG.exception(e)
                raise web.HTTPInternalServerError(
                    reason=("Model defined schema is invalid, " "check server logs.")
                )
        elif schema is not None:
            try:
                if issubclass(schema, marshmallow.Schema):
                    self.has_schema = True
            except TypeError:
                raise web.HTTPInternalServerError(
                    reason=("Model defined schema is invalid, " "check server logs.")
                )
        else:
            self.has_schema = False

        self.response_schema = schema

    def _setup_cleanup(self):
        self._app.on_cleanup.append(self._close_executors)

    async def _close_executors(self, app):
        self._executor.shutdown()

    def _init_executor(self):
        n = self._workers
        executor = CancellablePool(max_workers=n)
        return executor

    @contextlib.contextmanager
    def _catch_error(self):
        name = self.name
        try:
            yield
        except AttributeError:
            raise web.HTTPNotImplemented(
                reason=("Not implemented by underlying model (loaded '%s')" % name)
            )
        except NotImplementedError:
            raise web.HTTPNotImplemented(
                reason=("Model '%s' does not implement this functionality" % name)
            )
        except Exception as e:
            LOG.error(
                "An exception has happened when calling method on " "'%s' model." % name
            )
            LOG.exception(e)
            if isinstance(e, web.HTTPException):
                raise e
            else:
                raise web.HTTPInternalServerError(reason=e)

    def validate_response(self, response):
        """Validate a response against the model's response schema, if set.

        If the wrapped model has defined a ``response`` attribute we will
        validate the response that

        :param response: The response that will be validated.
        :raises exceptions.InternalServerError: in case the reponse cannot be
            validated.
        """
        if self.has_schema is not True:
            raise web.HTTPInternalServerError(
                reason=(
                    "Trying to validate against a schema, but I do not "
                    "have one defined"
                )
            )

        try:
            self.response_schema().load(response)
        except marshmallow.ValidationError as e:
            LOG.exception(e)
            raise web.HTTPInternalServerError(
                reason="ERROR validating model response, check server logs."
            )
        except Exception as e:
            LOG.exception(e)
            raise web.HTTPInternalServerError(
                reason="Unknown ERROR validating response, check server logs."
            )

        return True

    def get_metadata(self):
        """Obtain model's metadata.

        If the model's metadata cannot be obtained because it is not
        implemented, we will provide some generic information so that the
        call does not fail.

        :returns dict: dictionary containing model's metadata
        """
        try:
            d = self.model_obj.get_metadata()
        except (NotImplementedError, AttributeError):
            d = {
                "id": "0",
                "name": self.name,
                "description": (
                    "Could not load description from "
                    "underlying model (loaded '%s')" % self.name
                ),
            }
        return d

    def _run_in_pool(self, func, *args, **kwargs):
        fn = functools.partial(func, *args, **kwargs)
        ret = self._loop.create_task(self._executor.apply(fn))
        return ret

    async def warm(self):
        """Warm (i.e. load, initialize) the underlying model.

        This method is called automatically when the model is loaded. You
        should use this method to initialize the model so that it is ready for
        the first prediction.

        The model receives no arguments.
        """
        try:
            func = self.model_obj.warm
        except AttributeError:
            LOG.debug("Cannot warm (initialize) model '%s'" % self.name)
            return

        try:
            n = self._workers
            LOG.debug("Warming '%s' model with %s workers" % (self.name, n))
            fs = [self._run_in_pool(func) for _ in range(0, n)]
            await asyncio.gather(*fs)
            LOG.debug("Model '%s' has been warmed" % self.name)
        except NotImplementedError:
            LOG.debug("Cannot warm (initialize) model '%s'" % self.name)

    @staticmethod
    def predict_wrap(predict_func, *args, **kwargs):
        """Wrapper function to allow returning files from predict
        This wrapper exists because buffer objects are not pickable,
        thus cannot be returned from the executor.
        """
        ret = predict_func(*args, **kwargs)
        if isinstance(ret, io.BufferedReader):
            ret = ReturnedFile(filename=ret.name)

        return ret

    def predict(self, *args, **kwargs):
        """Perform a prediction on wrapped model's ``predict`` method.

        :raises HTTPNotImplemented: If the method is not
            implemented in the wrapper model.
        :raises HTTPInternalServerError: If the call produces
            an error
        :raises HTTPException: If the call produces an
            error, already wrapped as a HTTPException
        """
        for key, val in kwargs.items():
            if isinstance(val, web.FileField):
                fd, name = tempfile.mkstemp()
                fd = os.fdopen(fd, "w+b")
                fd.write(val.file.read())
                fd.close()
                aux = UploadedFile(
                    name=val.name,
                    filename=name,
                    content_type=val.content_type,
                    original_filename=val.filename,
                )
                kwargs[key] = aux
                # FIXME(aloga); cleanup of tmpfile here

        with self._catch_error():
            return self._run_in_pool(
                self.predict_wrap, self.model_obj.predict, *args, **kwargs
            )

    def train(self, *args, **kwargs):
        """Perform a training on wrapped model's ``train`` method.

        :raises HTTPNotImplemented: If the method is not
            implemented in the wrapper model.
        :raises HTTPInternalServerError: If the call produces
            an error
        :raises HTTPException: If the call produces an
            error, already wrapped as a HTTPException
        """

        with self._catch_error():
            return self._run_in_pool(self.model_obj.train, *args, **kwargs)

    def get_train_args(self):
        """Add training arguments into the training parser.

        :param parser: an argparse like object

        This method will call the wrapped model ``add_train_args``.
        """
        try:
            args = self.model_obj.get_train_args()
        except (NotImplementedError, AttributeError):
            args = {}
        return args

    def get_predict_args(self):
        """Add predict arguments into the predict parser.

        :param parser: an argparse like object

        This method will call the wrapped model ``get_predict_args``.
        """
        try:
            args = self.model_obj.get_predict_args()
        except (NotImplementedError, AttributeError):
            args = {}
        return args

class NonDaemonProcess(multiprocessing.context.SpawnProcess):
    """Processes must use 'spawn' instead of 'fork' (which is the default
    in Linux) in order to work CUDA [1] or Tensorflow [2].

    [1] https://pytorch.org/docs/stable/notes/multiprocessing.html
    #cuda-in-multiprocessing
    [2] https://github.com/tensorflow/tensorflow/issues/5448
    #issuecomment-258934405
    """

    @property
    def daemon(self):
        return False

    @daemon.setter
    def daemon(self, value):
        pass

class NonDaemonPool(multiprocessing.pool.Pool):
    # Based on https://stackoverflow.com/questions/6974695/
    def Process(self, *args, **kwds):  # noqa
        proc = super(NonDaemonPool, self).Process(*args, **kwds)
        proc.__class__ = NonDaemonProcess

        return proc

class CancellablePool(object):
    def __init__(self, max_workers=None):
        self._free = {self._new_pool() for _ in range(max_workers)}
        self._working = set()
        self._change = asyncio.Event()

    def _new_pool(self):
        return NonDaemonPool(1, context=multiprocessing.get_context("spawn"))

    async def apply(self, fn, *args):
        """
        Like multiprocessing.Pool.apply_async, but:
         * is an asyncio coroutine
         * terminates the process if cancelled
        """
        while not self._free:
            await self._change.wait()
            self._change.clear()
        pool = usable_pool = self._free.pop()
        self._working.add(pool)

        loop = asyncio.get_event_loop()
        fut = loop.create_future()

        def _on_done(obj):
            ret = {"output": obj, "finish_date": str(datetime.datetime.now())}
            loop.call_soon_threadsafe(fut.set_result, ret)

        def _on_err(err):
            loop.call_soon_threadsafe(fut.set_exception, err)

        pool.apply_async(fn, args, callback=_on_done, error_callback=_on_err)

        try:
            return await fut
        except asyncio.CancelledError:
            # This is ugly, but since our pools only have one slot we can
            # kill the process before termination
            try:
                pool._pool[0].kill()
            except AttributeError:
                os.kill(pool._pool[0].pid, signal.SIGKILL)
            pool.terminate()
            usable_pool = self._new_pool()
        finally:
            self._working.remove(pool)
            self._free.add(usable_pool)
            self._change.set()

    def shutdown(self):
        for p in self._working:
            p.terminate()
        self._free.clear()


from abc import ABC, abstractmethod
import threading
import boto3
import botocore
import sys
import logging
import logging.config
from enum import Enum
from itertools import cycle
from botocore.config import Config
from botocore.endpoint import MAX_POOL_CONNECTIONS
from collections.abc import Iterable

class AWS_SVC_BASE(ABC):
    ''' Represent an AWS service that contain multiple resources(workers) '''
    aws_config = Config(
        retries=dict(
            total_max_attempts=25,
            mode='adaptive'
        ),
        max_pool_connections=MAX_POOL_CONNECTIONS,
    )

    def __init__(self, svc_type, session, svc_config):
        if not isinstance(session, boto3.Session):
            logging.error('session must be of type boto3.Session')
            raise(ValueError)
        if not isinstance(svc_type, AWS_SVC_TYPE):
            logging.error('svc_type must be of type AWS_SVC_TYPE')
            raise(ValueError)
        if not isinstance(svc_config, dict):
            logging.error('svc_config must be of type AWS_SVC_TYPE')
            raise(ValueError)
        self.session = session
        self.account_id = 0
        self.service_type = svc_type
        self.svc_config = svc_config
        self.rsc_prefix = svc_config['resource_prefix']
        self._key_lock = threading.Lock()
        self.worker_cycle = cycle(list())
        super().__init__()

    @abstractmethod
    def get_existing_workers(self):
        ''' Query the existing workers based on the rsc_prefix '''
        # pass

    @abstractmethod
    def create_workers(self):
        ''' Create workers/resources of this service '''
        # pass

    @abstractmethod
    def delete_workers(self):
        ''' Delete the workers created by create_workers() function '''
        # pass

    @abstractmethod
    def _check_existing_identity(self, identiy_arn):
        ''' Check if identiy_arn exists in AWS '''
        # pass

    def check_existing_user(self, aws_id, target_user, aws_partition = 'aws'):
        ''' Check if the target_user exists in AWS account aws_id '''
        user_arn = 'arn:{}:iam::{}:user/{}'.format(aws_partition, aws_id, target_user)
        return self._check_existing_identity(user_arn)

    def check_existing_role(self, aws_id, target_role, aws_partition = 'aws'):
        ''' Check if the target_role exists in AWS account aws_id '''
        role_arn = 'arn:{}:iam::{}:role/{}'.format(aws_partition, aws_id, target_role)
        return self._check_existing_identity(role_arn)

    def precheck(self):
        ''' Check if there is at least one resrouce to perform the test '''
        # If no object is in the cycle, the default value None will be returned
        if next(self.worker_cycle, None) is None:
            return False
        return True

    def _get_next_worker(self):
        with self._key_lock:
            try:
                return next(self.worker_cycle)
            except StopIteration:
                logging.error('Empty worker cycle')
                return None

    def _set_worker_cycle(self, iterable_obj):
        if not isinstance(iterable_obj, Iterable):
            logging.error('set_worker_cycle function expects an Iterable input')
            return
        self.worker_cycle = cycle(iterable_obj)

    def _check_boto3_response(self, resp):
        return 'ResponseMetadata' in resp and resp['ResponseMetadata']['HTTPStatusCode'] >= 200 and resp['ResponseMetadata']['HTTPStatusCode'] < 300

    def _enable_logging(self):
        logging.config.dictConfig({
            'version': 1,
            'disable_existing_loggers': True,
        })
        logging.basicConfig(level=logging.DEBUG, format='%(module)s: %(message)s')

class AWS_SVC_TYPE(Enum):
    IAM = 'iam'
    S3 = 's3'
    KMS = 'kms'
    SQS = 'sqs'


import os
import pickle
import shutil
import numpy as np
from tqdm import tqdm
import time

class ModelManager:
    '''
    Model manager is designed to load and save all models
    No matter what dataset name.
    '''
    path_name = './checkpoints/'
    @classmethod
    def __init__(cls, cfg):
        if not cfg.MODEL.TRAINING and cfg.PATH.MODEL_PATH is not None:
            cls.path_name = cfg.PATH.MODEL_PATH
        elif cfg.MODEL.TRAINING and cfg.MODEL.MODEL_NAME:
            cls.path_name += cfg.MODEL.MODEL_NAME+"-"+ time.strftime("%Y_%m_%d__%H_%M_%S", time.localtime()) +'/'
            cfg.PATH.MODEL_PATH = cls.path_name
        else:
            raise Exception('Model path initialization error, please check your config.py')
    def save_model(self, model, model_name):
        '''
        Save model to model/ dir
        :param model: model to be saved
        :param model_name: model name
        :return: None
        '''
        if 'pkl' not in model_name:
            model_name += '.pkl'
        if not os.path.exists('checkpoints'):
            os.makedirs('checkpoints')
        if not os.path.exists( self.path_name):
            os.makedirs(self.path_name)
        pickle.dump(model,open(self.path_name+model_name,'wb'))

    def save_config(self,cfg):
        '''
        Save config to model/ dir as yaml file
        :param cfg: config
        :return: None
        '''
        if not os.path.exists(self.path_name):
            os.makedirs(self.path_name)
        cfg.PATH.CONFIG_PATH = self.path_name+'config.yaml'
        with open(self.path_name+'config.yaml','w') as f:
            f.write(cfg.dump())

    def load_model(self, model_name):
        '''
        load model from model/ dir
        :param model_name: model name
        :return: model
        '''

        if 'pkl' not in model_name:
            model_name += '.pkl'
        if not os.path.exists(self.path_name+model_name):
            raise Exception('Model not found %s'%(self.path_name+model_name))
        return pickle.load(open(self.path_name+model_name,'rb'))

    def save_test_result(self,test_result):
        '''
        Save test result to model/ dir
        :param test_result: test result, as txt file
        :return: None
        '''
        if not os.path.exists(self.path_name):
            os.makedirs(self.path_name)
        with open(self.path_name+'test_result.txt','w') as f:
            for item in test_result:
                f.write(str(item)+'\n')

    @staticmethod
    def clean_workspace():
        '''
        clean model/ dir
        :return: None
        '''
        if os.path.exists('checkpoints'):
            shutil.rmtree('checkpoints')

def get_time_cost(begin_time, end_time):
    '''
    get the time cost
    :param begin_time: the start time
    :param end_time: the end time
    :return: the time cost
    '''
    time_cost = end_time - begin_time
    return "%d day %d hour %d minute %.2f second"%(time_cost // 86400, time_cost % 86400 // 3600, time_cost % 3600 // 60, time_cost % 60)

def k_neighbors(sim_vector, k):
    '''
    input the  similarity matrix, the index of the user, and the k
    return the k nearest neighbor of the user
    :param sim_vector: the similarity matrix
    :param k: the k
    :return: the k nearest neighbor of the user and the similarity between the user and the neighbor
    '''
    # get the similarity matrix
    sim_vector = sim_vector
    # get the k
    k = k
    # get the k nearest neighbor of the user
    neighbor = np.argsort(sim_vector)[-k-1:-1]
    neighbor_sim = np.sort(sim_vector)[-k-1:-1] # do not include the user itself
    return neighbor, neighbor_sim

def get_score_matrix(train_rating,user_map,movie_map):
    '''
        get the score  matrix
        @param: train_rating, the train rating
        @param: user_map, the user map
        @param: movie_map, the movie map
        @return: score_matrix, the movie popularity, the movie count
    '''
    print("<<<< begin to conduct the score matrix")
    score_matrix = np.zeros((len(user_map.keys()),len(movie_map.keys())))
    movie_popular = np.zeros(len(movie_map.keys()))
    movie_count = len(movie_map.keys())
    tqdm_process = tqdm(total=train_rating.shape[0])
    for row in train_rating.itertuples(index=True,name="Pandas"):
        user = user_map[getattr(row,'userId')]
        movie = movie_map[getattr(row,'movieId')]
        rate = getattr(row,'rating')
        score_matrix[user][movie] = rate
        movie_popular[movie] += 1
        tqdm_process.update(1)
    tqdm_process.close()
    print(">>>> end to conduct the score matrix")
    print("@ score matrix shape:",score_matrix.shape)
    print('movie_popular shape:',movie_popular.shape)
    print('movie_count:',movie_count)
    return score_matrix, movie_popular, movie_count

def calculate_movie_similarity(train_set,pre_sim_calcul = False):
    '''
    calculate the tfidf of the movies
    :param train_set: the train set, a tuple of (trainset,user_map,movie_map,movie_type_features)
    :return: score_matrix, movie_popular, movie_sim, movie_count
    '''
    # get the train set
    train_rating, user_map, movie_map, movie_type_features = train_set

    score_matrix, movie_popular, movie_count = get_score_matrix(train_rating,user_map,movie_map)

    movie_sim= np.zeros((movie_count, movie_count))

    if pre_sim_calcul:
        print("<<<< begin to conduct the movie similarity matrix")
        begin_time = time.time()  # record the start time
        for i in tqdm(range(movie_count)):
            movie_sim[i][i] = 1
            for j in range(i+1,movie_count):
                movie_sim[i][j] = cosine_similarity(movie_type_features[i],movie_type_features[j])
                movie_sim[j][i] = movie_sim[i][j]
        end_time = time.time()  # record the end time

        print(">>>> end to conduct the movie similarity matrix")
        print("@ time cost: %s"%get_time_cost(begin_time,end_time))
    else:
        print("post calculate  the similarity during prediction!")
    return score_matrix, movie_popular, movie_sim, movie_count,user_map,movie_map,movie_type_features

def cosine_similarity(list1,list2):
    '''
    calculate the cosine_similarity of list1 and list2

    :param list1: the first list
    :param list2: the second list
    :return: the cosine_similarity
    '''
    # get the number of common items
    assert(len(list1) == len(list2))
    n = len(list1)
    assert(n > 0)
    # calculate the sum of the two lists
    sum1 = sum(list1*list2)
    # calculate the square of the two lists
    den = np.sqrt(sum(list1**2)) * np.sqrt(sum(list2**2))
    # calculate the cosine similarity
    if den == 0:
        return 0
    else:
        return sum1/den

def calculate_user_sim_matrix(train_set,pre_sim_calcul = True):
    '''
    calculate the similarity matrix between users
    :param train_set: the train set, a tuple of (trainset,user_map,movie_map,movie_type_features)
        """
    :return: the score_matrix, the similarity matrix, movie_popular, movie_count
    '''
    # conduct the score matrix
    print("<<<<<< begin to caculate the similarity matrix, the movie popularity and the movie count")

    train_rating, user_map, movie_map, movie_type_features  = train_set

    score_matrix, movie_popular, movie_count = get_score_matrix(train_rating,user_map,movie_map)

    # get the similarity matrix between users
    user_sim_matrix = np.zeros((score_matrix.shape[0],score_matrix.shape[0]))
    if pre_sim_calcul:
        user_sim_matrix = get_user_sim_matrix(score_matrix)
    else:
        print("post calculate  the similarity during prediction!")
    print(">>>> end to caculate the similarity matrix.")
    print('user_sim_matrix shape:',user_sim_matrix.shape)
    return score_matrix,user_sim_matrix, movie_popular, movie_count,user_map,movie_map,movie_type_features

def get_user_sim_matrix(input_matrix):
    '''
    get the similarity matrix between users with pearson similarity
    :param input_matrix: the input matrix with shape (n_users, n_items)
    :return: the similarity matrix
    '''
    # get the shape of the input matrix
    begin_time = time.time()  # record the start time
    print("<<<< begin to get the similarity matrix")
    input_matrix = np.array(input_matrix) # convert to numpy array
    print('input score matrix shape:',input_matrix.shape)
    # get the number of users
    n_users = input_matrix.shape[0]
    # calculate the similarity matrix between users with person similarity
    user_sim_matrix = np.zeros((n_users, n_users))
    print('user_sim_matrix shape:',user_sim_matrix.shape)

    for i in tqdm(range(n_users)):
        user_sim_matrix[i][i] = 1
        for j in range(i+1,n_users):
            user_sim_matrix[i][j] = pearson_sim(input_matrix[i],input_matrix[j])
            user_sim_matrix[j][i] = user_sim_matrix[i][j]
    print(">>>> end to get the similarity matrix")
    end_time = time.time()  # record the end time
    print('@ time cost: '+get_time_cost(begin_time, end_time))
    return user_sim_matrix

def pearson_sim(list1,list2):
    '''
    calculate the pearson similarity between two lists
    :param list1: the first list
    :param list2: the second list
    :return: the pearson similarity
    '''
    # get the number of common items
    assert len(list1) == len(list2)
    n = len(list1)
    assert n > 0
    # calculate the sum of the two lists
    avg1 = sum(list1)/n
    avg2 = sum(list2)/n
    norm1 = list1 - avg1
    norm2 = list2 - avg2
    # calculate the sum of the two lists
    sum1 = sum(norm1*norm2)
    # calculate the square of the two lists
    den = np.sqrt(sum(norm1**2)) * np.sqrt(sum(norm2**2))
    # calculate the pearson similarity
    if den == 0:
        return 0.0
    else:
        return sum1/den

def SSE_error(prediction,real_rating):
    '''
    calculate the SSE error
    :param prediction: the prediction of the user
    :param real_rating: the real rating of the user
    :return: the SSE error
    '''
    # get the prediction and the real rating
    prediction = np.array(prediction)
    real_rating = np.array(real_rating)
    # calculate the SSE error
    SSE = sum((prediction - real_rating)**2)
    return SSE

if __name__ == '__main__':
    # test the similarity matrix
    from dataset import Dataset
    from config import cfg
    dataset = Dataset(cfg)
    train_set = dataset.get_trainset()

    a = pearson_sim(np.array([1,2,3,4,5]),np.array([1,2,3,4,5]))
    b = pearson_sim(np.array([1,2,3,4,5]),np.array([5,4,3,2,1]))
    print(a,b)
    score_matrix,user_sim_matrix, movie_popular, movie_count,user_map,movie_map = calculate_user_sim_matrix(train_set,pre_sim_calcul = False)
    pickle.dump(user_sim_matrix,open('user_map.pkl','wb'))
    pickle.dump(movie_map,open('movie_map.pkl','wb'))
    # print(user_sim_matrix, movie_popular, movie_count)

    # pickle.dump(train_set, open('./checkpoints\CF-2022_04_11__11_32_40/trainset.pkl', 'wb'))
    # pickle.dump(score_matrix,open('checkpoints\CF-2022_04_11__11_32_40\score_matrix.pkl','wb'))

    # test the model_manager
    # model_manager = ModelManager(cfg)
    # model_manager.clean_workspace()
    # model_manager.save_model(user_sim_matrix, 'user_sim_matrix')
    # model_manager.save_model(movie_popular, 'movie_popular')
    # model_manager.save_model(movie_count, 'movie_count')
    # d = model_manager.load_model('score_matrix')
    # a = model_manager.load_model('user_sim_mat')
    # b = model_manager.load_model('movie_popular')
    # c = model_manager.load_model('movie_count')
    # print(a[0:3],b,c,d[0:3])

    # test the time cost
    # begin_time = time.time()  # record the start time
    # time.sleep(3)
    # end_time = time.time()  # record the end time

    # # print the time cost
    # print('@ time cost:',get_time_cost(begin_time,end_time))


import os

scriptPath = os.path.dirname(os.path.abspath(__file__))
projRootPath = os.path.abspath(
    os.path.join(scriptPath ,
                os.path.join('..', '..')))

import numpy as np

# matplotlib for displaying the output
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

from scipy import signal
from scipy.io import wavfile

# and IPython.display for audio output
import IPython.display

# Librosa for audio
import librosa

# And the display module for visualization
import librosa.display

#### Path to data
# Get data files
two_up =  os.path.abspath(os.path.join('.' ,"../.."))
print("Project root path is: ", two_up)

dataDirName = "data"
rawDataDirName = "converted_wav"
className = "violin"
# className = "guitar"
data_path = os.path.join(projRootPath, dataDirName, rawDataDirName, className)

print(data_path)
root_paths = []

# Get all files from data_path
# r=root, d=directories, f = files
(_, d, allFiles) = next(os.walk(data_path))
wavFiles = [f for f in allFiles if f.endswith(".wav")]

file = wavFiles[1]
sample_rate, samples = wavfile.read(os.path.join(data_path, file))
frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)

# all spectrogram
plt.pcolormesh(times, frequencies, spectrogram)
plt.imshow(spectrogram)
plt.ylabel('Frequency')
plt.gca().invert_yaxis()
plt.xlabel('Time')
plt.show()


import requests
import ast
import sys
import getopt

class XkcdClient():

    def api_call(self, url):
        self.urls = url

        r = requests.get(url = self.urls)
        byte_str = r.content
        dict_str = byte_str.decode("UTF-8")
        my_data = ast.literal_eval(dict_str)
        return my_data

    def get_image(self,img_url):
        self.img_name = img_url.split('/')[-1]
        img_data = requests.get(img_url).content
        with open(self.img_name, 'wb') as handler:
            handler.write(img_data)

# client = XkcdClient()
# response = client.api_call('https://xkcd.com/info.0.json')
# print(response)

if __name__ == '__main__':

    cmd_line_args = sys.argv[1:]
    unix_args = 'hn:os'
    gnu_args = ['help','comicnum=','print','save-image']

    oplist, args = getopt.getopt(cmd_line_args,unix_args,gnu_args)

    print(args) #Extra arguments that are not part of the uni_args or gnu_args
    print(oplist) #oplist is a list of tuples
    comic_num = ''
    client = XkcdClient()
    url_latest = 'https://xkcd.com/info.0.json'
    for opt, arg in oplist:
        print(opt)
        print(arg)

        if opt == '-h' or opt == '--help':
            print('help message')
            print('Use -n or --comicnum to specify the comic number you want use 0 as argument for latest comic')
            print('Use -o or --print to get info in text/json format')
            print('Use -s or --save-image to download image in this directory')
        elif opt == '-n' or opt == '--comicnum':
            comic_num = arg
            if comic_num is '0': #default get the latest comic
                print('Get the comic number ' + str(arg))
                response = client.api_call(url_latest)
                print(response)
            else:
                url_specific = 'http://xkcd.com/'+arg+'/info.0.json'
                response = client.api_call(url_specific)
        elif opt == '-o' or opt == '--print':
            if comic_num:
                if comic_num is '0':
                    print('print output in format json/text')
                    print(response)
                else:
                    print('The output in json/text is')
                    print(response)
            else:
                print('Set the -n parameter first')
        elif opt == '-s' or opt == '--save-image':
            if comic_num:
                img_url = response['img']
                client.get_image(img_url)
            else:
                print('Set the -n parameter first')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 22 16:48:54 2019

@author: xiaohaoren
"""

import json
import pickle
import numpy as np

negative_word = ['悶熱','吵雜','髒','髒亂','加強','改進','缺點']

def Load_All_Info(json_path,pickle_path):

    with open(json_path,'r') as fp:
        json_data = json.load(fp)
    with open(pickle_path, 'rb') as fp:
        pickle_data = pickle.load(fp)

    keys = list(json_data.keys())

    return json_data,pickle_data,keys

def FilteringAndRanking(querys,places,corpus,review_list=None):
    """
    query = ['冷氣','衛生',...]
    place = ['春山茶水舖','小川拉麵',...]
    corpus = {'春山茶水舖':{'不錯':(正向次數,評論編號),'五花肉':(正向分數,評論編號),...}}
    """
    scoreboard = {}

    for i,place in enumerate(places):

        #N = corpus[place]['__termNum__']
        N = corpus[place]['__reviewNum__']

        scoreboard[place]=0

        if place not in corpus:
            continue

        for term in querys:
            term_score = 0
            term_sign = -1 if term in negative_word else 1
            if term not in corpus[place]:
                continue
            else:
                keyword_data = corpus[place][term]
                for rid,p in keyword_data.items():
                    term_score += (term_sign * p)

                    if review_list is not None:
                        rid = int(rid)
                        review_content = review_list[rid]
                        print('"%s"由於「%s」中的"%s"而加%d分' % (place,review_content,term,term_sign*p))

            scoreboard[place] += term_score

        scoreboard[place] = scoreboard[place]/(N*len(querys)) * 100

    return scoreboard

if __name__ == "__main__":

    corpus_path = '../data/place_dict.json'
    reviewContent_path = '../data/review_list.json'
    querys = ['乾淨','衛生']

    corpus,review_list,places = Load_All_Info(json_path=corpus_path,pickle_path=reviewContent_path)

    scoreboard = FilteringAndRanking(querys=querys,places=places,corpus=corpus,review_list=review_list)


import argparse
from distutils.util import strtobool
import pathlib

import siml

import convert_raw_data

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'settings_yaml',
        type=pathlib.Path,
        help='YAML file name of settings.')
    parser.add_argument(
        'raw_data_directory',
        type=pathlib.Path,
        help='Raw data directory')
    parser.add_argument(
        '-p', '--preprocessors-pkl',
        type=pathlib.Path,
        default=None,
        help='Preprocessors.pkl file')
    parser.add_argument(
        '-o', '--out-dir',
        type=pathlib.Path,
        default=None,
        help='Output directory name')
    parser.add_argument(
        '-f', '--force-renew',
        type=strtobool,
        default=0,
        help='If True, overwrite existing data [False]')
    parser.add_argument(
        '-l', '--light',
        type=strtobool,
        default=0,
        help='If True, compute minimum required data only [False]')
    parser.add_argument(
        '-n', '--read-npy',
        type=strtobool,
        default=1,
        help='If True, read .npy files instead of original files '
        'if exists [True]')
    parser.add_argument(
        '-r', '--recursive',
        type=strtobool,
        default=1,
        help='If True, process directory recursively [True]')
    parser.add_argument(
        '-e', '--elemental',
        type=strtobool,
        default=0,
        help='If True, create also elemental features [False]')
    parser.add_argument(
        '-a', '--convert-answer',
        type=strtobool,
        default=1,
        help='If True, convert answer [True]')
    parser.add_argument(
        '-s', '--skip-interim',
        type=strtobool,
        default=0,
        help='If True, skip conversion of interim data [False]')
    args = parser.parse_args()

    main_setting = siml.setting.MainSetting.read_settings_yaml(
        args.settings_yaml)
    if not args.convert_answer:
        main_setting.conversion.required_file_names = ['*.msh', '*.cnt']
    main_setting.data.raw = args.raw_data_directory
    if args.out_dir is None:
        args.out_dir = args.raw_data_directory
        main_setting.data.interim = [siml.prepost.determine_output_directory(
            main_setting.data.raw,
            main_setting.data.raw.parent / 'interim', 'raw')]
        main_setting.data.preprocessed = [
            siml.prepost.determine_output_directory(
                main_setting.data.raw,
                main_setting.data.raw.parent / 'preprocessed', 'raw')]
    else:
        main_setting.data.interim = [args.out_dir / 'interim']
        main_setting.data.preprocessed = [args.out_dir / 'preprocessed']

    if not args.skip_interim:
        conversion_function = convert_raw_data.HeatConversionFuncionCreator(
            create_elemental=args.elemental,
            convert_answer=args.convert_answer,
            light=args.light)
        raw_converter = siml.prepost.RawConverter(
            main_setting,
            conversion_function=conversion_function,
            filter_function=convert_raw_data.filter_function_heat,
            force_renew=args.force_renew,
            recursive=args.recursive,
            to_first_order=True,
            write_ucd=False,
            read_npy=args.read_npy, read_res=args.convert_answer)
        raw_converter.convert()

    preprocessor = siml.prepost.Preprocessor(
        main_setting, force_renew=args.force_renew,
        allow_missing=True)
    preprocessor.convert_interim_data(preprocessor_pkl=args.preprocessors_pkl)

    return

if __name__ == '__main__':
    main()


import re
import time
import textwrap
from copy import copy
import torch.nn.functional as F
from training_utils import *

BASH_FORMATTING = {
                   'PURPLE': '\033[95m',
                   'CYAN': '\033[96m',
                   'DARKCYAN': '\033[36m',
                   'BLUE': '\033[94m',
                   'GREEN': '\033[92m',
                   'YELLOW': '\033[93m',
                   'RED':'\033[91m',
                   'BOLD': '\033[1m',
                   'UNDERLINE': '\033[4m',
                   'END': '\033[0m'
}

def bash_format_text(text, *args):
    formatting = ''
    for arg in args:
        formatting += BASH_FORMATTING[arg]
    return formatting + text + BASH_FORMATTING['END']

# maybe add do_sample ?
# randomly select one of two values that evaluate to true or false lols hehehe
def transfer_learning_bot(model, tokenizer, max_length, top_k, top_p):
    '''
    for chatbot trained using transfer learning
    '''
    input_sentence = input('User >> ')
    input_sentence = input_sentence.lower()
    context = copy(input_sentence)
    input_sentence = tokenizer.encode(input_sentence, truncation = True, max_length = 128, return_tensors = 'pt')
    continue_convo = True
    while continue_convo:
        print(bash_format_text('Typing...', 'YELLOW', 'BOLD'), end='\r' )
        uni_temp = round(torch.rand(1).clamp(0.1).item(), 2)
        repeat_penalty = round((torch.rand(1) * 5).clamp(1).item(), 2)
        ngram =  int(np.random.choice([2,3,4], 1)[0])
        bot_reply = model.generate(input_sentence, max_length = max_length, top_k = top_k, top_p = top_p, temperature = uni_temp,
                                   repetition_penalty = repeat_penalty, skip_special_tokens = True,
                                   no_repeat_ngram_size=ngram, pad_token_id = tokenizer.eos_token_id)
                                   # length_penalty=length_penalty)
        bot_reply = tokenizer.decode(bot_reply.squeeze()).replace('<|endoftext|>', '')
        bot_reply = textwrap.fill(bot_reply, width=75)
        print(bash_format_text('Aubrey: {}'.format(bot_reply), 'YELLOW', 'BOLD'))
        response = input('User >> ')
        if (response == 'q' or response == 'quit' or response == 'exit'):
            continue_convo = False
        input_sentence = tokenizer.encode(response.lower(), truncation= True, max_length = 128, return_tensors = 'pt')


from unittest import TestCase
import unittest
import requests

# import sys
#
# sys.path.insert(0, '../../src')

class TestLoadTimeSeries(TestCase):
    def test_load_data_success(self):
        f = open("tests/routes/time_series_covid19_recovered_global.csv", "rb")
        file = f.read()
        url = 'https://covid-monitor-61.herokuapp.com/time_series/data?type=recovered'
        r = requests.post(url, data=file, headers={"Content-Type": "text/csv"})
        f.close()
        self.assertEqual(r.status_code, 200)

    def test_query_data(self):
        url = 'https://covid-monitor-61.herokuapp.com/time_series/cases'
        body = {"return_type": "json",
                "start_date": "01/26/20",
                "end_date": "01/28/20",
                "types": ["Recovered"],
                "locations":
                    [
                        {"Country/Region": "Albania"},
                        {"Country/Region": "Canada", "Province/State": "Ontario"},
                        {"Country/Region": "Australia"}
                    ]
                }
        r = requests.post(url, json=body, headers={"Content-Type": "application/json"})
        self.assertEqual(r.status_code, 200)

if __name__ == '__main__':
    unittest.main()


import numpy as np
import sys

from vispy import app, visuals, scene

# build visuals
Plot3D = scene.visuals.create_visual_node(visuals.line.line.LineVisual)

# build canvas
canvas = scene.SceneCanvas(keys='interactive', title='plot3d', show=True)

# Add a ViewBox to let the user zoom/rotate
view = canvas.central_widget.add_view()
view.camera = 'turntable'
view.camera.fov = 45
view.camera.distance = 6

# prepare data
x, y, z, segments = [], [], [], []
for start, i in enumerate(np.linspace(-5, 5, 1000)):
    N = 6000
    x.append(np.sin(np.linspace(-5-i, 5+1, N)*np.pi))
    y.append(np.cos(np.linspace(-5+i, 5-i, N)*np.pi))
    z.append(np.linspace(-5-i, 5-i, N))
    start_idx = 1000 * start
    idxs = np.arange(start_idx, start_idx+N-1)
    idxs = np.stack([idxs, idxs+1], axis=-1)
    segments.append(idxs)
x, y, z = np.concatenate(x), np.concatenate(y), np.concatenate(z)
segments = np.concatenate(segments, axis=0)

# plot
pos = np.c_[x, y, z]
Plot3D(pos, width=10.0,
    color=(1.0, 0.0, 0.0, 1.0), method='gl',
    connect=segments,
    parent=view.scene)

if __name__ == '__main__':
    if sys.flags.interactive != 1:
        app.run()


from env import data
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from time import sleep

#Program Isi Data
def program(data_set):
    #masuk website
    browser = webdriver.Chrome()
    actions = ActionChains(browser)
    browser.get(data['linkActive'])
    print('==== Welcome To Bangef, Automated Post-Test ====');

    try:
        for d in data_set:
            # mengecek apakah elemen input sudah ada dan mengisikannya
            # Nama
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][0]))
            ).send_keys(d['namaLengkap'])

            # Email
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][1]))
            ).send_keys(d['email'])

            # Nomer Telepon
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][2]))
            ).send_keys('0'+d['noTelpon'])

            # Jenis Kelamin
            elementJK = browser.find_element(By.ID, data['selectorById'][3])
            Select(elementJK).select_by_value(d['jk'])

            # Usia
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][4]))
            ).send_keys(d['usia'])

            # Pekerjaan
            elementPekerjaan = browser.find_element(By.ID, data['selectorById'][5])
            Select(elementPekerjaan).select_by_value(d['pekerjaan'])

            # Komunitas
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][6]))
            ).send_keys(d['organisasi'])

            # Pendidikan
            elementPendidikan = browser.find_element(By.ID, data['selectorById'][7])
            Select(elementPendidikan).select_by_value(d['pendidikan'])

            # Provinsi
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][8]))
            ).send_keys(d['provinsi'], Keys.RETURN)

            # Kota Asal
            elementCities = browser.find_element(By.ID, data['selectorById'][9])
            elementCities.send_keys(d['kotaAsal'], Keys.RETURN)

            # Captcha
            browser.execute_script("arguments[0].scrollIntoView();", elementCities)
            captcha = input('Masukan validasi captcha (sample : 9*9): \n');
            arr = list(captcha)
            if arr[1] == '+' :
                result = int(arr[0]) + int(arr[2])
            else :
                result = int(arr[0]) * int(arr[2])

            browser.find_element(By.ID, data['selectorById'][10]).send_keys(result)

            # Select Radio Button
            q1 = browser.find_element(By.ID, "1070-"+d['qSatu'])
            browser.execute_script("arguments[0].scrollIntoView();", q1)
            sleep(.5)
            q1.click()

            q2 = browser.find_element(By.ID, "1071-"+d['qDua'])
            browser.execute_script("arguments[0].scrollIntoView();", q2)
            q2.click()

            q3 = browser.find_element(By.ID, "1072-"+d['qTiga'])
            browser.execute_script("arguments[0].scrollIntoView();", q3)
            q3.click()

            q4 = browser.find_element(By.ID, "1073-"+d['qEmpat'])
            actions.move_to_element(q4).click().perform()
            browser.execute_script("arguments[0].scrollIntoView();", q4)
            sleep(.5)
            q4.click()

            q5 = browser.find_element(By.ID, "1076-"+d['qLima'])
            browser.execute_script("arguments[0].scrollIntoView();", q5)
            q5.click()

            # submit
            footer = browser.find_element(By.CSS_SELECTOR, '#__next > div > div.footer.mt-3')
            browser.execute_script("arguments[0].scrollIntoView();", footer)
            sleep(1)
            browser.find_element(By.XPATH, data['selectorByXpath']).click()
            sleep(3)

            # kembali ke page sebelumnya
            browser.get(data['linkActive'])
            WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.ID, data['selectorById'][0]))
            )
            print('Data dengan atas nama '+d['namaLengkap']+' berhasil ✔')

        print('Total Data : '+ str(d['id']) +' Selesai Post Test')

    except Exception as err:
        print('Data Selesai Terakhir : "id": "'+str(d['id'])+'".')
        print(err)
        browser.quit()


import matplotlib.pyplot as plt
import numpy as np

with open('scores.txt', 'r') as f:
    scores = f.read().splitlines()
    scores = list(map(int, scores))

    mean = []
    max_list = []
    for i,j in enumerate(scores):
        if i % 1000 == 0:
            mean.append(np.average(scores[i:i+1000]))
            max_list.append(max(scores[i:i+1000]))

    episode = range(len(mean))
    plt.ylabel('episode scores')
    plt.xlabel('training episodes')
    plt.yticks(np.arange(min(mean), max(mean)+1, 5000))
    plt.title('mean scores')
    plt.plot(episode, mean)
    plt.show()

    plt.ylabel('episode scores')
    plt.xlabel('training episodes')
    plt.yticks(np.arange(min(max_list), max(max_list)+1, 10000))
    plt.title('max scores')
    plt.plot(episode, max_list)
    plt.show()


import argparse
import glob
import os

import h5py
import hdbscan
import numpy as np
from scipy.ndimage import binary_erosion
from skimage.filters import gaussian
from skimage.segmentation import watershed
from sklearn.cluster import MeanShift

def expand_labels_watershed(seg, raw, erosion_iters=4):
    bg_mask = seg == 0
    # don't need to  do anything if we only have background
    if bg_mask.size == int(bg_mask.sum()):
        return seg

    hmap = gaussian(raw, sigma=1.)

    bg_mask = binary_erosion(bg_mask, iterations=erosion_iters)
    seg_new = seg.copy()
    bg_id = int(seg.max()) + 1
    seg_new[bg_mask] = bg_id

    seg_new = watershed(hmap, seg_new)

    seg_new[seg_new == bg_id] = 0
    return seg_new

def cluster(emb, clustering_alg, semantic_mask=None):
    output_shape = emb.shape[1:]
    # reshape (E, D, H, W) -> (E, D * H * W) and transpose -> (D * H * W, E)
    flattened_embeddings = emb.reshape(emb.shape[0], -1).transpose()

    result = np.zeros(flattened_embeddings.shape[0])

    if semantic_mask is not None:
        flattened_mask = semantic_mask.reshape(-1)
        assert flattened_mask.shape[0] == flattened_embeddings.shape[0]
    else:
        flattened_mask = np.ones(flattened_embeddings.shape[0])

    if flattened_mask.sum() == 0:
        # return zeros for empty masks
        return result.reshape(output_shape)

    # cluster only within the foreground mask
    clusters = clustering_alg.fit_predict(flattened_embeddings[flattened_mask == 1])
    # always increase the labels by 1 cause clustering results start from 0 and we may loose one object
    result[flattened_mask == 1] = clusters + 1

    return result.reshape(output_shape)

def cluster_hdbscan(emb, min_size, eps, min_samples=None, semantic_mask=None):
    clustering = hdbscan.HDBSCAN(min_cluster_size=min_size, cluster_selection_epsilon=eps, min_samples=min_samples)
    return cluster(emb, clustering, semantic_mask)

def cluster_ms(emb, bandwidth, semantic_mask=None):
    clustering = MeanShift(bandwidth=bandwidth, bin_seeding=True)
    return cluster(emb, clustering, semantic_mask)

def run_clustering(emb, clustering, delta_var, min_size, expand_labels, remove_largest):
    assert clustering in ['ms', 'hdbscan']
    if clustering == 'hdbscan':
        clusters = cluster_hdbscan(emb, min_size, delta_var)
    else:
        clusters = cluster_ms(emb, delta_var)

    # watershed the empty (i.e. noise) region
    if expand_labels:
        clusters = expand_labels_watershed(clusters, raw)

    if remove_largest:
        ids, counts = np.unique(clusters, return_counts=True)
        clusters[ids[np.argmax(counts)] == clusters] = 0

    return clusters

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Segment embryos')
    parser.add_argument('--emb_dir', type=str, help='Path to embedding predictions directory', required=True)
    parser.add_argument('--clustering', type=str, help='Clustering algorithm: ms or hdbscan', required=True)
    parser.add_argument('--seg_ds', type=str, help='Output seg dataset name', required=True)
    parser.add_argument('--delta_var', type=float, help='delta_var param', default=0.5)
    parser.add_argument('--min_size', type=int, help='HDBSCAN min_size param', default=50)
    parser.add_argument('--remove_largest', help='Remove largest instance (BG)', action='store_true')
    parser.add_argument('--expand_labels', help='Expand labels with watershed', action='store_true')
    parser.add_argument('--min_instance_size', type=int, help='Min instance size filtering', required=False,
                        default=None)

    args = parser.parse_args()

    assert os.path.isdir(args.emb_dir)

    for file_path in glob.glob(os.path.join(args.emb_dir, '*predictions.h5')):
        _, filename = os.path.split(file_path)

        print(f'Processing {filename}')

        with h5py.File(file_path, 'r+') as f:
            raw_sequence = f['raw_sequence'][:]
            embedding_sequence = f['embedding_sequence1'][:]
            seg_sequence = []

            i = 0
            for raw, emb in zip(raw_sequence, embedding_sequence):
                i += 1
                print(f'Processing patch {i}')
                seg = run_clustering(emb, args.clustering, args.delta_var, args.min_size, args.expand_labels,
                                     args.remove_largest)
                seg_sequence.append(seg)

            if args.seg_ds in f:
                del f[args.seg_ds]

            segments = np.stack(seg_sequence, axis=0)
            f.create_dataset(args.seg_ds, data=segments, compression='gzip')

    print('Done')


#import networkx as nx
#import matplotlib.pyplot as plt
import json
import pprint
from TwitterModule import *
import time
from datetime import datetime
#Set up api and global variables
twitter_api = oauth_login()#twitter api for grabbing data
#dates = [330,331,401,402,403]
dates = [401,402,403,404,405,406,407]

for day in dates:
    print(day)
    names = ['@itsnotdrew','@davidhogg111','@IngrahamAngle','@sleepnumber','@ATT','@Allstate','@esurance','@Bayer','@RocketMortgage','@LibertyMutual','@Arbys','@TripAdvisor','@Nestle','@hulu','@Wayfair','@FoxNews','#BoycottIngramAdverts','#boycottLauraIngraham','#FireIngraham','#FireLauraIngraham']
    errorLogName = 'errorLog' + str(day) + '_4' +  '.txt'
    errorLog = open(errorLogName,'w')
    for q in names:
        try:
            dateStr = str(day)
            dateDay = dateStr[1:]
            dateDayPlusOne = str(int(dateDay)+1)
            dateMonth = dateStr[0]

            if (dateStr == '331'): #dirty code to fix a logic bug when switching months
                dateDayPlusOne = '01'
                dateMonth = '4'

            until = '2018-0' + dateMonth + '-' + dateDayPlusOne
            tweetsDicitonary = {}
            name = q[1:]
            nameFile = name + dateStr +'_4'+  '.json'
            file = open(nameFile,'w')

            '''
            First search call to twitter_api
            Parameters:
                q: is the search term
                result_type: is whether we want recent, popular or mixed tweets. Currently set to recent
                max_results: is how many results we wan to take in a single call. Is currently 10 for testing
                until: specifies the date that all tweets returned form this call should come before
                        (so all tweets from this call are from 3/28/2018)

            getMaxID parses the maxID from the appropriate string in the search return metadata
            maxid will then be used to call the next batch of tweets. More info on maxid is Available on the search api documentation
            '''
            print(q + 'at ' + str(datetime.now())) #prints twitter user being processed

            response = make_twitter_request(twitter_api.search.tweets,q=q,result_type='recent',count=5, until=until)
            try:
                next_results = response['search_metadata']['next_results']
                getMaxID = dict([ kv.split('=') for kv in next_results[1:].split("&") ])
                maxid = getMaxID['max_id']
            except:
                next_results = ""
                maxid = 0
                line = "\nretrieval error at " + str(datetime.now()) + " while processing beginning call of " + q
                errorLog.write(line)

            '''
            Parameters in response:
                most are the same
                -result_type is mixed (testing)
                -max_results is 100 (testing, but really it should be kept like this)
                -max_id field is at the end of the call, allowing each call of the function to retrieve older and older tweets

            time.sleep(5): Can only call the search api 180 times in 15 minutes, so ~5 seconds. Right now set to one because testing, but should probably be set to 10self.
                            Or, we can edit the make-twitter_request function to handle this error for us

            '''
            for i in range(1,101): #top possible tweets 10,000
                #print(i) #testing code
                try:
                    response = make_twitter_request(twitter_api.search.tweets,q=q,result_type='recent',count=100,until=until,max_id=maxid)
                    next_results = response['search_metadata']['next_results']
                    if (next_results == None):
                        break
                    getMaxID  = dict([ kv.split('=') for kv in next_results[1:].split("&") ])#to get the nextID
                    maxid = getMaxID['max_id']
                   # print(maxid)
                    time.sleep(5)
                except:
                    line = "\nretrieval error at " + str(datetime.now()) + " while processing " + q + ' at loop number ' + str(i)
                    errorLog.write(line)
                    break
                for tweet in response['statuses']:#add each tweet to a dictionary
                    try:
                        tweetsDicitonary[tweet['id']] = tweet
                    except:
                        line = "\ndicitonary error at " + str(datetime.now()) + " while processing " + str(tweet['id'])
                        errorLog.write(line)
                file.seek(0)
            file.seek(0)
            json.dump(tweetsDicitonary,file)
            file.close()
        except:
            line = "\nFatal error at " + str(datetime.now()) + " while processing " + q
            errorLog.write(line)
            json.dump(tweetsDicitonary,file)
            file.close()


from typing import final
import pandas as pd
import numpy as np
import os

final_df=pd.read_csv("prepared_final_data.csv")

print(final_df)
values=final_df["pollution"].values
print(values)
print(final_df.columns)
"""# Normalized the data"""

from sklearn.preprocessing import MinMaxScaler

# values = final_df.values
print(values)
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_dataset = scaler.fit_transform(values.reshape(-1,1))

scaled_dataset

# Creating a window for previous data
def to_supervised(window_size,train):
  X = []
  Y = []
  for i in range(window_size, len(train)):
    X.append(train[i-window_size:i,:])
    Y.append(train[i,0:1])

  return np.array(X), np.array(Y)

feature,label = to_supervised(window_size=5, train=scaled_dataset)

n_train = 24*365
X_train, X_test = feature[n_train:,] , feature[:n_train,]
print('X_train' ,X_train.shape)
print('X_test' ,X_test.shape)

Y_train, Y_test = label[n_train:,] , label[:n_train,]
print('Y_train' ,Y_train.shape)
print('Y_test' ,Y_test.shape)

import keras

from keras.models import Sequential
from keras.layers import Dense, Dropout,LSTM

model = Sequential()

model.add(LSTM(units = 50, return_sequences = True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(0.2))

model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))

model.add(LSTM(units = 50))
model.add(Dropout(0.2))

model.add(Dense(units = 1))

model.compile(optimizer = 'adam', loss = 'mean_squared_error')

from keras.callbacks import EarlyStopping

es_callback = EarlyStopping(monitor='val_loss', patience=3,min_delta=0.01)
path = 'air_pollution_forecasting_model'

isdir = os.path.isdir(path)
print(isdir)
if isdir:
    reconstructed_model = keras.models.load_model("air_pollution_forecasting_model")
    model = reconstructed_model
else:
    model.fit(X_train, Y_train, validation_split = 0.1, epochs = 10, batch_size = 32, callbacks=[es_callback])
    model.save("air_pollution_forecasting_model")
breakpoint()
Y_pred = np.round(model.predict(X_test),2)

from sklearn.metrics import mean_squared_error
mse = mean_squared_error(Y_test, Y_pred)

rmse = np.sqrt(mse)
print(rmse)

# Scaling back to the original scale
d = scaled_dataset[:8760,:]
print('dummy',d.shape)
print('Y_pred',Y_pred.shape)
Y_predicted = np.concatenate((Y_pred,d[:8760,1:]), axis =1)
print('concat y_pred',Y_pred.shape)
Y_tested = np.concatenate((Y_test, d[:8760,1:]), axis = 1)
print('concat Y_test', Y_test.shape)

Y_predicted = scaler.inverse_transform(Y_predicted)
Y_tested = scaler.inverse_transform(Y_tested)
Y_predicted = Y_predicted[:,0:1]
Y_tested = Y_tested[:,0:1]
print('Y_tested', Y_tested.shape)
print('Y_predicted', Y_predicted.shape)

import matplotlib.pyplot as plt

plt.plot(Y_predicted[:100,:], color= 'green')
plt.plot(Y_tested[:100,:] , color = 'red')
plt.title("Air Pollution Prediction (Multivariate)")
plt.xlabel("Date")
plt.ylabel("Pollution level")
plt.savefig("results.png")

import pickle
pickle.dump(scaler, open('min_max_scaler.pkl','wb'))


"""
A collection of neural network code. The first part of the script includes
blocks, which are the building blocks of our models. The second part includes
the actual Pytorch models.
"""
import torch
import torchvision.transforms as transforms

class ConvBlock(torch.nn.Module):
    """
    A ConvBlock represents a convolution. It's not just a convolution however,
    as some common operations (dropout, activation, batchnorm, 2x2 pooling)
    can be set and run in the order mentioned.
    """

    def __init__(
        self,
        dim,
        n_out,
        kernel_size=3,
        stride=1,
        padding=1,
        batchnorm=False,
        dropout=0,
        activation=True,
    ):
        """ A convolution operation """
        super(ConvBlock, self).__init__()
        n_in = int(dim[0])
        self.conv2d = torch.nn.Conv2d(
            n_in, n_out, kernel_size=kernel_size, stride=stride, padding=padding
        )
        self.batchnorm = torch.nn.BatchNorm2d(n_out) if batchnorm else None
        self.activation = torch.nn.ReLU(inplace=True) if activation else None
        self.dropout = torch.nn.Dropout2d(dropout) if dropout else None
        dim[0] = n_out
        dim[1:] = 1 + (dim[1:] + padding * 2 - kernel_size) // stride
        self.n_params = n_out * (n_in * kernel_size * kernel_size + (3 if batchnorm else 1))
        print(
            "Conv2d in %4i out %4i h %4i w %4i k %i s %i params %9i"
            % (n_in, *dim, kernel_size, stride, self.n_params)
        )

    def forward(self, batch):
        """ Forward the 4D batch """
        out = self.conv2d(batch)
        if self.activation:
            out = self.activation(out)
        if self.batchnorm:
            out = self.batchnorm(out)
        if self.dropout:
            out = self.dropout(out)
        return out

class LinearBlock(torch.nn.Module):
    """
    A LinearBlock represents a fully connected layer. It's not just this, as
    some common operations (dropout, activation, batchnorm) can be set and run
    in the order mentioned.
    """

    def __init__(self, dim, n_out, batchnorm=False, dropout=0.0, activation=True):
        """ A fully connected operation """
        super(LinearBlock, self).__init__()
        n_in = int(dim[0])
        self.linear = torch.nn.Linear(n_in, n_out)
        dim[0] = n_out if type(n_out) in (int, float) else n_out[0]
        self.batchnorm = torch.nn.BatchNorm1d(dim[0]) if batchnorm else None
        self.dropout = torch.nn.Dropout(dropout) if dropout > 0.0 else None
        self.activation = torch.nn.ReLU(inplace=True) if activation else None
        self.n_params = n_out * (n_in + (3 if batchnorm else 1))
        print(
            "Linear in %4i out %4i                       params %9i" % (n_in, n_out, self.n_params)
        )

    def forward(self, batch):
        """ Forward the 2D batch """
        out = self.linear(batch)
        if self.activation:
            out = self.activation(out)
        if self.batchnorm:
            out = self.batchnorm(out)
        if self.dropout:
            out = self.dropout(out)
        return out

class PoolBlock(torch.nn.Module):
    """
    A PoolBlock is a pooling operation that happens on a matrix, often between
    convolutional layers, on each channel individually. By default only two are
    supported: max and avg.
    """

    def __init__(self, dim, pool="max", size=None, stride=None):
        """ A pooling operation """
        super(PoolBlock, self).__init__()

        stride = size if stride is None else stride
        if size:
            dim[1:] //= stride
        else:
            size = [int(x) for x in dim[1:]]
            dim[1:] = 1
        if pool == "max":
            self.pool = torch.nn.MaxPool2d(size, stride=stride, padding=0)
        elif pool == "avg":
            self.pool = torch.nn.AvgPool2d(size, stride=stride, padding=0)
        self.n_params = 0

    def forward(self, batch):
        """ Forward the 4D batch """
        out = self.pool(batch)
        return out

class ViewBlock(torch.nn.Module):
    """
    A ViewBlock restructures the shape of our activation maps so they're
    represented as 1D instead of 3D.
    """

    def __init__(self, dim, shape=-1):
        """ A reshape operation """
        super(ViewBlock, self).__init__()
        self.shape = shape
        if self.shape == -1:
            dim[0] = dim[0] * dim[1] * dim[2]
            dim[-2] = 0
            dim[-1] = 0
        else:
            dim[:] = shape

        self.n_params = 0
        print("View             d %4i h %4i w %4i" % (*dim,))

    def forward(self, batch):
        """ Forward the 4D batch into a 2D batch """
        return batch.view(batch.size(0), self.shape)

class Tiny(torch.nn.Module):
    """ A small and quick model """

    def __init__(self, in_dim, n_status, n_out):
        """
        Args:
            in_dim (list): The input size of each example
            n_status (int): Number of status inputs to add
            n_out (int): Number of values to predict
        """
        super(Tiny, self).__init__()
        self.n_status = n_status
        dim = in_dim.copy()
        self.feat = torch.nn.Sequential(
            ConvBlock(dim, 16),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 32),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 48),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 64),
            PoolBlock(dim, "max", 2),
        )
        self.view = ViewBlock(dim)
        dim[0] += n_status
        self.head = torch.nn.Sequential(LinearBlock(dim, n_out, activation=False))
        self.n_params = sum([x.n_params for x in self.feat]) + sum([x.n_params for x in self.head])
        print("Tiny                                          params %9i" % self.n_params)

    def forward(self, batch, status):
        """
        Args:
            batch (4D tensor): A batch of camera input.
            status (1D tensor): Status inputs indicating things like speed.
        """
        out = self.feat(batch)
        out = self.view(out)
        if self.n_status:
            out = torch.cat((out, status), 1)
        out = self.head(out)
        return out

class StarTree(torch.nn.Module):
    """
    A medium-sized model that uses layers with few activation maps to
    efficiently increase the number of layers, and therefore nonlinearities.
    """

    def __init__(self, in_dim, n_status, n_out):
        """
        Args:
            in_dim (list): The input size of each example
            n_status (int): Number of status inputs to add
            n_out (int): Number of values to predict
        """
        super(StarTree, self).__init__()
        self.n_status = n_status
        dim = in_dim.copy()
        self.feat = torch.nn.Sequential(
            ConvBlock(dim, 64, dropout=0.25),
            ConvBlock(dim, 16),
            ConvBlock(dim, 32),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 24),
            ConvBlock(dim, 48),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 32),
            ConvBlock(dim, 64),
            PoolBlock(dim, "max", 2),
            ConvBlock(dim, 40),
            ConvBlock(dim, 80, dropout=0.25),
            PoolBlock(dim, "max", 2),
        )
        self.view = ViewBlock(dim)
        dim[0] += n_status
        self.head = torch.nn.Sequential(
            LinearBlock(dim, 50), LinearBlock(dim, n_out, activation=False),
        )
        self.n_params = sum([x.n_params for x in self.feat]) + sum([x.n_params for x in self.head])
        print("StarTree                                  params %9i" % self.n_params)

    def forward(self, batch, status):
        """
        Args:
            batch (4D tensor): A batch of camera input.
            status (1D tensor): Status inputs indicating things like speed.
        """
        out = self.feat(batch)
        out = self.view(out)
        if self.n_status:
            out = torch.cat((out, status), 1)
        out = self.head(out)
        return out

def train_epoch(device, model, optimizer, criterion, loader):
    """ Run the optimzer over all batches in an epoch """
    model.train()
    epoch_loss = 0
    batch_index = 0
    for batch_index, (examples, statuses, labels) in enumerate(loader):
        optimizer.zero_grad()
        guesses = model(examples.to(device), statuses.to(device))
        loss = criterion(guesses, labels.to(device))
        loss.backward()
        optimizer.step()
        epoch_loss += loss.item()
    return epoch_loss / (batch_index + 1)

def test_epoch(device, model, criterion, loader):
    """ Run the evaluator over all batches in an epoch """
    model.eval()
    epoch_loss = 0
    batch_index = 0
    with torch.no_grad():
        for batch_index, (examples, statuses, labels) in enumerate(loader):
            guesses = model(examples.to(device), statuses.to(device))
            loss = criterion(guesses, labels.to(device))
            epoch_loss += loss.item()
    return epoch_loss / (batch_index + 1)

def compose_transforms(transform_config):
    """ Apply all image transforms """
    transform_list = []
    for perturb_config in transform_config:
        if perturb_config["name"] == "colorjitter":
            transform = transforms.ColorJitter(
                brightness=perturb_config["brightness"],
                contrast=perturb_config["contrast"],
                saturation=perturb_config["saturation"],
                hue=perturb_config["hue"],
            )
            transform_list.append(transform)
    transform_list.append(transforms.ToTensor())
    return transforms.Compose(transform_list)


import sys
import json
import time
import numpy as np
import argparse

from operator import itemgetter
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import dok_matrix
from math import sqrt
from math import log

from upper_learning_corpus import LearningCorpus
from sparse_matrix import *
from ranking import *

def convert_counts_to_pmi2(matrix, rowSum, colSum):

    totalSum = sum(rowSum.values())
    sys.stderr.write('Converting to csc_matrix format... ')
    startTime = time.time()
    matrix = coo_matrix(matrix)
    sys.stderr.write('done. Time taken: '+str(time.time()-startTime)+' secs\n')
    totalEntries = len(matrix.row)
    sys.stderr.write('Num entries: '+str(totalEntries)+'\n')

    numEntries = 1.
    # symmetric matrix
    for r, c, val in zip(np.nditer(matrix.row), np.nditer(matrix.col), np.nditer(matrix.data, op_flags=['readwrite'])):
        pi, pj, pij = (1.*val/rowSum[str(r)], 1.*val/colSum[str(c)], 1.*val/totalSum)
        val[...] = log(pij/(pi*pj))

        if numEntries% 1000000 == 0: sys.stderr.write(str(numEntries)+' ')
        numEntries += 1

    sys.stderr.write('done!\n')
    return csc_matrix((matrix.data, (matrix.row, matrix.col)), shape=matrix.shape)

def convert_counts_to_pmi(matrix, rowSum, colSum):

    totalSum = sum(rowSum.values())
    sys.stderr.write('Converting to dok_matrix format... ')
    startTime = time.time()
    matrix = dok_matrix(matrix)
    sys.stderr.write('done. Time taken: '+str(time.time()-startTime)+' secs\n')
    totalEntries = len(matrix)
    sys.stderr.write('Num entries: '+str(totalEntries)+'\n')

    r, c = matrix.shape
    numEntries = 1.
    # symmetric matrix
    if r == c:
        for key, val in matrix.iteritems():
            i, j = key
            i, j = (str(i), str(j))
            if int(i) <= int(j):
                pi, pj, pij = (1.*val/rowSum[i], 1.*val/colSum[j], 1.*val/totalSum)
                pmi = log(pij/(pi*pj))
                matrix[int(i), int(j)] = pmi
                matrix[int(j), int(i)] = pmi
            else:
                pass

            if numEntries% 1000000 == 0: sys.stderr.write(str(numEntries)+' ')
            numEntries += 1
    else:
        for key, val in matrix.iteritems():
            i, j = key
            i, j = (str(i), str(j))
            pi, pj, pij = (1.*val/rowSum[i], 1.*val/colSum[j], 1.*val/totalSum)
            matrix[int(i), int(j)] = log(pij/(pi*pj))

            if numEntries% 1000000 == 0: sys.stderr.write(str(numEntries)+' ')
            numEntries += 1

    sys.stderr.write('done!\n')
    return csc_matrix(matrix)

if __name__=='__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("-m", "--matrixfile", type=str, default=None, help="Matrix file name")
    parser.add_argument("-d", "--dictfile", type=str, help="Dictionary file name")
    parser.add_argument("-o", "--outputfile", type=str, default=None, help="Output file name")

    args = parser.parse_args()

    outFileName = args.outputfile
    dictFile = open(args.dictfile, 'r')
    values = dictFile.readline().strip().split()
    if len(values) == 3:
        colCutoff, rowCutoff, windowSize = values
    else:
        colCutoff, windowSize = values
        rowCutoff = 0.

    vocab = json.loads(dictFile.readline())
    wordFeatures = json.loads(dictFile.readline())
    rowSum = json.loads(dictFile.readline())
    colSum = json.loads(dictFile.readline())
    contextMat = load_sparse_matrix(args.matrixfile)

    sys.stderr.write("windowSize: "+str(windowSize)+" colCutoff: "+str(colCutoff)+" rowCutoff: "+str(rowCutoff)+'\n')
    sys.stderr.write("featLen: "+str(len(wordFeatures))+" vocabLen: "+str(len(vocab))+'\n')
    sys.stderr.write('Read the matrix!\n')

    ''' Convert the matrix here! '''
    contextMat = convert_counts_to_pmi(contextMat, rowSum, colSum)

    sys.stderr.write('Writing the matrix now... ')
    if outFileName is None: outFileName = args.dictfile.replace('.dict', '_pmi')
    save_sparse_matrix(outFileName, contextMat)
    sys.stderr.write('done!\n')


from flask import abort
from flask_restx import Resource, Namespace, Model, fields, reqparse
from infraestructura.alumnos_repo import AlumnosRepo
from api.cursos_api import modeloCurso

from flask_restx.inputs import date

repo = AlumnosRepo()

nsAlumno = Namespace('Alumnos', description='Administrador de Alumno')

modeloAlumnoSinID = Model('AlumnoSinID',{
    'nombre': fields.String(),
    'direccion': fields.String(),
    'sexo':fields.String(),
    'edad':fields.Integer(),
    'fecha_baja': fields.Date()
    })

modeloAlumno = modeloAlumnoSinID.clone('Alumno',{
    'id': fields.Integer(),
    #'cursos': fields.Nested(modeloCurso, skip_none=True)
})

# modeloBusqueda = Model('BusquedaFechas', {
#     'desde': fields.Date(),
#     'hasta': fields.Date()
# })
nsAlumno.models[modeloAlumno.name] = modeloAlumno
nsAlumno.models[modeloAlumnoSinID.name] = modeloAlumnoSinID
# nsAlumno.models[modeloBusqueda.name] = modeloBusqueda

nuevoAlumnoParser = reqparse.RequestParser(bundle_errors=True)
nuevoAlumnoParser.add_argument('nombre', type=str, required=True)
nuevoAlumnoParser.add_argument('direccion', type=str, required=True)
nuevoAlumnoParser.add_argument('sexo', type=str, required=True)
nuevoAlumnoParser.add_argument('edad', type=int, required=True)
nuevoAlumnoParser.add_argument('fecha_baja', type=date, required=False)

editarAlumnoParser = nuevoAlumnoParser.copy()
editarAlumnoParser.add_argument('id', type=int, required=True)

@nsAlumno.route('/')
class AlumnosResource(Resource):
    # @nsAlumno.marshal_list_with(modeloAlumno)
    # def get(self):
    #     return repo.get_all()

    @nsAlumno.marshal_list_with(modeloAlumno)
    def get(self):
        return repo.get_all()

    @nsAlumno.expect(modeloAlumnoSinID)
    @nsAlumno.marshal_with(modeloAlumno)
    def post(self):
        data = nuevoAlumnoParser.parse_args()
        Alumno = repo.agregar(data)
        if Alumno:
            return Alumno, 201
        abort(500)

@nsAlumno.route('/<int:id>')
class AlumnoResource(Resource):
    @nsAlumno.marshal_with(modeloAlumno)
    def get(self, id):
        Alumno = repo.get_by_id(id)
        if Alumno:
            return Alumno, 200
        abort(404)

    @nsAlumno.expect(modeloAlumno)
    def put(self, id):
        data = editarAlumnoParser.parse_args()
        if repo.modificar(id, data):
            return 'Alumno actualizado', 200
        abort(404)

# @nsAlumno.route('/buscar/<string:desde>/<string:hasta>/')
# class AlumnoResource(Resource):
#     @nsAlumno.marshal_list_with(modeloAlumno)
#     def get(self, desde, hasta):
#         l = repoLep.buscar(desde, hasta)
#         if l:
#              a = []
#              for x in l:
#               h = repo.get_by_id(x.Alumno_id)
#              a.append(h)
#              return l, 200
#         abort(404)

@nsAlumno.route('/baja/<int:id>')
class AlumnoResource(Resource):

    def put(self, id):
        if repo.baja(id):
            return 'Alumno dado de baja', 200
        abort(400)

@nsAlumno.route('/buscar/<int:curso>')
class AlumnoResource(Resource):
    @nsAlumno.marshal_list_with(modeloAlumno)
    def get(self, curso):
        l = repo.get_alumno_curso(curso)
        if l:

             return l, 200
        abort(404)


import pandas as pd

def read_fasta(file_path):
    sequences = {"Header": [], "Sequence": []}
    current_header = None
    current_sequence = ""

    with open(file_path, "r") as file:
        for line in file:
            line = line.strip()
            if line.startswith(">"):
                # New header found
                if current_header is not None:
                    sequences["Header"].append(current_header)
                    sequences["Sequence"].append(current_sequence)
                current_header = line[1:]
                current_sequence = ""
            else:
                # Continue building the sequence
                current_sequence += line

        # Add the last sequence
        if current_header is not None:
            sequences["Header"].append(current_header)
            sequences["Sequence"].append(current_sequence)

    return pd.DataFrame(sequences)

def extract_label(header):
    # Extract label after the "|" symbol
    parts = header.split("|")
    if len(parts) > 1:
        return parts[1].strip()
    else:
        return None

file_path = "data/pharos/pharos.fasta"
fasta_df = read_fasta(file_path)

fasta_df["Label"] = fasta_df["Header"].apply(extract_label)
tclin_df = fasta_df[fasta_df["Label"] == "Tclin"]
tdark_df = fasta_df[fasta_df["Label"] == "Tdark"]
length_tclin_df = len(tclin_df)

random_tdark_df = tdark_df.sample(n=length_tclin_df, random_state=42)

from sklearn.model_selection import train_test_split
import os

# Assuming tclin_df and tdark_df are already defined

# Define the test size
test_size = 0.2

# Split the positive sequences (Tclin) into train and test sets
tclin_train, tclin_test = train_test_split(
    tclin_df, test_size=test_size, random_state=42
)

# Split the negative sequences (Tdark) into train and test sets
tdark_train, tdark_test = train_test_split(
    random_tdark_df, test_size=test_size, random_state=42
)

# Create folders if they don't exist

train_folder = "data/pharos/fastadata/Train"
test_folder = "data/pharos/fastadata/Independent_Test"

# Create folders if they don't exist
for folder in [train_folder, test_folder]:
    if not os.path.exists(folder):
        os.makedirs(folder)

if not os.path.exists(train_folder):
    os.makedirs(train_folder)
if not os.path.exists(test_folder):
    os.makedirs(test_folder)

# Function to extract header before the '|' symbol
def extract_header(identifier):
    return identifier.split("|")[0]

# Function to write sequences to fasta file
def write_fasta(filename, dataframe):
    with open(filename, "w") as file:
        for index, row in dataframe.iterrows():
            header = extract_header(row["Header"])
            file.write(f">{header}\n{row['Sequence']}\n")

# Save the sequences to FASTA files in the train and test folders
write_fasta(os.path.join(train_folder, "positive_train_sequence.fasta"), tclin_train)
write_fasta(os.path.join(test_folder, "positive_test_sequence.fasta"), tclin_test)
write_fasta(os.path.join(train_folder, "negative_train_sequence.fasta"), tdark_train)
write_fasta(os.path.join(test_folder, "negative_test_sequence.fasta"), tdark_test)


import cv2
import os

# Input folder containing the saved images
image_folder = '/Users/tobieabel/Desktop/video_frames/ConcatVideo/'

# Output video file path
output_video_path = '/Users/tobieabel/Desktop/video_frames/Youtube/v3_a demo.mp4'

# Get the list of image files in the input folder
image_files = os.listdir(image_folder)
image_files.remove('.DS_Store')
def file_sort_key(filename):
    # Extract the numeric portion of the filename
    number = int(os.path.splitext(filename)[0])

    return number

# Sort the files chronologically
sorted_files = sorted(image_files, key=file_sort_key)
print(sorted_files)
# Get the dimensions of the first image to initialize the video writer
first_image_path = os.path.join(image_folder, sorted_files[0])
first_image = cv2.imread(first_image_path)
height, width, _ = first_image.shape #need to be careful of this.  I scrapped a video from youtube whose resolution was an odd width 1740, height 988
#and these dimensions didn't work with cv2.VideoWriter so I had to use cv2.resize to change the images to 1920x1080 which is the closest accepatable format

# Define the codec and create the video writer
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
fps = 30  # Adjust as needed
video_writer = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height))

# Loop through the image files and write them to the video
for i in sorted_files:
    image_path = os.path.join(image_folder, i)
    image = cv2.imread(image_path)

    # Write the image to the video writer
    video_writer.write(image)

# Release the video writer
video_writer.release()

print(f"Video saved to: {output_video_path}")


from typing import Any, Dict, List, Type, TypeVar, Union

from attrs import define as _attrs_define
from attrs import field as _attrs_field

from ..types import UNSET, Unset

T = TypeVar("T", bound="FollowUpPriorityV2ResponseBody")

@_attrs_define
class FollowUpPriorityV2ResponseBody:
    """
    Example:
        {'description': 'A follow-up that requires immediate attention.', 'id': '01GNW4BAQ7XRMFF6FHKNXDFPRW', 'name':
            'Urgent', 'rank': 10}

    Attributes:
        id (str): Unique identifier for the follow-up priority option Example: 01GNW4BAQ7XRMFF6FHKNXDFPRW.
        name (str): Name of the follow-up priority option Example: Urgent.
        rank (int): Rank is used to order the follow-up priority options correctly Example: 10.
        description (Union[Unset, str]): Description of the follow-up priority option Example: A follow-up that requires
            immediate attention..
    """

    id: str
    name: str
    rank: int
    description: Union[Unset, str] = UNSET
    additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        id = self.id
        name = self.name
        rank = self.rank
        description = self.description

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "id": id,
                "name": name,
                "rank": rank,
            }
        )
        if description is not UNSET:
            field_dict["description"] = description

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        id = d.pop("id")

        name = d.pop("name")

        rank = d.pop("rank")

        description = d.pop("description", UNSET)

        follow_up_priority_v2_response_body = cls(
            id=id,
            name=name,
            rank=rank,
            description=description,
        )

        follow_up_priority_v2_response_body.additional_properties = d
        return follow_up_priority_v2_response_body

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


from jinja2 import Environment, BaseLoader
from io import BytesIO
import plotly
import base64

'''
export = ExportHTML('testclass.html')
export.render()
'''

class ExportHTML:
    __template_vars = {'title':'Hello World','body':'Hello World !!!'}
    __template_html = '''

            <meta charset="UTF-8">
            <title>{{ title }}</title>

Header

             {{ body }}

Next Page

        '''

    def encode_graph(self, fig):
        tmpfile = BytesIO()
        fig.savefig(tmpfile, format='png', bbox_inches='tight')
        encoded = base64.b64encode(tmpfile.getvalue()).decode('utf-8')

        fig_html = ''.format(encoded)
        return fig_html

    def plotly_img_uri(self, fig, height=300, width=1200, orca_path='C:/Users/Administrator/anaconda3/orca_app/orca.exe'):
        plotly.io.orca.config.executable = orca_path

        img_uri = base64.b64encode(plotly.io.to_image(fig, width=width, height=height)).decode('ascii')
        return ''.format(width=width, height=height, img_uri=img_uri)

    @property
    def template_vars(self):
        return self.__template_vars

    @template_vars.setter
    def template_vars(self, var_dict):
        self.__template_vars = var_dict

    @property
    def template_html(self):
        return self.__template_html

    @template_html.setter
    def template_html(self, htmlString):
        self.__template_html = htmlString

    def render(self, output_file):
        template = Environment(loader=BaseLoader()).from_string(self.template_html)
        template_vars = self.template_vars
        html_out = template.render(template_vars)

        with open(output_file, "w") as fh:
            fh.write(html_out)


import matplotlib.pyplot as plt
tiempo = [0,1,2,3,4,5]
sensor = [4,5,6,8,9, 10]
plt.plot(tiempo,sensor,'--,r')
plt.title('Grafico sensor contra el tiempo')
plt.xlabel('Tiempo(s)')
plt.ylabel('Voltaje(v)')
plt.savefig('sensor.png')
plt.show()

# Nota: se le puede poner el simbolo para que se grafique('--'), si no se pone nada se grafica como una linea recta

# DICCIONARIO
diccionario = {}
diccionario['NombresEstudiantes'] = ['Andrea', 'Nicolle', 'Isabel', 'Santiago']
diccionario['EdadEstudiantes'] = [18,20,19,15]
diccionario['Peso'] = [60,55,70,78]
print(diccionario)

print(diccionario['NombresEstudiantes'][-1],diccionario['EdadEstudiantes'][-1],diccionario['Peso'][-1])


import time
import constants as cons
import matplotlib.pyplot as plt
from preprocessing.images_reader import ImagesReader

start_time = time.time()

print('reading images...')
reader = ImagesReader(cons.PREPROCESSED_DATASET_DIR)

train_images = reader.read_train_images()

classes = [None] * len(train_images)
samples = [None] * len(train_images)

for i, image_class in enumerate(train_images):
    classes[i] = image_class
    samples[i] = len(train_images[image_class])

plt.plot(classes, samples)
plt.show()

end_time = time.time()
print('done in {:.2f}s'.format(end_time - start_time))