import time
import pygame
from Bate import Bate
from Ball import Ball
import bloques

pygame.init()

# Creo la ventana del juego
WIDTH = 640
HEIGHT = 490
ventana = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("ARKANOID")
fps = 60
clock = pygame.time.Clock()

# Crea el objeto bate y obtengo su rectángulo
bate = Bate(7,"imagenes/torch.png")
bate.rect.move_ip(240,450)

# Creo la pelota y le damos la velocidad y posición inicial
ball = Ball([2,2],"imagenes/ball.png")
ball.rect.move_ip(222,222)
ballrect = ball.rect.move(ball.speed)

# Creo los bloques del juego y su patrón de aparición
listaBloques = []
for pos_x in range(22):
    for pos_y in range(5):
        listaBloques.append(bloques.Block(34 * pos_x, 34 * pos_y, "imagenes/ladrillo1.png"))

# Creo el fondo del juego con una imágen
fondo = pygame.image.load("imagenes/fondo.png")
fondorect = fondo.get_rect()

# Inserto la música de fondo y la repetimos para que no se pare
musica = pygame.mixer.Sound("sonidos/minecraftmusic.mp3")
pygame.mixer.Sound.play(musica, 5)

# Muevo la pelota y compruebo si llega a los bordes de la ventana
ballrect = ball.rect.move(ball.speed)

# Inserto el bucle principal del juego
jugando = True
while jugando:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            jugando = False

     # Compruebo si se ha pulsado alguna tecla
    keys = pygame.key.get_pressed()
    ballrect = ballrect.move(ball.speed)
    # Dibujo de nuevo los elementos del juego
    ventana.blit(fondo, (0, 0))
    ventana.blit(bate.imagen, bate.rect)
    # Indico que los bloques se borren de la lista y le añado un sonido
    for Block in listaBloques:
        ventana.blit(Block.image, Block.rect)
        if ballrect.colliderect(Block.rect):
            listaBloques.remove(Block)
            sonido_ladrillo = pygame.mixer.Sound("sonidos/break_sound.mp3")
            pygame.mixer.Sound.play(sonido_ladrillo)
            ball.speed[1] = -ball.speed[1]

    # Establezco los límites de los márgenes para la barra
    if keys[pygame.K_LEFT] and bate.rect.left > 0:
        bate.rect.left -= bate.speed
    if keys[pygame.K_RIGHT] and bate.rect.right < 640:
        bate.rect.right += bate.speed

    # Compruebo si hay colisión entre el bate y la pelota
    if bate.rect.colliderect(ballrect):
        ball.speed[1] = -ball.speed[1]
    ballrect = ballrect.move(ball.speed)
    if ballrect.left < 0 or ballrect.right > ventana.get_width():
        ball.speed[0] = -ball.speed[0]
    if ballrect.top < 0 or ballrect.bottom > ventana.get_height():
        ball.speed[1] = -ball.speed[1]
    ventana.blit(ball.imagen, ballrect)

    # Indico que los bloques se rompan con los impactos de la pelota
    if ballrect.colliderect(Block.rect):
        listaBloques.pop(Block)

    # Añado la imagen que salta cuando pierdas
    if ballrect.bottom > 480:
        youlose = pygame.image.load("imagenes/youlose.png")
        youloserect = youlose.get_rect()
        ventana.blit(youlose, (0, 0))
        sonido_youlose = pygame.mixer.Sound("sonidos/youlose.mp3")
        pygame.mixer.Sound.play(sonido_youlose)
        jugando = False

    # Añado una imágen que salta cuando ganas
    if len(listaBloques) == 0:
        youwin = pygame.image.load("imagenes/youwin.jpg")
        youwinrect = youwin.get_rect()
        ventana.blit(youwin, (0, 0))
        sonido_youwin = pygame.mixer.Sound("sonidos/youwin.mp3")
        pygame.mixer.Sound.play(sonido_youwin)
        jugando = False

    # Recargo los elementos de la pantalla y la tasa de refresco
    pygame.display.flip()
    clock.tick(fps)

# Establezco el tiempo de espera para cerrar la ventana
time.sleep(5)
pygame.quit()


from setuptools import setup, find_packages

with open("README.md", "r") as readme:
    long_description = readme.read()

setup(
    name='fasrc_sample_tools',
    version="0.0.1",
    url='https://github.com/fasrc/tools_for_cwl_sample_workflows',
    license='',
    author='FAS RC Research Computing',
    author_email='mbouzinier@g.harvard.edu',
    description='Sample CWL Workflow',
    long_description=long_description,
    long_description_content_type="text/markdown",
    package_dir={
        "fasrc_sample_tools": "./src/fasrc_sample_tools"
    },
    install_requires=[
        "fiona",
        "geopandas",
        "pandas",
        "pygeos",
        "shapely"
    ],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: Harvard University :: Development",
        "Operating System :: OS Independent",
    ]
)


"""
Note:
    1. As per the discussion with Mahendra San , output file of this job is not changed to hive style partition
    due to business reason as everytime all the files are getting processed
    Jira - https://marutide.atlassian.net/browse/M1-97
    2. Refactoring only includes paramterizing the input
"""

import awswrangler as wr
import pandas as pd
import numpy as np
import boto3
from datetime import date, datetime, timedelta
import logging
import sys
from awsglue.utils import getResolvedOptions

log = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s :: %(message)s', level=logging.INFO)
logging.info("info")

if __name__ == '__main__':
    try:

        args = getResolvedOptions(sys.argv,
                                  [
                                      "redshift_connection_string",
                                      "input_location_path",
                                      "stage_file_path",
                                      "redshift_table",
                                      "redshift_schema",
                                      "archival_path"
                                  ])

        redshift_connection_string = args['redshift_connection_string']
        input_location_path = args['input_location_path']
        stage_file_path = args['stage_file_path']
        redshift_table = args['redshift_table']
        redshift_schema = args['redshift_schema']
        archival_path = args['archival_path']

        df = wr.s3.read_csv(path=input_location_path)
        df['days'] = df['days'].astype(np.int32)
        df['mileage'] = df['mileage'].astype(np.int32)
        df = df[['product_model_code', 'segmentation', 'subject_english', 'causal_parts_no', 'date_of_incident',
                 'mileage_using_time', 'days_used', 'causal_parts_name_english', 'session', 'glue_last_updated',
                 'pmonth_year', 'Defect_resp', 'sbpr_no', 'uid', 'days', 'mileage', 'defect_response', 'base_model_code']]

        log.info("---Data Loaded----")
        table_schema = {'product_model_code': 'string',
                        'segmentation': 'string',
                        'subject_english': 'string',
                        'causal_parts_no': 'string',
                        'date_of_incident': 'timestamp',
                        'mileage_using_time': 'string',
                        'days_used': 'string',
                        'causal_parts_name_english': 'string',
                        'session': 'string',
                        'glue_last_updated': 'timestamp', 'pmonth_year': 'timestamp', 'Defect_resp': 'string',
                        'sbpr_no': 'string', 'uid': 'string',
                        'days': 'int', 'mileage': 'int', 'defect_response': 'string', 'base_model_code': 'string'}
        log.info("---Schema Extracted----")

        con = wr.redshift.connect("dcp_redshift_connect_Acc_7673")
        log.info("---Redshift Connected----")

        wr.s3.delete_objects(stage_file_path)
        log.info("stage file folder cleaning done")

        wr.redshift.copy(df=df, path=stage_file_path, table="ftir_data", schema="amalgo_zone", con=con,
                                 use_threads=True, mode='append', dtype=table_schema)
        log.info("--- Data moved to Redshift----")

        log.info("--- Archiving started----")
        obj = input_location_path.split("/")[-1]
        now = datetime.now() + timedelta(hours=5, minutes=30)
        suffix = now.strftime('%Y-%m-%d %H:%M:%S')
        file = obj.split(".")[0] + suffix + "." + obj.split(".")[1]
        output_path = f"{archival_path}/{file}"

        wr.s3.to_csv(df=df, path=output_path, index=False)
        wr.s3.delete_objects(input_location_path)
        log.info("--- Files Archived----")

        log.info("--- code Running End----")
    except Exception as e:
        log.info(f"Error Occurred:{e}")
        raise


# pylint: disable=undefined-variable, no-member

import tensorflow as tf
import os
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import BinaryCrossentropy

from face_mask_detection import config as cfg
from face_mask_detection import models

# CREATE THE MODEL AND LOAD ITS DATA
model, data_mask, data_nomask = models.get_model_and_data(cfg.target)
data = np.concatenate((data_mask ,data_nomask), axis = 0)

# CREATE LABELS
label = np.concatenate((np.ones(data_mask.shape[0]), np.zeros(data_nomask.shape[0])), axis = 0)

# SPLIT DATA INTO TRAINING AND VALIDATION AND CREATE TF DATASET
tf.random.set_seed(cfg.TF_SEED)
np.random.seed(cfg.TF_SEED)

(trainX, valX, trainY, valY) = train_test_split(data, label, test_size=0.20, random_state=cfg.TRAIN_TEST_SPLIT_SEED)
train = tf.data.Dataset.from_tensor_slices((trainX, trainY))
val = tf.data.Dataset.from_tensor_slices((valX, valY))
train_dataset = train.repeat().shuffle(cfg.SHUFFLE_BUFFER_SIZE).batch(cfg.BATCH_SIZE)
val_dataset = val.batch(cfg.BATCH_SIZE)

trainX, valX, trainY, valY = [None] * 4
train = None
val = None

# SET OPTIMIZER
optimizer = Adam(lr=cfg.LR)
model.compile(loss=BinaryCrossentropy(from_logits=True),
              optimizer=optimizer, metrics=['accuracy'])

if cfg.DEBUG_NET:
    model.summary()

# CALLBACKS FOR MONITORING AND SAVING
checkpointer = ModelCheckpoint(filepath=f'{cfg.MODEL_PATH}/{cfg.target}.model.weights.hdf5', verbose = 1, save_best_only=True)
log_dir = f'{cfg.LOG_DIR}/{cfg.target}/{datetime.now().strftime("%Y%m%d-%H%M%S")}'
tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)

# ACTUAL TRAIN FUNCTION
history = model.fit(train_dataset, epochs=cfg.EPOCHS,
                    callbacks = [checkpointer, tensorboard_callback],
                    validation_data=val_dataset,
                    steps_per_epoch=110)

# FINAL PLOTS
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title("Loss during the training process")
plt.xlabel('# epochs')
plt.ylabel('Loss')
plt.legend()
plt.savefig(f'{log_dir}/loss.png')
if cfg.SHOW_PLOT:
    plt.show()

plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title("Accuracy during the training process")
plt.xlabel('# epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.savefig(f'{log_dir}/accuracy.png')
if cfg.SHOW_PLOT:
    plt.show()


import sys
from PyQt5.QtWidgets import QMainWindow,QMessageBox,QApplication
from PyQt5.uic import loadUiType , loadUi
from PyQt5.QtWidgets import QMainWindow
from PyQt5.QtGui import  QIcon

import ui_setup,resources
from  main import Ui_MainWindow

class MainApp(QMainWindow,Ui_MainWindow):

    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)
        ui_setup.setup_ui(self)

        self.setWindowIcon(QIcon(r":\icons\create.png"))

    def CreateQRCFileThreadStatus(self,info):
        if info == "finish":
            self.create_btn.setEnabled(True)
            self.steps.setText('')

        if info =="start":
            self.create_btn.setEnabled(False)
            self.steps.setText('يتم إنشاء الملف الرجاء الإنتظار ...')
        if info =="no_arabic":
            QMessageBox.warning(self,"QRC Generator", "لابد أن تكون أسماء الملفات والمجلدات بالأحرف اللاتينية و الأرقام فقط")

    def CreateQRCFileThreadLink(self,link):
        self.path = link

    def CreateQRCFileThreadwrinting_file_error(self,error):
        QMessageBox.warning(self,"QRC Generator",f"حدث مشكل أثناء إنشاء الملف \n {error}" )

app = QApplication(sys.argv)
mainWindow = MainApp()
mainWindow.show()
sys.exit(app.exec())


from pathlib import Path
from torch import nn
from typing import Union
from transformers import T5TokenizerFast
import torch, json, csv, re, copy, math

def load_json(path):
    with open(path, 'r', encoding='UTF-8') as f:
        return json.load(f)

def load_csv(path):
    with open(path, 'r') as f:
        file = [x for x in csv.reader(f.readlines(), quoting=csv.QUOTE_NONNUMERIC)]
    return file

OPERATOR_DICT = {
    "begin_0": "BEGIN",
    "target_0": "TAR",
    "argument_0": "arg",
    "end_1": "END",
    "add_2": "+",
    "subtract_2": "-",
    "multiply_2": "*",
    "divide_2": "/",
    "equal_2": "=",
    "find_2": "find",
    "order_2": "ord"
}

OPERATOR_DICT_REV = {v: k for k, v in OPERATOR_DICT.items()}

class PositionalEncoding(nn.Module):
    def __init__(self, embedding_dim):
        super().__init__()
        self.embedding_dim = embedding_dim

        div_term = (torch.arange(0, embedding_dim) // 2) * 2
        div_term = torch.exp(div_term.float() * (-math.log(10000.0) / embedding_dim))
        multiplier = torch.zeros(2, embedding_dim, dtype=torch.float)
        multiplier[0, 1::2] = 1.0
        multiplier[1, 0::2] = 1.0

        self.register_buffer('_div_term', div_term)
        self.register_buffer('multiplier', multiplier)

    @property
    def device(self) -> torch.device:
        return self._div_term.device

    def before_trigonometric(self, indices: torch.Tensor) -> torch.Tensor:
        indices = indices.float()
        return indices * self._div_term

    def forward(self, index_or_range: Union[torch.Tensor, int, range], ignored_index=-100) -> torch.Tensor:
        with torch.no_grad():
            return self._forward(index_or_range, ignored_index)

    def _forward(self, index_or_range: Union[torch.Tensor, int, range], ignored_index=-100) -> torch.Tensor:
        if type(index_or_range) is int:
            indices = torch.tensor(index_or_range)
        elif type(index_or_range) is range:
            indices = torch.as_tensor(list(index_or_range))
        else:
            indices = index_or_range
        indices = indices.unsqueeze(-1)
        indices = indices.to(self.device)
        phase = self.before_trigonometric(indices)

        cos_value = phase.cos()
        sin_value = phase.sin()
        cos_multiplier = self.multiplier[0]
        sin_multiplier = self.multiplier[1]
        result_shape = [1] * (phase.dim() - 1) + [-1]
        cos_multiplier = cos_multiplier.view(*result_shape)
        sin_multiplier = sin_multiplier.view(*result_shape)

        result = cos_value * cos_multiplier + sin_value * sin_multiplier
        ignored_indices = (indices == ignored_index)
        if ignored_indices.any():
            result.masked_fill_(ignored_indices, 0.0)
        return result.contiguous()

class DecoderEmbed(torch.nn.Module):
    def __init__(self, device):
        super().__init__()
        self.device = device
        self.util_path = Path(__file__).parents[0] / "utils"
        self.fixed_idx_dict = load_json(self.util_path / "fixed_idx_dict.json")
        self.fixed_operator_emb = torch.Tensor(load_csv(self.util_path / "fixed_operator_emb.csv"))
        self.fixed_operand_emb = torch.Tensor(load_csv(self.util_path / "fixed_operand_emb.csv"))
        self.fixed_operator_emb = nn.Embedding.from_pretrained(self.fixed_operator_emb, freeze=True)
        self.fixed_operand_emb = nn.Embedding.from_pretrained(self.fixed_operand_emb, freeze=True)
        self.d_model = self.fixed_operator_emb.weight.shape[-1]
        self.operand_embedding = nn.Embedding(num_embeddings=3, embedding_dim=self.d_model)
        self.memory_embedding = PositionalEncoding(self.d_model)
        self.degrade_factor = 1.0
        self.operator_param = torch.nn.Parameter(torch.tensor(self.degrade_factor), requires_grad=True)
        self.operand_param = torch.nn.Parameter(torch.tensor(self.degrade_factor), requires_grad=True)
        self.opt_layer_norm = nn.LayerNorm(self.d_model, eps=1e-12)
        self.opnd_layer_norm = nn.LayerNorm(self.d_model, eps=1e-12)
        self.linear = nn.Linear(self.d_model * 3, self.d_model)

    def forward(self, tokens: list, float_operand_emb: list, float_idx_dict: dict):
        return self.get_embedding(tokens, float_operand_emb, float_idx_dict)

    def compute_input(self, res_token: list):
        output_embed = torch.cat([self.opt_layer_norm(res_token[0]),
                                  self.opnd_layer_norm(res_token[1]),
                                  self.opnd_layer_norm(res_token[2])], dim=1)
        output_embed = self.linear(output_embed)
        return output_embed

    def return_operand_embedding(self, float_operand_emb: list):
        operand_embedding = list()
        for k in self.fixed_idx_dict["constant"].keys():
            operand_embedding.append(self.get_opnd_fixed_embedding(k))
        for k in range(16):
            operand_embedding.append(self.get_opnd_memory_embedding(element=None, index=k))
        for k in range(len(float_operand_emb)):
            operand_embedding.append(self.get_opnd_float_embedding(float_operand_emb, index=k))
        return torch.cat(operand_embedding)

    def get_embedding(self, tokens: list, float_operand_emb: list, float_idx_dict: dict):
        result = list()
        for n1, tuples in enumerate(tokens):
            res_token = list()
            arity = 1
            for n2, element in enumerate(tuples):
                if int(arity) >= n2:
                    if element in OPERATOR_DICT_REV.keys():
                        res_token.append(self.get_opt_embedding(element, n1))
                        arity = OPERATOR_DICT_REV[element].split("_")[1]
                    elif element in float_idx_dict.keys():
                        index = float_idx_dict[element]
                        res_token.append(self.get_opnd_float_embedding(float_operand_emb, index))
                    elif 'R' in element:
                        res_token.append(self.get_opnd_memory_embedding(element=element, index=None))
                    elif element in self.fixed_idx_dict["constant"].keys():
                        res_token.append(self.get_opnd_fixed_embedding(element))
                    else:
                        raise NotImplementedError(f"{element}: Token is not in target dictionary.")
                else:
                    res_token.append(torch.Tensor([0] * self.d_model).unsqueeze(0).to(self.device))
            result.append(self.compute_input(res_token))
        return torch.cat(result)

    def get_opt_embedding(self, element, n):
        c_f = self.operator_param
        e_f = self.fixed_operator_emb(
            torch.LongTensor([self.fixed_idx_dict["operator"][OPERATOR_DICT_REV[element]]]).to(
                self.device))
        pe = self.memory_embedding(n+1)
        return c_f * e_f + pe

    def get_opnd_float_embedding(self, float_operand_emb, index):
        c_a = self.operand_param
        u_num = self.operand_embedding(torch.LongTensor([0]).to(self.device))
        e_aij = float_operand_emb[index]
        return c_a * u_num + e_aij

    def get_opnd_memory_embedding(self, element=None, index=None):
        c_a = self.operand_param
        u_expr = self.operand_embedding(torch.LongTensor([1]).to(self.device))
        if element and not index:
            index = int(element.replace('R', ''))
        pe = self.memory_embedding(index+1)
        return c_a * u_expr + pe

    def get_opnd_fixed_embedding(self, element):
        c_a = self.operand_param
        u_const = self.operand_embedding(torch.LongTensor([2]).to(self.device))
        e_c = self.fixed_operand_emb(
            torch.LongTensor([self.fixed_idx_dict["constant"][element]]).to(self.device))
        return c_a * u_const + e_c

class DecoderDecode:
    def __init__(self, device):
        self.device = device
        self.util_path = Path(__file__).parents[0] / "utils"
        self.fixed_idx_dict = load_json(self.util_path / "fixed_idx_dict.json")
        self.operator_dict = {v: k for k, v in self.fixed_idx_dict["operator"].items()}
        self.operand_dict = {v: k for k, v in self.fixed_idx_dict["constant"].items()}
        self.operand_dict.update({v + len(self.operand_dict): k for v, k in enumerate([f'R{i}' for i in range(16)])})
        self.operand_embedding = load_csv(Path(__file__).parents[0] / "utils" / "fixed_operand_emb.csv")

    def __call__(self, indexes: list, float_idx_dict: dict):
        float_idx_dict = {v + len(self.operand_dict): k for k, v in float_idx_dict.items()}
        operand_dict_temp = copy.deepcopy(self.operand_dict)
        operand_dict_temp.update(float_idx_dict)
        return self.get_decoded(indexes, operand_dict_temp)

    def get_decoded(self, indexes: list, operand_dict: dict):
        result = list()
        for index in indexes:
            arity = int(self.operator_dict[index[0]].split("_")[1])
            if arity == 2:
                res = [OPERATOR_DICT[self.operator_dict[index[0]]],
                       operand_dict[index[1]].split("_")[0],
                       operand_dict[index[2]].split("_")[0]]
            elif arity == 1:
                res = [OPERATOR_DICT[self.operator_dict[index[0]]],
                       operand_dict[index[1]].split("_")[0],
                       '']
            else:
                res = [OPERATOR_DICT[self.operator_dict[index[0]]],
                       '',
                       '']
            result.append(res)
        return result

    def return_index(self, tokens: list, float_idx_dict: dict, max_len: int):
        float_idx_dict = {k: v + len(self.operand_dict) for k, v in float_idx_dict.items()}
        operand_dict_temp = {k: v for v, k in self.operand_dict.items()}
        operand_dict_temp.update(float_idx_dict)
        result = list()
        for tuples in tokens:
            arity = int(OPERATOR_DICT_REV[tuples[0]].split('_')[1])
            if arity == 2:
                res = [self.fixed_idx_dict["operator"][OPERATOR_DICT_REV[tuples[0]]],
                       operand_dict_temp[tuples[1]],
                       operand_dict_temp[tuples[2]]]
            else:
                res = [self.fixed_idx_dict["operator"][OPERATOR_DICT_REV[tuples[0]]],
                       -100,
                       -100]
            result.append(res)
        result = result + ([[-100, -100, -100]] * (max_len - len(result)))
        return torch.LongTensor(result).to(self.device)

class ExtractInfo:
    def __init__(self, tokenizer: T5TokenizerFast):
        self.tokenizer = tokenizer

    def __call__(self, text, encoder_last_hidden_state):
        num_index = self.extract_num_index(text)
        embedding_matrix = self.get_embedding_matrix(encoder_last_hidden_state, num_index)
        index_dict = {k: n for n, k in enumerate(num_index.keys())}
        return index_dict, embedding_matrix

    def get_embedding_matrix(self, encoder_last_hidden_state, num_index):
        embedding_matrix = list()
        for k, v in num_index.items():
            embedding_matrix.append(encoder_last_hidden_state[v[0]:v[1] + 1].mean(dim=0).unsqueeze(0))
        return embedding_matrix

    def regx_hunter_with_index(self, regx, text):
        found = re.finditer(regx, text)
        result = [(i.group(), i.start(), i.end()) for i in found]
        return result

    def extract_num_index(self, text):
        num_found = self.regx_hunter_with_index('[0-9\-]*[./]?[0-9]+', text)
        token_mapping = self.tokenizer([text], is_split_into_words=True, return_offsets_mapping=True).offset_mapping
        num_index = dict()
        for num in num_found:
            flag = False
            for n, j in enumerate(token_mapping):
                if num[1] == j[0]+1:
                    idx1 = copy.copy(n)
                    flag = True
                if num[2] <= j[1] and flag:
                    idx2 = copy.copy(n)
                    key = num[0]
                    if key in num_index.keys():
                        i = 2
                        while f"{key}_{i}" in num_index.keys():
                            i += 1
                        key = f"{key}_{i}"
                    num_index[key] = (idx1, idx2)
                    break
        return num_index

    def make_noun_label(self, text, tokens):
        noun = make_noun_regx(tokens)
        noun_idx = self.extract_noun(text, noun)
        return [noun_idx[i[1]] for i in tokens if (i[0] == 'TAR')]

    def extract_noun(self, text, noun):
        noun_found = self.regx_hunter_with_index(noun, text)
        token_mapping = self.tokenizer([text], is_split_into_words=True, return_offsets_mapping=True).offset_mapping
        noun_index = dict()
        for noun in noun_found:
            for n, j in enumerate(token_mapping):
                if noun[1] < j[1]:
                    key = copy.copy(noun[0])
                    if key in noun_index.keys():
                        break
                    noun_index[key] = n
                    break
        return noun_index

def regx_hunter(regx, text):
    reg = re.compile(regx)
    return reg.findall(text)

def make_noun_regx(token):
    noun_list = [i[1] for i in token if i[0] == 'TAR']
    return '|'.join(noun_list)

def emb_matrix_tensor(operand_embeddings, device):
    batch_size = len(operand_embeddings)
    max_len = max([i.shape[0] for i in operand_embeddings])
    embedding_size = operand_embeddings[0].shape[1]
    emb = torch.zeros(batch_size, max_len, embedding_size).to(device)
    mask = torch.zeros(batch_size, max_len).to(device)
    for n, i in enumerate(operand_embeddings):
        emb[n][:i.shape[0]] = i
        mask[n][i.shape[0]:] = 1
    mask = mask.bool()
    return emb, mask

def weight_to_index(lm_output, operand_left, operand_right, pad_mask):
    operand_left = torch.masked_fill(operand_left, pad_mask.unsqueeze(1), float('-inf'))
    operand_right = torch.masked_fill(operand_right, pad_mask.unsqueeze(1), float('-inf'))
    operator_argmax = lm_output.argmax(dim=2)
    operand_left_argmax = operand_left.argmax(dim=2)
    operand_right_argmax = operand_right.argmax(dim=2)
    index_list = torch.cat([operator_argmax.unsqueeze(2), operand_left_argmax.unsqueeze(2),
                            operand_right_argmax.unsqueeze(2)], dim=2).tolist()
    return index_list

def get_float_info(model, extractor, input_str, input_ids, device):
    float_dict, float_emb = list(), list()
    dummy_input = torch.zeros(len(input_ids), 1, dtype=torch.int).to(device)
    encoder_last_hidden_state = model(input_ids=input_ids, decoder_input_ids=dummy_input).encoder_last_hidden_state
    for text, hidden in zip(input_str, encoder_last_hidden_state):
        num_dict, num_emb = extractor(text, hidden)
        float_dict.append(num_dict)
        float_emb.append(num_emb)
    return float_dict, float_emb

def get_noun_info(extractor, input_str, label, device):
    noun_label = list()
    max_token_len = max([len(i) for i in label])
    for text, tokens in zip(input_str, label):
        noun_info = extractor.make_noun_label(text, tokens)
        noun_label.append(noun_info + ([-100] * (max_token_len - len(noun_info))))
    return torch.LongTensor(noun_label).to(device)

def compute_accuracy(target, outputs, ignore_index=None):
    arg = outputs.argmax(dim=1)
    if ignore_index is None:
        n_total = target.shape[0] * target.shape[1]
        n_correct = torch.sum(torch.tensor(arg == target))
    else:
        ignore_mask = torch.tensor(target != ignore_index)
        n_total = torch.sum(ignore_mask)
        n_correct = torch.sum(torch.logical_and(torch.tensor(arg == target), ignore_mask))
    return int(n_correct), int(n_total)

def build_masks(label_idx, device, generate):
    if generate:
        return None
    else:
        label_idx = label_idx.select(dim=2, index=0)
        label = torch.cat([torch.zeros(label_idx.shape[0], 1).to(device), label_idx], dim=1)
        masks = (label != -100).to(device)
    return masks


import matplotlib.pyplot as plt
import os
import torch
from torch.utils.data import DataLoader, TensorDataset, Dataset
from torchvision import datasets
from torchvision.transforms import ToTensor
from torchvision import models, transforms
from torchvision.models import ResNet50_Weights
import numpy as np
from PIL import Image
import pickle
import torch.nn as nn

################################
# Methods for data preprocessing.
################################

def create_directory(dir: str):
    """
    Create the given directory if doesn't exist.
    """
    if not os.path.exists(dir):
        os.makedirs(dir)
    return

def get_a1_dataset(data_dir: str="/data/yuquand/a1_demo/") -> dict:
    """
    Parameters:
    data_dir(str): The directory of a1 dataset where the images are arranged
    in this way by default:

    data_dir/part_x/image_AAAAA_BBBB_CCCC_DDDD.png

    where AAAAA is the Frame order, BBBB is vel_x, CCCC is vel_y, and DDDD is
    steering angle.

    Returns:
    dict: A dictonary containing tuples (img_data, label) with corresponding key
    equals to its batch dataset directory(e.g. part_1).
    """
    a1_datasets = {x: get_batch_dataset(os.path.join(data_dir, x)) for x in ['part_1', 'part_2']}

    return a1_datasets

def get_batch_dataset(directory: str):
    """
    Load batch dataset from the given directory.

    Parameters:
    directory(str): The directory of the batch dataset.

    Returns:
    tuple: (img_data, cmd_label) where both imag_data and cmd_label
           are numpy arrays.
    """
    print("-" * 100)

     # Load file path and sort based on the order of the images.
    filenames = os.listdir(directory)
    filenames = sorted(filenames)

    # Parse images and labels respectively.
    images, labels = [], []
    for filename in filenames:
        f = os.path.join(directory, filename)

        # Parse image
        img = Image.open(f)
        images.append(np.array(img))

        # Parse label (vel_x, vel_y, theta)
        prefix = 'image_'
        suffix = '.png'
        cmd = filename[len(prefix):-len(suffix)].split("_")[1:]

        # Delete the vel_y column and only keep (vel_x, theta)
        cmd = np.delete(np.asarray(cmd, dtype=np.float64), 1, 0)
        labels.append(cmd)

    images = np.array(images)
    # N, H, W, C = images.shape
    # images = images.reshape((N, C, H, W)) # reshape into [N, C, H, W]
    images = images.transpose(0, 3, 1, 2)
    labels = np.array(labels)

    print(f"images shape: {images.shape}")
    print(f"labels shape: {labels.shape}\n")
    print(f"Finished loading a1 batch dataset from directory {directory}.")
    print("-" * 100)

    return images, labels

def extract_batch_feature(device, model, batch_data, batch_size:int=128):
    """
    Extract features from the raw images but keep the order
    of the image and command pairs.

    Return
    """
    model.eval()
    input_feature, label_feature = [], []
    batch_inputs, batch_labels = batch_data

    val_transforms =  transforms.Compose([
        transforms.ToPILImage(),
        transforms.Resize((224, 224)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    dataset = CustomTensorDataset(
        tensors=(torch.tensor(batch_inputs), torch.tensor(batch_labels)),
        transform=val_transforms
    )

    # Set shuffle to false for keeping the order of images.
    data_loader = DataLoader(
        dataset=dataset,
        shuffle=False,
        batch_size=batch_size
    )

    with torch.no_grad():
        for i, data in enumerate(data_loader):
            input, label = data
            input = input.to(device)
            label = label.to(device)
            output = model(input)

            # Reshape Batch_dim xFeature_dim x 1 x 1 to Batch_dim xFeature_dim.
            batch_dim, feature_dim = output.size()[0:2]
            input_feature.append(output.reshape((batch_dim, feature_dim)))
            label_feature.append(label)

    # Convert tensors to Numpy array.
    input_feature = torch.cat(input_feature, dim=0).detach().cpu().numpy()
    label_feature = torch.cat(label_feature, dim=0).detach().cpu().numpy()

    # TODO: comment it out for debugging
    # print("-"*100)
    # print(f"inputs feature shape: {input_feature.shape}")
    # print(f"labels shape: {label_feature.shape}")
    # print("Finished extract feature from current batch data.")
    # print("-"*100)

    return input_feature, label_feature

class CustomTensorDataset(Dataset):
    """
    TensorDataset with support of transforms.
    """
    def __init__(self, tensors, transform=None) -> None:
        assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        self.transform = transform

    def __getitem__(self, index):
        x = self.tensors[0][index]
        if self.transform:
            x = self.transform(x)

        y = self.tensors[1][index]

        return x, y

    def __len__(self):
        return self.tensors[0].size(0)

def get_a1_dataloader(feature_dataset, batch_size: int, interval: int, train_ratio:float=0.7, ):
    """
    Given feature_datatset, feed data in pretrained ResNet50 and split dataset
    as train, val dataloader.
    """
    imgs, cmd_labels = concat_imgs(feature_dataset, interval)

    # Random Split into train and validation set.
    train_size = int(np.floor(imgs.shape[0] * train_ratio))
    print(f"training set size: {train_size}, validation set size: {imgs.shape[0]-train_size}")
    print("-"*100)

    dataset = TensorDataset(torch.tensor(imgs), torch.tensor(cmd_labels))
    train_set, val_set = torch.utils.data.random_split(dataset, [train_size, imgs.shape[0] - train_size])
    train_loader = DataLoader(dataset=train_set, shuffle=True, batch_size=batch_size)
    val_loader = DataLoader(dataset=val_set, shuffle=True, batch_size=batch_size)

    return train_loader, val_loader

def concat_imgs(feature_dataset, interval):
    """
    Concatenate the original images with their goal images by the given
    interval.

    Return a pair of concatenated images features and their lables as a
    tuple of numpy arrays with size ((N, 4096), (N, 2)) where 4096 is
    the number of concatenated features.
    """
    # Concatenate start and goal image.
    imgs, cmd_labels = [], []
    for batch_dir in feature_dataset:
        img_data, cmd_label = feature_dataset[batch_dir]

        original_img = img_data[:-interval, :]
        goal_img = img_data[interval:, :]
        cat_img = np.concatenate((original_img, goal_img), axis=-1)
        # interval+1 for including the goal image.
        cat_cmd = average_cmd(cmd_label, interval+1)  # Take the average command over the interval.

        imgs.append(cat_img)
        cmd_labels.append(cat_cmd)

        print(f"interval: {interval}")
        print(f"[concatenated] image data size : {cat_img.shape}")
        print(f"[concatenated] cmd_label size : {cat_cmd.shape}")
        print("-"*100)

    imgs = np.concatenate(imgs)
    cmd_labels = np.concatenate(cmd_labels)

    return imgs, cmd_labels

def average_cmd(cmd_label, interval):
    cmd_label = cmd_label
    vel_x = cmd_label[:, 0]
    steering_angle = cmd_label[:, 1]
    print("-"*100)
    print(f"vel_x size: {vel_x.shape}, steering_angle size:{steering_angle.shape}")

    kernel = np.full((interval,), 1)
    vel_x_avg = np.convolve(vel_x, kernel, mode='valid') / len(kernel)
    steering_angle_avg = np.convolve(steering_angle, kernel, mode='valid') / len(kernel)

    cat_cmd = np.stack((vel_x_avg, steering_angle_avg), axis=1)
    print(f"[before convoluted] cmd_label size: {cmd_label.shape}")
    print(f"[after convoluted] cmd_label size: {cat_cmd.shape}")
    print("-"*100)
    return torch.from_numpy(cat_cmd).float()

############################################################
# Other utils methods(plot data, save and load dataset, etc)
############################################################
def pretrained_ResNet50(device):
    # Initailize ResNet50 as Pretrain model and treat it as fix feature extractor
    resnet50 = models.resnet50(weights=ResNet50_Weights.DEFAULT)
    modules = list(resnet50.children())[:-1]
    resnet50 = nn.Sequential(*modules)
    for p in resnet50.parameters():
        p.requires_grad = False
    resnet50.to(device)
    resnet50.eval()
    print("Set pretrained resnet50 into evaluation mode.")
    return resnet50

def save(file, filepath:str) -> None:
    directory = os.path.dirname(filepath)
    if not os.path.exists(directory):
        os.makedirs(directory)

    with open(filepath, 'wb') as fp:
        pickle.dump(file, fp)
    print(f"Saved as {filepath}")
    print("-"*100)

def load(filepath:str):
    file = pickle.load(open(filepath, "rb"))
    print(f"Loaded file {filepath}")
    print("-"*100)
    return file

def log_metrics_figure(train_result, valid_result, path: str='../images_outputs', name: str=None, loss: bool=True):
    """
    Function to save the loss and accuracy plots to disk.
    """
    # Create the directory if it doesn't exist.
    if not os.path.exists(path):
        os.makedirs(path)

    train_label = 'train loss' if loss else 'train accuracy'
    validation_label = 'validataion loss' if loss else 'validataion accuracy'
    xlabel = 'Epochs'
    ylabel = 'Loss' if loss else 'Accuracy'
    name = name if name is not None else ('loss' if loss else 'accuracy')

    # Plot figure.
    plt.figure(figsize=(10, 7))
    plt.plot(
        train_result, color='tab:blue', linestyle='-',
        label=train_label
    )
    plt.plot(
        valid_result, color='tab:red', linestyle='-',
        label=validation_label
    )
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.legend()
    plt.savefig(os.path.join(path, name + '.png'))
    print(f"Successfully saved {os.path.join(path, name + '.png')}")


from django.http import Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse, reverse_lazy
from django.views.generic.edit import CreateView,DeleteView,UpdateView
from django.views.generic import ListView,DetailView
from .forms import *
from django.contrib.auth.decorators import login_required
from user.decoraters import euser_required
from django.utils.decorators import method_decorator
from django.template.loader import render_to_string

from .models import *

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class StocksListView(ListView):
    model = UserPortfolioStock
    template_name = 'stocks/liststocks.html'
    context_object_name = 'stockslist'

    def get_queryset(self):
        queryset = super().get_queryset()
        price_sort = self.request.GET.get('price_sort')
        if price_sort == 'asc':
            queryset = queryset.order_by('currentPrice')
        elif price_sort == 'desc':
            queryset = queryset.order_by('-currentPrice')
        return queryset

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class StocksCreateView(CreateView):
    # fields = ('user','portfolio','stocks','qty','purchaseDate','investPrice','currentPrice')

    model = UserPortfolioStock
    form_class = StocksCreationForm
    template_name = 'stocks/createstock.html'
    # success_url = '/user/euserdash'

    def form_valid(self, form):
        form.instance.user = self.request.user
        return super().form_valid(form)

    # def get_initial(self):
    #     initial = super().get_initial()
    #     initial['portfolio'] = Portfolio.objects.filter(user=self.request.user).values_list('id', flat=True)
    #     return initial

    def get_success_url(self):
        print(f"self.object.pk = {self.object.pk}")
        portfolio = self.object.portfolio

    # Get the id of the Portfolio object
        pid = portfolio.id

    # Use the id of the Portfolio object as the value for the pk argument
        return reverse('portfoliodetail', kwargs={'pk': pid})

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class StockDeleteView(DeleteView):
    model = UserPortfolioStock
    def get(self, request, *args, **kwargs):
        return self.delete(request, *args, **kwargs)

    # success_url = '/user/euserdash/'
    def get_success_url(self):
        print(f"self.object.pk = {self.object.pk}")
        portfolio = self.object.portfolio

    # Get the id of the Portfolio object
        pid = portfolio.id

    # Use the id of the Portfolio object as the value for the pk argument
        return reverse('portfoliodetail', kwargs={'pk': pid})

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class StockUpdateView(UpdateView):
    model = UserPortfolioStock
    form_class = StocksUpdateForm
    template_name = 'stocks/updatestock.html'

    def get_success_url(self):
        print(f"self.object.pk = {self.object.pk}")
        portfolio = self.object.portfolio

    # Get the id of the Portfolio object
        pid = portfolio.id

    # Use the id of the Portfolio object as the value for the pk argument
        return reverse('portfoliodetail', kwargs={'pk': pid})

    # def get_success_url(self):
    #     return f'/stocks/pdetails/{self.object.pk}/'

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class PortfolioCreateView(CreateView):

     form_class = PortfolioCreateForm
     template_name = 'stocks/createportfolio.html'
     success_url = '/user/euserdash'

     def get_initial(self):
        initial = super().get_initial()
        initial['user'] = self.request.user
        return initial

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class PortfolioUpdateView(UpdateView):
    model = Portfolio
    fields = ('portfolioName',)
    template_name = 'stocks/createportfolio.html'
    success_url = '/user/euserdash/'

@method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
class PortfolioDeleteView(DeleteView):
    model = Portfolio
    success_url = reverse_lazy('euserdash')

    def get_object(self, queryset=None):
        """Hook to ensure object is owned by request.user."""
        obj = super(PortfolioDeleteView, self).get_object()
        if not obj.user == self.request.user:
            raise Http404
        return obj

@method_decorator([login_required(login_url='/user/login'), euser_required], name='dispatch')
class PortfolioDetailView(DetailView):
    model = UserPortfolioStock
    template_name = 'stocks/portfoliodetails.html'
    context_object_name = 'portfoliodetails'

    def get_labels_and_data(self):
        labels = []
        data = []
        data1 = []
        data2 = []
        profit_loss = []
        user_portfolio = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk'])
        stocks = Stocks.objects.filter(userportfoliostock__portfolio=self.kwargs['pk']).distinct()

        for stock in stocks:
            stock_name = stock.stockName
            stock_price = user_portfolio.filter(stocks=stock).values_list('currentPrice', flat=True).last()
            invest_price = user_portfolio.filter(stocks=stock).values_list('investPrice', flat=True).last()
            stock_invest_price = user_portfolio.filter(stocks=stock).values_list('investPrice', flat=True).last()
            if stock_price is not None:
                labels.append(stock_name)
                data.append(stock_price)
                data2.append(invest_price)
                if stock_invest_price is not None:
                    profit_loss.append(stock_price - stock_invest_price)
                    data1.append(abs(stock_price - stock_invest_price))
                else:
                    profit_loss.append(None)
                    data1.append(None)
                print(data1)
        return labels, data, data1, data2, profit_loss

    def get(self, request, *args, **kwargs):
        labels, data, data1,data2, profit_loss = self.get_labels_and_data()

        print(f"request.user = {request.user}")
        portfolio = Portfolio.objects.filter(userportfoliostock=self.kwargs['pk'],user=request.user).values()
        userporfolio = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk'])
        portfolioName = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioName', flat=True).get()
        pid = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('id', flat=True).get()

        if request.method == 'GET':
            price_sort = request.GET.get('price_sort')
            if price_sort == 'asc':
                userporfolio = userporfolio.order_by('currentPrice')
        elif price_sort == 'desc':
            userporfolio = userporfolio.order_by('-currentPrice')

        return render(request, self.template_name, {'portfoliodetails': self.get_object(),
                                                'portfolio': portfolio,
                                                'ups': userporfolio,
                                                'portfolioName': portfolioName,
                                                'pid': pid,
                                                'labels': labels,
                                                'data': data,
                                                'data1': data1,
                                                'data2': data2,
                                                'label1': 'Price difference',
                                                'profit_loss': profit_loss})

    def get_object(self, queryset=None):
        try:
            return super().get_object(queryset)
        except Http404:
            return redirect('portfoliodetail', pk=self.kwargs['pk'])

class AdviseListView(ListView):
    model = Advise
    template_name = 'user/advised_stocks.html'
    context_object_name = 'advises'

    def get_queryset(self):
        return Advise.objects.filter(usser=self.request.user).values('adviser__username', 'advise', 'recompdate', 'rstocks__stockName', 'reason')

#commented code in portfoliodetailview after context object name-----------------------------------------
# def get_labels_and_data(self):
    #     labels = []
    #     data = []
    #     # pname = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioName', flat=True)
    #     pname = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk']).values_list('stockName', flat=True)
    #     pscore = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk']).values_list('currentPrice', flat=True)

    #     # pname = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioName',flat=True).get()
    #     # pname =Portfolio.objects.filter().values_list('portfolioName',flat=True)
    #     # pscore = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioScore', flat=True)

    #     for i in pname:
    #         labels.append(i)
    #     for i in pscore:
    #         data.append(i)

    #     return labels, data

# class UserPortfolioStockListView(ListView):
#     model = UserPortfolioStock
#     template_name = 'stocks/portfoliodetails.html'
#     context_object_name = 'ups'

#     def get_queryset(self):
#         # Get the portfolio name from the URL parameter
#         portfolio_name = self.request.GET.get('portfolioName', '')

#         # Filter the UserPortfolioStock queryset based on the portfolio name
#         queryset = UserPortfolioStock.objects.filter(portfolio__portfolioName=portfolio_name)

#         return queryset

 # if userporfolio.exclude(stocks__isnull=True).exists():
        #   if userporfolio.doesnotexists():
        #   message = "No user portfolio stock found matching the query."
        #   print(message)

        # # Handle the case where no results are returned
        #     message = "No user portfolio stock found matching the query."
        #     # partial = render_to_string('stocks/partial_template.html', {'message': message})
        #     return render(request, self.template_name, {'portfoliodetails': self.get_object(), 'portfolio': portfolio, 'message': message})
        # userporfolio = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk']).select_related('stock').values()

#Not in use portfoliodetail view ------------------------------------------------------------
# @method_decorator([login_required(login_url='/user/login'),euser_required],name='dispatch')
# class PortfolioDetailView(DetailView):
#     model = UserPortfolioStock
#     template_name = 'stocks/portfoliodetails.html'
#     context_object_name = 'portfoliodetails'

#     labels=[]
#     data=[]

#     pname = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('id',flat=True)
#     pscore = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioScore',flat=True)

#     for i in pname:
#             labels.append(i)
#     for i in pscore:
#             data.append(i)

    # def get(self, request, *args, **kwargs):
    #     # print(f"self.kwargs['pk'] = {self.kwargs['pk']}")
    #     print(f"request.user = {request.user}")
    #     portfolio = Portfolio.objects.filter(userportfoliostock=self.kwargs['pk'],user=request.user).values()
    #     # print(f"portfolio = {portfolio}")
    #     userporfolio = UserPortfolioStock.objects.filter(portfolio=self.kwargs['pk'])
    #     portfolioName = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('portfolioName',flat=True).get()
    #     pid = Portfolio.objects.filter(id=self.kwargs['pk']).values_list('id',flat=True).get()

    #     print(userporfolio)
    #     if request.method == 'GET':
    #            price_sort = request.GET.get('price_sort')
    #     if price_sort == 'asc':
    #            userporfolio = userporfolio.order_by('currentPrice')
    #     elif price_sort == 'desc':
    #            userporfolio = userporfolio.order_by('-currentPrice')

    #     return render(request, self.template_name, {'portfoliodetails': self.get_object(),'portfolio':portfolio,'ups':userporfolio,'portfolioName':portfolioName,'pid':pid,'labels':self.labels,'data':self.data})

    # def get_object(self, queryset=None):
    #     try:
    #         return super().get_object(queryset)
    #     except Http404:
    #         return redirect('portfoliodetail', pk=self.kwargs['pk'])

#commented code for stock list view------------------------------------------------------------
#     queryset = super().get_queryset()
    #     queryset = queryset.filter(user=self.request.user)
    #     return queryset

    # def get_queryset(self):
    #     queryset = super().get_queryset()
    #     price_filter = self.request.GET.get('price_filter')
    #     if price_filter:
    #         min_price, max_price = price_filter.split('-')
    #         queryset = queryset.filter(currentPrice__gte=min_price, currentPrice__lte=max_price)
    #     return queryset

# def calculate_total_difference(portfolio_id):
#     total_difference = 0
#     for stock in UserPortfolioStock.objects.filter(portfolio=portfolio_id):
#         difference = stock.currentPrice - stock.investPrice
#         total_difference += difference

#     return total_difference
#     print(f"total_difference = {total_difference}")


# Streamlit implementation of recipe recognition model
# Image --> Recipe Title
# Adapted from Tensorflow's "Image captioning with visual attention" tutorial
# https://www.tensorflow.org/tutorials/text/image_captioning
# Layer construction taken directly from tutorial with slight modifications

import tensorflow as tf
import pandas as pd
import numpy as np
import re
import string
import einops
import streamlit as st
import cv2

RELMODELPATH = 'streamlitapp/models/'
def get_model():
    IMAGE_SHAPE = (224, 224, 3)     # Mobilenet Image Embedder Compatibility
    DF_PATH = RELMODELPATH+"df.csv"
    WEIGHTS_PATH = RELMODELPATH+"cp-0022.ckpt"

    def standardize(s):
        s = tf.strings.lower(s)
        s = tf.strings.regex_replace(s, f'[{re.escape(string.punctuation)}]', '')
        s = tf.strings.regex_replace(s, f'\d+', '')
        s = tf.strings.join(['startSeq', s, 'endSeq'], separator=' ')
        return s
    # Image Feature Extractor
    mobilenet = tf.keras.applications.MobileNetV3Small(
        input_shape=IMAGE_SHAPE,
        include_top=False,
        include_preprocessing=True,
        weights='imagenet'
    )

    # Initialize Tokenizer with vocabulary of 1,000
    df = pd.read_csv(DF_PATH)
    vocab_size = 1000
    tokenizer = tf.keras.layers.TextVectorization(
        max_tokens=vocab_size,
        standardize=standardize,
        ragged=True
    )
    tokenizer.adapt(np.asarray(df['Title']).astype(str))

    # Create Tensorflow-compatible string-lookup tables
    # to map indices to words
    word_to_index = tf.keras.layers.StringLookup(
        mask_token="",
        vocabulary=tokenizer.get_vocabulary())
    index_to_word = tf.keras.layers.StringLookup(
        mask_token="",
        vocabulary=tokenizer.get_vocabulary(),
        invert=True)

    # Model layer Definitions
    class SeqEmbedding(tf.keras.layers.Layer):
        def __init__(self, vocab_size, max_length, depth):
            super().__init__()
            self.pos_embedding = tf.keras.layers.Embedding(input_dim=max_length, output_dim=depth)

            self.token_embedding = tf.keras.layers.Embedding(
            input_dim=vocab_size,
            output_dim=depth,
            mask_zero=True)

            self.add = tf.keras.layers.Add()

        def call(self, seq):
            seq = self.token_embedding(seq)

            x = tf.range(tf.shape(seq)[1])
            x = x[tf.newaxis, :]
            x = self.pos_embedding(x)

            return self.add([seq,x])

    class CausalSelfAttention(tf.keras.layers.Layer):
        def __init__(self, **kwargs):
            super().__init__()
            self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)
            # Use Add instead of + so the keras mask propagates through.
            self.add = tf.keras.layers.Add()
            self.layernorm = tf.keras.layers.LayerNormalization()

        def call(self, x):
            attn = self.mha(query=x, value=x, use_causal_mask=True)
            x = self.add([x, attn])
            return self.layernorm(x)

    class CrossAttention(tf.keras.layers.Layer):
        def __init__(self, **kwargs):
            super().__init__()
            self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)
            self.add = tf.keras.layers.Add()
            self.layernorm = tf.keras.layers.LayerNormalization()

        def call(self, x, y):
            attn, attention_scores = self.mha( query=x, value=y, return_attention_scores=True)

            self.last_attention_scores = attention_scores

            x = self.add([x, attn])
            return self.layernorm(x)

    class FeedForward(tf.keras.layers.Layer):
        def __init__(self, units, dropout_rate=0.1):
            super().__init__()
            self.seq = tf.keras.Sequential([
            tf.keras.layers.Dense(units=2*units, activation='relu'),
            tf.keras.layers.Dense(units=units),
            tf.keras.layers.Dropout(rate=dropout_rate),
            ])

            self.layernorm = tf.keras.layers.LayerNormalization()

        def call(self, x):
            x = x + self.seq(x)
            return self.layernorm(x)

    class DecoderLayer(tf.keras.layers.Layer):
        def __init__(self, units, num_heads=1, dropout_rate=0.1):
            super().__init__()
            self.self_attention = CausalSelfAttention(num_heads=num_heads, key_dim=units, dropout=dropout_rate)
            self.cross_attention = CrossAttention(num_heads=num_heads, key_dim=units, dropout=dropout_rate)
            self.ff = FeedForward(units=units, dropout_rate=dropout_rate)

        def call(self, inputs, training=False):
            in_seq, out_seq = inputs
            # Text inputs
            out_seq = self.self_attention(out_seq)
            out_seq = self.cross_attention(out_seq, in_seq)
            self.last_attention_scores = self.cross_attention.last_attention_scores
            out_seq = self.ff(out_seq)
            return out_seq

    class TokenOutput(tf.keras.layers.Layer):
        def __init__(self, tokenizer, banned_tokens=('', '[UNK]', 'startSeq'), **kwargs):
            super().__init__()

            self.dense = tf.keras.layers.Dense(units=tokenizer.vocabulary_size())
            self.tokenizer = tokenizer
            self.banned_tokens = banned_tokens
            self.bias = None

        def call(self, x):
            x = self.dense(x)
            return x

    output_layer = TokenOutput(tokenizer, banned_tokens=('', '[UNK]', 'startSeq'))

    # Model Construction
    class Captioner(tf.keras.Model):
        @classmethod
        def add_method(cls, fun):
            setattr(cls, fun.__name__, fun)
            return fun

        def __init__(self, tokenizer, feature_extractor, output_layer, num_layers=1, units=256, max_length=50, num_heads=1, dropout_rate=0.1):
            super().__init__()
            self.feature_extractor = feature_extractor
            self.tokenizer = tokenizer
            self.word_to_index = tf.keras.layers.StringLookup(mask_token="", vocabulary=tokenizer.get_vocabulary())
            self.index_to_word = tf.keras.layers.StringLookup(mask_token="", vocabulary=tokenizer.get_vocabulary(), invert=True)
            self.seq_embedding = SeqEmbedding(vocab_size=tokenizer.vocabulary_size(), depth=units, max_length=max_length)
            self.decoder_layers = [
            DecoderLayer(units, num_heads=num_heads, dropout_rate=dropout_rate) for n in range(num_layers)
            ]
            self.output_layer = output_layer

    @Captioner.add_method
    def call(self, inputs):
        image, txt = inputs

        if image.shape[-1] == 3:
            image = self.feature_extractor(image)
        image = einops.rearrange(image, 'b h w c -> b (h w) c')

        if txt.dtype == tf.string:
            txt = tokenizer(txt)
        txt = self.seq_embedding(txt)

        for dec_layer in self.decoder_layers:
            txt = dec_layer(inputs=(image, txt))

        txt = self.output_layer(txt)
        return txt

    @Captioner.add_method
    def simple_gen(self, image, temperature=1):
        initial = self.word_to_index([['startSeq']])    # Batch, sequence
        img_features = self.feature_extractor(image[tf.newaxis, ...])
        tokens = initial
        for n in range(50):
            preds = np.array(self((img_features, tokens)))
            preds = preds[:,-1,:]
            if temperature == 0:
                next = tf.argmax(preds, axis=-1)[:, tf.newaxis]
            else:
                next = tf.random.categorical(preds / temperature, num_samples=1)
            tokens = tf.concat([tokens,next], axis=1)

            if next[0] == self.word_to_index('endSeq'):
                break
        words = index_to_word(tokens[0, 1:-1])
        result = tf.strings.reduce_join(words, axis=-1, separator=' ')
        return result

    # Cache This ********************************************
    model = Captioner(tokenizer, feature_extractor=mobilenet, output_layer=output_layer,
                    units=225, dropout_rate=0.5, num_layers=2, num_heads=2)
    model.load_weights(WEIGHTS_PATH)
    # Cache This ********************************************
    return model


def fix_sentence(sentence):
    sentence = ''.join(c.lower() for c in sentence if c.isalnum() or c.isspace())
    fixed_s = sentence.strip()
    fixed_s = " ".join(fixed_s.split())
    return fixed_s

def print_dict(n):
    if not n:
        return
    # for s in n.sources:
    #     print(s.offset, end=",")
    print("\t", n.letter, n.sons.keys())
    for s in n.sons:
        print_dict(n.sons[s])


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('playlist', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='suggestedsongs',
            name='suggestedTo',
            field=models.ForeignKey(related_query_name=b'touser', related_name='suggested_to_user', db_column=b'suggested_to', to=settings.AUTH_USER_MODEL),
        ),
    ]


import pandas as pd
"""
s1 = pd.Series([3,2,0,1])
s2 = pd.Series([0,3,7,2])

data = dict(apples= s1 , oranges= s2)
#Sözlük yapısına çevrilerek apples ve oranges kolon ismi olarak kayıt edilir
#Diğer sayılar value olarak indislere atanır
print(data)
df = pd.DataFrame(data)
#pd.DataFrame komutu ile tablo yapısı oluşturulur
print(df)
"""

list = [['İphone 7',2017,5000],['İphone 8',2018,6000],['İphone 9',2019,7000]]
#Data frame içerisine ilk parametreye eklenen her tuple,liste,dizi dataya 1 satır ekler
#Data içerisindeki dizi elemanları ise dataya birer sütun ekler
df = pd.DataFrame(list, columns=['Sütun-1','Sütun-2','Sütun-3'],index=['m1','m2','m3'])
print(df)
#ALTERNATİVE 2 şekilde gösterebilriiz 1- Normal liste içinde 2-Sözlük şeklinde yazarak

dict = {'Phone Name':['İphone 7','İphone 8','İphone 9'],'Release Year':[2017,2018,2019],'Price':[5000,6000,7000]}

dict_df = pd.DataFrame(dict,index=[1,2,3])
print(dict_df)
"""
dict_list = [
                {SATIR 1}#Key Value değerleri vardır. Her key(value) bir sütuna denk gelir
                {SATIR 2}
                {SATIR 3}
                {SATIR 4}
                {SATIR 5}
                ]
"""


# section 5.5 of the EPI book
# delete duplicate elements from a sorted array
def deletedupes(arr):
    if len(arr) == 0:
        return arr

    print("input is {}".format(arr))
    # think of end as the index before which (ie, till end-1) you have everything sorted out
    # ie, this is the index, before which everything is distinct
    # this index only increments if the elements encountered by index i are different
    end = 1
    for i in range(1, len(arr)):
        if arr[end-1] != arr[i]:
            arr[end] = arr[i]
            end +=1

    print("output is {}".format(arr[:end]))
    return arr[:end]

(deletedupes([1,2,2,2,2,3,4,4,4,5,6,7,7]))
(deletedupes([1,2,2]))
(deletedupes([1,1,1,1,1]))


import numpy as np

def clean_data(data):

    data['match_round'] = data['match_round'].astype('str')

    # Remove games in finals series'. Brownlow votes are not awarded in final's series'.
    data = data[(data['match_round'] != 'Elimination Final') &
                (data['match_round'] != 'Finals Week 1') &
                (data['match_round'] != 'Qualifying Final') &
                (data['match_round'] != 'Semi Final') &
                (data['match_round'] != 'Semi Finals') &
                (data['match_round'] != 'Preliminary Final') &
                (data['match_round'] != 'Preliminary Finals') &
                (data['match_round'] != 'Grand Final')]

    data['year'] = data.apply(lambda row: row.match_date[0:4], axis=1)
    data['year'] = data['year'].astype(float)

    # Add columns to show whether the game was a win or a loss
    data['win'] = np.where((data['match_margin']) > 0, 1, 0)
    data['draw'] = np.where((data['match_margin']) == 0, 1, 0)
    data['loss'] = np.where((data['match_margin']) < 0, 1, 0)

    # drop the first unnecessary column
    data = data.drop('Unnamed: 0', axis=1)

    # normalize variables as a proportion of that game
    normalisable_variables = ['kicks', 'marks', 'handballs', 'disposals', 'effective_disposals', 'goals',
                              'behinds', 'hitouts', 'tackles', 'rebounds', 'inside_fifties',
                              'clearances', 'clangers', 'free_kicks_for', 'free_kicks_against',
                              'contested_possessions', 'uncontested_possessions', 'contested_marks',
                              'marks_inside_fifty', 'one_percenters', 'bounces', 'goal_assists',
                              'afl_fantasy_score', 'centre_clearances',
                              'stoppage_clearances', 'score_involvements', 'metres_gained',
                              'turnovers', 'intercepts', 'tackles_inside_fifty', 'contest_def_losses',
                              'contest_def_one_on_ones', 'contest_off_one_on_ones', 'contest_off_wins',
                              'def_half_pressure_acts', 'effective_kicks', 'f50_ground_ball_gets',
                              'ground_ball_gets', 'hitouts_to_advantage', 'intercept_marks',
                              'marks_on_lead', 'pressure_acts', 'rating_points', 'ruck_contests',
                              'score_launches', 'shots_at_goal', 'spoils']

    for variable in normalisable_variables:
        name = variable + "_s"
        data[name] = data.groupby('match_id', group_keys=False).apply(lambda g: g[variable] / g[variable].sum())

    return data


import os
from praatio import tgio
from num2words import num2words
import decimal
import re

multiword_mappings = {
    '[ __ ]': '',
    'claudine white': 'cloud9 white',
    'a evil cat': 'aevilcat',
    'tanner metro': 'tannermetro',
    'leah panda': 'leahpanda',
    'leah pandas': "leahpanda's",

}

mapping = {
    '-': '',
    'breech': 'breach',
    'vcc': 'vct',
    'athena': 'athxna',
    'sagitting': 'sajedene',
    'genji': 'gen g',
    'valerant': 'valorant',
    'dualist': 'duelist',

}

conditional_replacements = {
    ('Game_Changers', 'TSM'): {'kat': 'cath', 'kath': 'cath',},

}

acceptable_values = set(mapping.values())
acceptable_values.update(
    ['gx3'])
acceptable_values.update([x + "'s" for x in acceptable_values])

raw_dir = r'D:\Data\speech\esports\valorant\raw'
output_dir = r'D:\Data\speech\esports\valorant\cleaned'

sub_dirs = [
    r'game_changers',
    r'platchat_valorant'
]

multiword_checks = [
                    ]

for folder in sub_dirs:
    root = os.path.join(raw_dir, folder)
    out_dir = os.path.join(output_dir, folder)
    os.makedirs(out_dir, exist_ok=True)
    files = os.listdir(root)
    for i, file in enumerate(files):
        if not file.endswith('.TextGrid'):
            #if file.endswith('.flac'):
            #    os.symlink(os.path.join(root, file), os.path.join(out_dir, file))
            #    error
            continue
        print(file, i, len(files))
        textgrid_path = os.path.join(root, file)
        out_textgrid_path = os.path.join(out_dir, file)
        tg = tgio.openTextgrid(textgrid_path)
        new_tg = tgio.Textgrid()
        tier_name = tg.tierNameList[0]
        tier = tg.tierDict[tg.tierNameList[0]]
        word_tier = tgio.IntervalTier(tier_name, [], 0, tg.maxTimestamp)
        filtered_intervals = []
        for interval in tier.entryList:
            text = interval.label.lower()
            if text.startswith('[') and text.endswith(']'):
                continue
            text = text.split()
            if len(text) == 1 and filtered_intervals:
                text = filtered_intervals[-1][2].split() + text
                filtered_intervals[-1] = (filtered_intervals[-1][0], interval.end, ' '.join(text))
            else:
                filtered_intervals.append((interval.start, interval.end, ' '.join(text)))

        # second pass

        for i, interval in enumerate(filtered_intervals):
            text = interval[2]
            for x, r in multiword_mappings.items():
                if x in text:
                    text = re.sub(r'(?<= ){}(?= )'.format(re.escape(x)), r, text)
                    text = re.sub(r'^{}(?= )'.format(re.escape(x)), r, text)
                    text = re.sub(r'(?<= ){}$'.format(re.escape(x)), r, text)
                    text = re.sub(r'^{}$'.format(re.escape(x)), r, text)
            text = text.split()
            if i != len(filtered_intervals) - 1:
                next_text = filtered_intervals[i + 1][2]
                next_text = next_text.split()
                combined = text[-1] + ' ' + next_text[0]
                if len(text) == 1:
                    more_combined = None
                else:
                    more_combined = text[-2] + ' ' +text[-1] + ' ' + next_text[0]
                if combined in multiword_mappings:
                    text[-1] = multiword_mappings[combined]
                    next_text = next_text[1:]
                    filtered_intervals[i + 1] = (filtered_intervals[i + 1][0],
                                                 filtered_intervals[i + 1][1],
                                                 ' '.join(next_text))
                elif more_combined in multiword_mappings:
                    text = text[:-1]
                    text[-1] = multiword_mappings[more_combined]
                    next_text = next_text[1:]
                    filtered_intervals[i + 1] = ( filtered_intervals[i + 1][0],
                                                  filtered_intervals[i + 1][1],
                                                  ' '.join(next_text))
                #elif text[-1] in multiword_checks:
                #    print('Combination')
                #    print(text)
                #    print(next_text)
                #    print(combined)
                #elif next_text[0] in multiword_checks:
                #    print('Combination')
                #    print(text)
                #    print(next_text)
                #    print(combined)
            filtered_intervals[i] = (interval[0], interval[1], ' '.join(text))

        for ind, interval in enumerate(filtered_intervals):
            text = interval[2]
            if "[ __ ]" in text:
                text = text.replace("[ __ ]", '')
            text = text.split()
            new_label = []
            for t in text:
                if not t:
                    continue
                try:
                    if re.search(r'[,]$', t):
                        t = t[:-1]
                    for k, v in conditional_replacements.items():
                        conditional_check = True
                        for conditional in k:
                            if conditional not in file:
                                conditional_check = False
                                break
                        if not conditional_check:
                            continue
                        if t in v:
                            t = v[t]
                    if t in mapping:
                        t = mapping[t]
                    elif "'" in t:
                        pre, suf = t.split("'", maxsplit=1)
                        if pre in mapping:
                            t = mapping[pre] + "'" + suf
                    elif "-" in t:
                        pre, suf = t.split("-", maxsplit=1)
                        if pre in mapping:
                            t = mapping[pre] + "-" + suf

                    if not re.search(r'\d', t):
                        new_label.append(t)
                        continue

                    if re.match(r'^\$[\d,]+$', t):
                        t = t[1:].replace(',', '')
                        t = num2words(t)
                        if t == 'one':
                            t += ' dollar'
                        else:
                            t += ' dollars'
                    elif re.match(r'^\$0\.\d+$', t):
                        t = t.split('.')[-1]
                        t = num2words(t)
                        if t == 'one':
                            t += ' cent'
                        else:
                            t += ' cents'
                    elif re.match(r'^\d+[%]$', t):
                        t = t[:-1]
                        t = num2words(t)
                        t += ' percent'
                    elif re.match(r'^\d+\.\d+[%]$', t):
                        t = t[:-1]
                        nums = t.split('.')
                        t = ' point '.join([num2words(x) for x in nums])
                        t += ' percent'
                    elif re.match(r'^\d+[kK]$', t):
                        t = t[:-1]
                        t = num2words(t)
                        t += ' k.'
                    elif re.match(r'^\d+\.\d+[kK]$', t):
                        t = t[:-1]
                        nums = t.split('.')
                        t = ' point '.join([num2words(x) for x in nums]) + ' k.'
                    elif re.match(r'^\d+-\d+$', t):
                        nums = t.split('-')
                        t = [num2words(x) for x in nums]
                        t = ' to '.join(t)
                    elif re.match(r'^\d+-[a-zA-Z]+$', t):
                        t, suffix = t.split('-')
                        t = num2words(t)
                        t += ' ' + suffix
                    elif re.match(r'^\d+:00$', t):
                        t = t[:-3]
                        t = num2words(t)
                    elif re.match(r'^\d+:\d\d$', t):
                        nums = t.split(':')
                        t = [num2words(x) for x in nums]
                        t = ' '.join(t)
                    elif re.match(r'^\dv\d$', t):
                        nums = t.split('v')
                        t = ' v. '.join([num2words(x) for x in nums])
                    elif re.match(r'\d+(th|rd|st|nd)$', t):
                        t = t[:-2]
                        t = num2words(t, to='ordinal')
                    elif re.match(r'^\d+\.\d+$', t):
                        nums = t.split('.')
                        t = ' point '.join([num2words(x) for x in nums])
                    elif re.match(r'^\d+:\d+$', t):
                        nums = t.split(':')
                        t = ' to '.join([num2words(x) for x in nums])
                    elif re.match(r'^[\d,]+$', t):
                        t = num2words(t.replace(',', ''))
                    elif re.match(r'^[-]\d+$', t):
                        t = t[1:]
                        prefix = 'minus '
                        t = prefix + num2words(t)
                    elif re.match(r'^[^0-9]-?\d+$', t):
                        t = t.replace('-', '')
                        prefix = t[0]
                        t = t[1:]
                        t = prefix + '. ' + num2words(t)
                    elif re.match(r'^[^0-9]+-\d+$', t):
                        prefix, t = t.split('-')
                        t = prefix + ' ' + num2words(t)
                    elif re.match(r'^\d+\D$', t):
                        suffix = t[-1]
                        t = t[:-1]
                        t = num2words(t) + ' ' + suffix + '.'
                    elif re.match(r'^\d+(hp|hb|cp|cb|am|pm|ms|hv|mg)$', t):
                        suffix = t[-2]
                        if suffix in ['hb', 'hv']:
                            suffix = 'hp'
                        elif suffix in ['cb']:
                            suffix = 'cp'
                        t = t[:-2]
                        t = num2words(t) + ' ' + suffix
                    elif re.match(r'^\d+(fps|ish)$', t):
                        suffix = t[-3]
                        t = t[:-3]
                        t = num2words(t) + ' ' + suffix
                    if not t or t in ['[', ']', '__']:
                        continue
                    if re.search(r'\d', t) and t not in acceptable_values:
                        print(text)
                        print(new_label)
                        print(t)
                    t = t.replace('/', ' ')
                    new_label.append(t)
                except decimal.InvalidOperation:
                    print(text)
                    print(t)
                    raise
            if False:
                for x in multiword_checks:
                    if x in new_label:
                        print()
                        print('FOUND:', x)
                        if x == new_label[0]:
                            print('PREVIOUS:', word_tier.entryList[-1][2])
                        print('OLD:', text)
                        print('CURRENT:', new_label)
                        if x == new_label[-1] and ind < len(filtered_intervals) - 1:
                            print('NEXT:', filtered_intervals[ind + 1][2])
            new_label = ' '.join(new_label)
            if word_tier.entryList and word_tier.entryList[-1][2] == '' and new_label == '':
                word_tier.entryList[-1] = (word_tier.entryList[-1][0], interval[1], new_label)
            else:
                word_tier.entryList.append((interval[0], interval[1], new_label))
            # print(interval)
        new_tg.addTier(word_tier)
        new_tg.save(out_textgrid_path)


import socket
socks = socket.socket()
socks.connect(('data.pr4e.org',80))
cmd = 'GET http://data.pr4e.org/romeo.txt HTTP/1.0\r\n\r\n'.encode()
socks.send(cmd)

while True:
    data = socks.recv(512)
    if len(data) <1 :
        break
    print(data.decode(),end='')

socks.close()


import pandas

datContent = [read.strip().split() for read in open("input_roundrobin.dat").readlines()]
n = int(datContent[0][0])
j = 1
arrival = []
burst = []
arrival2=[]
burst2=[]
process= []
priority = []
priority2=[]
for i in range(0,n):
    a ,b,c = datContent[j][0] ,  datContent[j][1], datContent[j][2]
    a = int(a)
    b = int(b)
    c = int(c)
    arrival.append(a)
    arrival2.append(a)
    burst.append(b)
    burst2.append(b)
    priority.append(c)
    priority2.append(c)
    j = j+1

quantum = int(datContent[n + 1][0])
queue = int(datContent[n + 2][0])

for i in range(0,n):
    process.append(i)

zipp = zip(process,arrival,burst,priority)
tag = sorted((zipp),key=lambda x: (x[1],x[2],))

arr2 = []
bur2 = []
proc2 =[]
prio2= []
for i in tag:
    arr2.append(i[1])
    bur2.append(i[2])
    proc2.append(i[0])
    prio2.append(i[3])

progress = []
time = 0
time_count  = 0
turn_around= []
wait_time = []
temp_proc = []
exit_time = []
rem_time = []
for i in bur2:
    rem_time.append(i)
remain = n
rq = 0
flag = 0
time = 0
start = []
count = 0
fire = 1

time_count = []
time_count.append(0)
progress_count = []

print(rem_time)
while ( remain !=0  ):
    if rem_time[count] <= quantum and rem_time[count] >0 :
        if fire ==1:
            progress.append(count)

        time = time + rem_time[count]
        time_count.append(time)
        progress_count.append(proc2[count])
        rem_time[count] = 0
        flag = 1
    elif rem_time[count] >0:
        if fire==1:
            progress.append(count)

        rem_time[count] = rem_time[count] - quantum
        time = time + quantum
        time_count.append(time)
        progress_count.append(proc2[count])

    if rem_time[count]==0 and flag==1:
        if fire ==1:
            progress.append(count)
            fire =1
        exit_time.append(time)
        remain = remain - 1
        temp_proc.append(proc2[count])
        turn_around.append(time - arr2[count])

        wait_time.append(time - arr2[count]-bur2[count])
        flag=0

    if count == n-1:

        count = 0
    elif arr2[count +1]<=time:

        count = count + 1
        #time_me.append([time,count])
        start.append(time)
    else:
        count = 0

print("Printing Order",temp_proc)
print("Turn Around",turn_around)
print("Wait Time",wait_time)
print("Gantt Time",start)
print("Progress",progress)
print("Time count",time_count)
print("Progress Count",progress_count)
start_time = []
for i in temp_proc:
    k = progress_count.index(i)
    start_time.append(time_count[k])
print("Start time",start_time)

#Response Time
response_time = []
for i in range(0,n):

    k = temp_proc[i]
    response_time.append(start_time[i]-arrival2[k])

print("Response Time",response_time)

print("Avg Turn-Around",sum(turn_around)/n)
print("Avg Waiting",sum(wait_time)/n)
print("Avg Response",sum(response_time)/n)


import h5py
from sklearn.linear_model import LinearRegression
import nltk
import numpy as np
import pickle
import os
import copy
import logging
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence

class Encoder(nn.Module):
    """GenSen Encoder.

    Original source in https://github.com/Maluuba/gensen
    """

    def __init__(
        self, vocab_size, embedding_dim, hidden_dim, num_layers, rnn_type="GRU"
    ):
        """Initialize all the parameters.

        Args:
            vocab_size (int): Size of the vocabulary.
            embedding_dim (int): the size of each embedding vector
            hidden_dim (int): the size of each hidden vector
            num_layers (int): the number of layers.
            rnn_type (str): Type of RNN.
        """
        super(Encoder, self).__init__()
        self.rnn_type = rnn_type
        rnn = getattr(nn, rnn_type)
        self.src_embedding = nn.Embedding(
            num_embeddings=vocab_size, embedding_dim=embedding_dim
        )

        self.encoder = rnn(
            input_size=embedding_dim,
            hidden_size=hidden_dim,
            num_layers=num_layers,
            batch_first=True,
            bidirectional=True,
        )

    def set_pretrained_embeddings(self, embedding_matrix):
        """Set embedding weights.

        Args:
            embedding_matrix(torch.Tensor): Embedding matrix.

        """
        if embedding_matrix.shape[0] != self.src_embedding.weight.size(
            0
        ) or embedding_matrix.shape[1] != self.src_embedding.weight.size(1):
            logging.info(
                """
                Warning pretrained embedding shape mismatch %d x %d
                expected %d x %d"""
                % (
                    embedding_matrix.shape[0],
                    embedding_matrix.shape[1],
                    self.src_embedding.weight.size(0),
                    self.src_embedding.weight.size(1),
                )
            )
            self.src_embedding = nn.Embedding(
                embedding_matrix.shape[0], embedding_matrix.shape[1]
            )
            self.src_vocab_size = embedding_matrix.shape[0]
            self.src_emb_dim = embedding_matrix.shape[1]

        try:
            self.src_embedding.weight.data.set_(
                torch.from_numpy(embedding_matrix)
            )
        except BaseException:
            self.src_embedding.weight.data.set_(
                torch.from_numpy(embedding_matrix).cuda()
            )

        self.src_embedding.cuda()

    def forward(self, input, lengths, return_all=False, pool="last"):
        """Propogate input through the encoder.

        Args:
            input(torch.Tensor): Embedding matrix
            lengths (torch.Tensor): list of sequences lengths of each batch element.
            return_all (bool): Return all or only the last hidden state.
            pool(str): Type of getting hidden state.

        Returns:
            torch.Tensor: Return last or all hidden states.

        """
        embedding = self.src_embedding(input)
        src_emb = pack_padded_sequence(embedding, lengths, batch_first=True)
        if self.rnn_type == "LSTM":
            h, (h_t, _) = self.encoder(src_emb)
        else:
            h, h_t = self.encoder(src_emb)

        # Get hidden state via max-pooling or h_t
        if pool == "last":
            h_t = torch.cat((h_t[-1], h_t[-2]), 1)
        elif pool == "max":
            h_tmp, _ = pad_packed_sequence(h, batch_first=True)
            h_t = torch.max(h_tmp, 1)[0].squeeze()
        else:
            raise ValueError("Pool %s is not valid " % (pool))

        # Return all or only the last hidden state
        if return_all:
            h, _ = pad_packed_sequence(h, batch_first=True)
            return h, h_t
        else:
            return h_t

class GenSen(nn.Module):
    """A wrapper class for multiple GenSen models.

    Original source in https://github.com/Maluuba/gensen
    """

    def __init__(self, *args, **kwargs):
        super(GenSen, self).__init__()
        self.gensen_models = args

    def vocab_expansion(self, task_vocab):
        """Expand the model's vocabulary with pretrained word embeddings.

        Args:
            task_vocab(list): Vocabulary for each task.

        """
        for model in self.gensen_models:
            model.vocab_expansion(task_vocab)

    def get_representation(
        self,
        sentences,
        pool="last",
        tokenize=False,
        return_numpy=True,
        add_start_end=True,
    ):
        """ Get model representations.

        Args:
            sentences(list): Sentences to get embeddings.
            pool(str): Type of getting hidden state.
            tokenize(bool): To tokenize or not.
            return_numpy(bool): To return a numpy array or not.
            add_start_end(bool): To add start and end notation (<s> </s>) to each
            sentence or not.

        Returns:
            torch.Tensor : Return last or all hidden states.

        """
        representations = [
            model.get_representation(
                sentences,
                pool=pool,
                tokenize=tokenize,
                return_numpy=return_numpy,
                add_start_end=add_start_end,
            )
            for model in self.gensen_models
        ]
        if return_numpy:
            return (
                np.concatenate([x[0] for x in representations], axis=2),
                np.concatenate([x[1] for x in representations], axis=1),
            )
        else:
            return (
                torch.cat([x[0] for x in representations], 2),
                torch.cat([x[1] for x in representations], 1),
            )

class GenSenSingle(nn.Module):
    """GenSen Wrapper.

    Original source in https://github.com/Maluuba/gensen
    """

    def __init__(
        self,
        model_folder,
        filename_prefix,
        pretrained_emb,
        cuda=False,
        rnn_type="GRU",
    ):
        """ Initialize params.

        Args:
            model_folder(str): Folder where the model resides.
            filename_prefix(str): Prefix for the model file.
            pretrained_emb(torch.Tensor): Pretrained Embedding vector.
            cuda(bool): Use Cuda or not.
            rnn_type(str): Type of RNN.
        """

        super(GenSenSingle, self).__init__()
        self.model_folder = model_folder
        self.filename_prefix = filename_prefix
        self.pretrained_emb = pretrained_emb
        self.cuda = cuda
        self.rnn_type = rnn_type
        self._load_params()
        self.vocab_expanded = False

    def _load_params(self):
        """Load pretrained params."""
        # Read vocab pickle files
        open(
            os.path.join(
                self.model_folder, "%s_vocab.pkl" % self.filename_prefix
            ),
            "rb",
        )
        model_vocab = pickle.load(
            open(
                os.path.join(
                    self.model_folder, "%s_vocab.pkl" % self.filename_prefix
                ),
                "rb",
            ),
            encoding="latin1",
        )

        # Word to index mappings
        self.word2id = model_vocab["word2id"]
        self.id2word = model_vocab["id2word"]
        self.task_word2id = self.word2id
        self.id2word = self.id2word

        encoder_model = torch.load(
            os.path.join(
                self.model_folder, "%s.model" % (self.filename_prefix)
            )
        )

        # Initialize encoders
        self.encoder = Encoder(
            vocab_size=encoder_model["src_embedding.weight"].size(0),
            embedding_dim=encoder_model["src_embedding.weight"].size(1),
            hidden_dim=encoder_model["encoder.weight_hh_l0"].size(1),
            num_layers=1 if len(encoder_model) < 10 else 2,
            rnn_type=self.rnn_type,
        )

        # Load pretrained sentence encoder weights
        self.encoder.load_state_dict(encoder_model)

        # Set encoders in eval model.
        self.encoder.eval()

        # Store the initial word embeddings somewhere to re-train vocab expansion multiple times.
        self.model_embedding_matrix = copy.deepcopy(
            self.encoder.src_embedding.weight.data.cpu().numpy()
        )

        # Move encoder to GPU if self.cuda
        if self.cuda:
            self.encoder = self.encoder.cuda()

    def first_expansion(self):
        """ Training linear regression model for the first time."""

        # Read pre-trained word embedding h5 file
        logging.info("Loading pretrained word embeddings")
        pretrained_embeddings = h5py.File(self.pretrained_emb)
        pretrained_embedding_matrix = pretrained_embeddings["embedding"].value
        pretrain_vocab = pretrained_embeddings["words_flatten"].value.split(
            "\n"
        )
        pretrain_word2id = {
            word: ind for ind, word in enumerate(pretrain_vocab)
        }

        # Set up training data for vocabulary expansion
        model_train = []
        pretrain_train = []

        for word in pretrain_word2id:
            if word in self.word2id:
                model_train.append(
                    self.model_embedding_matrix[self.word2id[word]]
                )
                pretrain_train.append(
                    pretrained_embedding_matrix[pretrain_word2id[word]]
                )

        logging.info("Training vocab expansion on model")
        lreg = LinearRegression()
        lreg.fit(pretrain_train, model_train)
        self.lreg = lreg
        self.pretrain_word2id = pretrain_word2id
        self.pretrained_embedding_matrix = pretrained_embedding_matrix

    def vocab_expansion(self, task_vocab):
        """ Expand the model's vocabulary with pretrained word embeddings.

        Args:
            task_vocab(list): Vocabulary for each task.
        """

        self.task_word2id = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

        self.task_id2word = {0: "<s>", 1: "<pad>", 2: "</s>", 3: "<unk>"}

        ctr = 4
        for idx, word in enumerate(task_vocab):
            if word not in self.task_word2id:
                self.task_word2id[word] = ctr
                self.task_id2word[ctr] = word
                ctr += 1

        if not self.vocab_expanded:
            self.first_expansion()

        # Expand vocabulary using the linear regression model
        task_embeddings = []
        oov_pretrain = 0
        oov_task = 0

        for word in self.task_id2word.values():
            if word in self.word2id:
                task_embeddings.append(
                    self.model_embedding_matrix[self.word2id[word]]
                )
            elif word in self.pretrain_word2id:
                oov_task += 1
                task_embeddings.append(
                    self.lreg.predict(
                        self.pretrained_embedding_matrix[
                            self.pretrain_word2id[word]
                        ].reshape(1, -1)
                    )
                    .squeeze()
                    .astype(np.float32)
                )
            else:
                oov_pretrain += 1
                oov_task += 1
                task_embeddings.append(
                    self.model_embedding_matrix[self.word2id["<unk>"]]
                )

        logging.info("Found %d task OOVs " % (oov_task))
        logging.info("Found %d pretrain OOVs " % (oov_pretrain))
        task_embeddings = np.stack(task_embeddings)
        self.encoder.set_pretrained_embeddings(task_embeddings)
        self.vocab_expanded = True

        # Move encoder to GPU if self.cuda
        if self.cuda:
            self.encoder = self.encoder.cuda()

    def get_minibatch(self, sentences, tokenize=False, add_start_end=True):
        """Prepare minibatch.

        Args:
            sentences(list): Sentences to get embeddings.
            tokenize(bool): To tokenize or not.
            add_start_end(bool): To add start and end notation (<s> </s>) to each
            sentence or not.

        Returns:
            dict: A dictionary with sentences, lengths and sentence embeddings.

        """
        if tokenize:
            sentences = [
                nltk.word_tokenize(sentence) for sentence in sentences
            ]
        else:
            sentences = [sentence.split() for sentence in sentences]

        if add_start_end:
            sentences = [
                ["<s>"] + sentence + ["</s>"] for sentence in sentences
            ]

        lens = [len(sentence) for sentence in sentences]
        sorted_idx = np.argsort(lens)[::-1]
        sorted_sentences = [sentences[idx] for idx in sorted_idx]
        rev = np.argsort(sorted_idx)
        sorted_lens = [len(sentence) for sentence in sorted_sentences]
        max_len = max(sorted_lens)

        sentences = [
            [
                self.task_word2id[w]
                if w in self.task_word2id
                else self.task_word2id["<unk>"]
                for w in sentence
            ]
            + [self.task_word2id["<pad>"]] * (max_len - len(sentence))
            for sentence in sorted_sentences
        ]

        with torch.no_grad():
            sentences = Variable(torch.LongTensor(sentences))
            rev = Variable(torch.LongTensor(rev))
        lengths = sorted_lens

        if self.cuda:
            sentences = sentences.cuda()
            rev = rev.cuda()

        return {"sentences": sentences, "lengths": lengths, "rev": rev}

    def get_representation(
        self,
        sentences,
        pool="last",
        tokenize=False,
        return_numpy=True,
        add_start_end=True,
    ):
        """Get model representations.

        Args:
            sentences(list): Sentences to get embeddings.
            pool(str): Type of getting hidden state.
            tokenize(bool): To tokenize or not.
            return_numpy(bool): To return a numpy array or not.
            add_start_end(bool): To add start and end notation (<s> </s>) to each
            sentence or not.

        Returns:
            torch.Tensor : Return last or all hidden states.

        """

        minibatch = self.get_minibatch(
            sentences, tokenize=tokenize, add_start_end=add_start_end
        )
        h, h_t = self.encoder(
            input=minibatch["sentences"],
            lengths=minibatch["lengths"],
            return_all=True,
            pool=pool,
        )
        h = h.index_select(0, minibatch["rev"])
        h_t = h_t.index_select(0, minibatch["rev"])
        if return_numpy:
            return h.data.cpu().numpy(), h_t.data.cpu().numpy()
        else:
            return h, h_t


from funboost import Booster, BoostersManager,BrokerEnum,BoosterParams

def add(a, b):
    print(a + b)

def my_push(quue_name, a, b):
    booster = BoostersManager.build_booster(BoosterParams(queue_name=quue_name, qps=0.2, consuming_function=add, broker_kind=BrokerEnum.REDIS))  # type: Booster
    # get_or_create_booster 这种就不会无数次去创建 消息队列连接了。
    booster.push(a, b)

for i in range(1000):
    queue_namx = f'queue_{i % 10}'
    my_push(queue_namx, i, i * 2)

print(BoostersManager.pid_queue_name__booster_map)
print(BoostersManager.queue_name__boost_params_map)
if __name__ == '__main__':

    for j in range(1):
        booster = BoostersManager.build_booster(BoosterParams(queue_name=f'queue_{j}', qps=0.2, consuming_function=add))  # type: Booster
        # booster.consume()
        booster.multi_process_consume(2)