import theano.tensor as T

from .base import Layer
from ..utils import as_tuple

from theano.tensor.signal import downsample

__all__ = [
    "MaxPool1DLayer",
    "MaxPool2DLayer",
    "FeaturePoolLayer",
    "FeatureWTALayer",
    "GlobalPoolLayer",
]

def pool_output_length(input_length, pool_size, stride,
                       ignore_border, pad):
    """
    Compute the output length of a pooling operator
    along a single dimension.

    Parameters
    ----------
    input_length : integer
        The length of the input in the pooling dimension
    pool_size : integer
        The length of the pooling region
    stride : integer
        The stride between successive pooling regions
    pad : integer
        The number of elements to be added to the input on each side.
    ignore_border: bool
        If True, partial pooling regions will be ignored.
        Must be True if pad != 0.

    Returns
    -------
    output_length
        * None if either input is None.
        * Computed length of the pooling operator otherwise.

    Notes
    -----
    When `ignore_border == True`, this is given by the number of full
    pooling regions that fit in the padded input length,
    divided by the stride (rounding down).

    If `ignore_border == False`, a single partial pooling region is
    appended if at least one input element would be left uncovered otherwise.
    """
    if input_length is None or pool_size is None:
        return None

    if ignore_border:
        output_length = input_length + 2 * pad - pool_size + 1
        output_length = (output_length + stride - 1) // stride

    # output length calculation taken from:
    # https://github.com/Theano/Theano/blob/master/theano/tensor/signal/downsample.py
    else:
        assert pad == 0

        if stride >= pool_size:
            output_length = (input_length + stride - 1) // stride
        else:
            output_length = max(
                0, (input_length - pool_size + stride - 1) // stride) + 1

    return output_length

class MaxPool1DLayer(Layer):

    """
    This layer performs max pooling over the final dimension
    of a 3D tensor.
    """

    def __init__(self, incoming, pool_size, stride=None, pad=0,
                 ignore_border=False, **kwargs):
        """
        Instantiates the layer.

        Parameters
        ----------
        incoming : a :class:`Layer` instance or tuple
            The layer feeding into this layer, or the expected input shape.
        pool_size : integer
            The length of the pooling region
        stride : integer or None
            The stride between sucessive pooling regions.
            If None, stride = pool_size.
        pad : integer
            The number of elements to be added to the input on each side.
            Must be less than stride.
        ignore_border : bool
            If True, partial pooling regions will be ignored.
            Must be True if pad != 0.

        Notes
        -----
        The value used to pad the input is chosen to be less than
        the minimum of the input, so that the output of each pooling region
        always corresponds to some element in the unpadded input region.
        """
        super(MaxPool1DLayer, self).__init__(incoming, **kwargs)
        self.pool_size = as_tuple(pool_size, 1)
        self.stride = self.pool_size if stride is None else as_tuple(stride, 1)
        self.pad = as_tuple(pad, 1)
        self.ignore_border = ignore_border

    def get_output_shape_for(self, input_shape):
        output_shape = list(input_shape)  # copy / convert to mutable list

        output_shape[-1] = pool_output_length(input_shape[-1],
                                              pool_size=self.pool_size[0],
                                              stride=self.stride[0],
                                              ignore_border=self.ignore_border,
                                              pad=self.pad[0],
                                              )

        return tuple(output_shape)

    def get_output_for(self, input, **kwargs):
        input_4d = T.shape_padright(input, 1)

        pooled = downsample.max_pool_2d(input_4d,
                                        ds=(self.pool_size[0], 1),
                                        st=(self.stride[0], 1),
                                        ignore_border=self.ignore_border,
                                        padding=(self.pad[0], 0),
                                        )
        return pooled[:, :, :, 0]

class MaxPool2DLayer(Layer):

    """
    This layer performs max pooling over the last two dimensions
    of a 4D tensor.
    """

    def __init__(self, incoming, pool_size, stride=None,
                 ignore_border=False, pad=(0, 0), **kwargs):
        """
        Instantiates the layer.

        Parameters
        ----------
        incoming : a :class:`Layer` instance or tuple
            The layer feeding into this layer, or the expected input shape.
        pool_size : integer or iterable
            The length of the pooling region in each dimension
        stride : integer, iterable or None
            The strides between sucessive pooling regions in each dimension.
            If None, stride = pool_size.
        pad : integer or iterable
            Number of elements to be added on each side of the input
            in each dimension. Each value must be less than
            the corresponding stride.
        ignore_border : bool
            If True, partial pooling regions will be ignored.
            Must be True if pad != (0, 0).

        Notes
        -----
        The value used to pad the input is chosen to be less than
        the minimum of the input, so that the output of each pooling region
        always corresponds to some element in the unpadded input region.
        """
        super(MaxPool2DLayer, self).__init__(incoming, **kwargs)

        self.pool_size = as_tuple(pool_size, 2)

        if stride is None:
            self.stride = self.pool_size
        else:
            self.stride = as_tuple(stride, 2)

        self.pad = as_tuple(pad, 2)

        self.ignore_border = ignore_border

    def get_output_shape_for(self, input_shape):
        output_shape = list(input_shape)  # copy / convert to mutable list

        output_shape[2] = pool_output_length(input_shape[2],
                                             pool_size=self.pool_size[0],
                                             stride=self.stride[0],
                                             ignore_border=self.ignore_border,
                                             pad=self.pad[0],
                                             )

        output_shape[3] = pool_output_length(input_shape[3],
                                             pool_size=self.pool_size[1],
                                             stride=self.stride[1],
                                             ignore_border=self.ignore_border,
                                             pad=self.pad[1],
                                             )

        return tuple(output_shape)

    def get_output_for(self, input, **kwargs):
        pooled = downsample.max_pool_2d(input,
                                        ds=self.pool_size,
                                        st=self.stride,
                                        ignore_border=self.ignore_border,
                                        padding=self.pad,
                                        )
        return pooled

# TODO: add reshape-based implementation to MaxPool*DLayer
# TODO: add MaxPool3DLayer

class FeaturePoolLayer(Layer):

    """
    This layer performs pooling across feature maps, which can be used
    to implement maxout.
    IMPORTANT: this layer requires that the number of feature maps is
    a multiple of the pool size.
    """

    def __init__(self, incoming, pool_size, axis=1, pool_function=T.max,
                 **kwargs):
        """
        Instantiates the layer.

        Parameters
        ----------
        pool_size : integer
            the number of feature maps to be pooled together
        axis : integer
            the axis along which to pool. The default value of 1 works
            for DenseLayer and Conv*DLayers
        pool_function : the pooling function to use
        """
        super(FeaturePoolLayer, self).__init__(incoming, **kwargs)
        self.pool_size = pool_size
        self.axis = axis
        self.pool_function = pool_function

        num_feature_maps = self.input_shape[self.axis]
        if num_feature_maps % self.pool_size != 0:
            raise RuntimeError("Number of input feature maps (%d) is not a "
                               "multiple of the pool size (pool_size=%d)" %
                               (num_feature_maps, self.pool_size))

    def get_output_shape_for(self, input_shape):
        output_shape = list(input_shape)  # make a mutable copy
        output_shape[self.axis] = input_shape[self.axis] // self.pool_size
        return tuple(output_shape)

    def get_output_for(self, input, **kwargs):
        input_shape = tuple(input.shape)
        num_feature_maps = input_shape[self.axis]
        num_feature_maps_out = num_feature_maps // self.pool_size

        pool_shape = (input_shape[:self.axis] +
                      (num_feature_maps_out, self.pool_size) +
                      input_shape[self.axis+1:])

        input_reshaped = input.reshape(pool_shape)
        return self.pool_function(input_reshaped, axis=self.axis + 1)

class FeatureWTALayer(Layer):

    """
    Perform 'Winner Take All' across feature maps: zero out all but
    the maximal activation value within a group of features.
    IMPORTANT: this layer requires that the number of feature maps is
    a multiple of the pool size.
    """

    def __init__(self, incoming, pool_size, axis=1, **kwargs):
        """
        Instantiates the layer.

        Parameters
        ----------
        pool_size : integer
            the number of feature maps per group.
        axis : integer
            the axis along which the groups are formed.
        """
        super(FeatureWTALayer, self).__init__(incoming, **kwargs)
        self.pool_size = pool_size
        self.axis = axis

        num_feature_maps = self.input_shape[self.axis]
        if num_feature_maps % self.pool_size != 0:
            raise RuntimeError("Number of input feature maps (%d) is not a "
                               "multiple of the group size (pool_size=%d)" %
                               (num_feature_maps, self.pool_size))

    def get_output_for(self, input, **kwargs):
        num_feature_maps = input.shape[self.axis]
        num_pools = num_feature_maps // self.pool_size

        pool_shape = ()
        arange_shuffle_pattern = ()
        for k in range(self.axis):
            pool_shape += (input.shape[k],)
            arange_shuffle_pattern += ('x',)

        pool_shape += (num_pools, self.pool_size)
        arange_shuffle_pattern += ('x', 0)

        for k in range(self.axis + 1, input.ndim):
            pool_shape += (input.shape[k],)
            arange_shuffle_pattern += ('x',)

        input_reshaped = input.reshape(pool_shape)
        max_indices = T.argmax(input_reshaped, axis=self.axis + 1,
                               keepdims=True)

        arange = T.arange(self.pool_size).dimshuffle(*arange_shuffle_pattern)
        mask = T.eq(max_indices, arange).reshape(input.shape)

        return input * mask

class GlobalPoolLayer(Layer):

    """
    Layer that pools globally across all trailing dimensions beyond the 2nd.
    """

    def __init__(self, incoming, pool_function=T.mean, **kwargs):
        super(GlobalPoolLayer, self).__init__(incoming, **kwargs)
        self.pool_function = pool_function

    def get_output_shape_for(self, input_shape):
        return input_shape[:2]

    def get_output_for(self, input, **kwargs):
        return self.pool_function(input.flatten(3), axis=2)


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 20 00:07:47 2022

@author: USER07
"""

from keras.datasets import reuters
(train_data,train_labels),(test_data,test_labels)=reuters.load_data(num_words=10000)

train_labels[0]

import numpy as np
def vectorize_sequence(sequences,dimension=10000):
    results=np.zeros((len(sequences),dimension))
    for i,sequence in enumerate(sequences):
        results[i,sequence]=1

    return results

x_train=vectorize_sequence(train_data)
x_test=vectorize_sequence(test_data)

from tensorflow.keras.utils import to_categorical

#one_hot_train_labels=to_categorical(train_labels)
#one_hot_test_labels=to_categorical(test_labels)

y_train = to_categorical(train_labels)
y_test = to_categorical(test_labels)

from tensorflow import keras
from tensorflow.keras import layers

model=keras.Sequential([
    layers.Dense(64, activation="relu"),
    layers.Dense(64, activation="relu"),
    layers.Dense(46, activation="softmax")
    ])

model.compile(optimizer="rmsprop",
              loss="categorical_crossentropy",
              metrics=["accuracy"])

x_val = x_train[:1000]
partial_x_train = x_train[1000:]
y_val = y_train[:1000]
partial_y_train = y_train[1000:]

history = model.fit(partial_x_train,
partial_y_train,
epochs=20,
batch_size=512,
validation_data=(x_val, y_val))

import matplotlib.pyplot as plt
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend()
plt.show()

plt.clf()
acc = history.history["accuracy"]
val_acc = history.history["val_accuracy"]
plt.plot(epochs, acc, "bo", label="Training accuracy")
plt.plot(epochs, val_acc, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend()
plt.show()

predictions = model.predict(x_test)

predictions[0].shape

np.argmax(predictions[0])

model = keras.Sequential([
layers.Dense(64, activation="relu"),
layers.Dense(2, activation="relu"),
layers.Dense(46, activation="softmax")
])
model.compile(optimizer="rmsprop",
loss="categorical_crossentropy",
metrics=["accuracy"])
model.fit(partial_x_train,
partial_y_train,
epochs=20,
batch_size=128,
validation_data=(x_val, y_val))


import argparse
from collections import defaultdict
import os
from pathlib import Path
import pandas as pd
import sqlalchemy
import glob

import csv

import io

import logging

parser = argparse.ArgumentParser()
dtypes: defaultdict = defaultdict(lambda: str)

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.WARNING)
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
    logger.setLevel(logging.WARNING)

def make_tablename(filename):
    return f"census_" + Path(filename).stem

def main():
    parser.add_argument("--path", type=str, required=True)

    args = parser.parse_args()
    print(args)
    engine = sqlalchemy.create_engine(
        "postgresql://postgres@localhost/postgres"
    )

    files = glob.glob(f'{args.path}/*.txt')

    for txtfile in files:
        connection = engine.raw_connection()
        cursor = connection.cursor()
        tablename = Path(txtfile).stem
        print(f'Dropping {tablename}')
        cursor.execute(f"DROP TABLE IF EXISTS IF EXISTS {make_tablename(tablename)}")

    for txtfile in files:
        tablename = Path(txtfile).stem
        print(f'Loading {txtfile} / {make_tablename(tablename)}')
        #file = open(f"{txtfile}", "rb")
        #file_bytes = file.read()
        #file.close()
        for chunk in pd.read_csv(
            #io.BytesIO(file_bytes),
            txtfile,
            delimiter="|",
            dtype=dtypes,
            on_bad_lines="skip",
            engine="python",
            quoting=csv.QUOTE_NONE,
            encoding="utf-8",
            chunksize=10000
        ):
            chunk.to_sql(name=make_tablename(tablename),
                        if_exists='append',
                         con=engine)

if __name__ in "__main__":
    main()


from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from santhob.preprocessing import ImageToArrayPreProcessor
from santhob.preprocessing import SimplePreprocessor
from santhob.datasets import SimpleDatasetLoader
from santhob.nn.cnn import ShallowNet
from keras.optimizers import SGD
from imutils import paths
import matplotlib.pyplot as plt
import numpy as np
import argparse
from tqdm import tqdm
import os

DATADIR = "/home/guru/Desktop/pyimagebook/chapter12/santhob/datasets"

CATEGORIES = ["dog", "cat","panda"]

for category in CATEGORIES:  # do dogs and cats
    path = os.path.join(DATADIR,category)  # create path to dogs and cats
    for img in os.listdir(path):  # iterate over each image per dogs and cats
        img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE)  # convert to array
        plt.imshow(img_array, cmap='gray')  # graph it
        plt.show()  # display!

        break  # we just want one for now so break
    break

#grab the list of images that we are describing

print("Images are loading...")
imagePaths = list(paths.list_images(args["dataset"]))
#imagepaths = list(paths.list_images(args['dataset']))

#initialize the image preprocessor

sp=SimplePreprocessor(32,32)
iap=ImageToArrayPreProcessor()

#load the dataset from disk and scale to range between [0,1]

sd1=SimpleDatasetLoader(preprocessors=[sp,iap])
(data,labels)=sd1.load(imagePaths,verbose=500)

data=data.astype("float") / 255.0

#partition the dataser to train and test dataset

(trainx,testx,trainy,testy)=train_test_split(data,labels,test_size=0.25,random_state=42)

#convert the labels from integer to vectors

trainy=LabelBinarizer().fit_transform(trainy)
testy=LabelBinarizer().fit_transform(testy)

#initialize the model and optimizer

print("compiling the model")

opt=SGD(lr=0.001)
model=ShallowNet.build(width=32,height=32,depth=3,classes=3)
model.compile(loss="categorical_crossentropy",optimizer=opt,metrics=['accuracy'])

print("train the network")

H=model.fit(trainx,trainy,validation_data=(testx,testy),epochs=10,verbose=1)

#evaluate the model

print("evaluating the model....")
predictions=model.predict(testx)
print(classification_report(testy.argmax(axis=1),predictions.argmax(axis=1),target_names=["cat","dog","panda"]))

# plot the training loss and accuracy
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, 10), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, 10), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, 10), H.history["acc"], label="train_acc")
plt.plot(np.arange(0, 10), H.history["val_acc"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.show()


"""
14.2 平移
平移就是将对象换一个位置。如果你要沿（x，y）方向移动，移动的距离
是（tx，ty），你可以以下面的方式构建移动矩阵：
        [ 1 0 tx ]
    M = [ 0 1 ty ]

你可以使用Numpy 数组构建这个矩阵（数据类型是np.float32），然
后把它传给函数cv2.warpAffine()
"""
import cv2
import numpy as np

img = cv2.imread("../../../resource/img/1.jpg");
rows, cols = img.shape[:2];

M = np.float32([[1,0,100],[0,1,100]]);
"""
cv2.warpAffine(src, M, dsize[, dst[, flags[, borderMode[, borderValue]]]]) → dst
Applies an affine transformation仿射变换 to an image.
Parameters:
    src – input image.
    M – 2 * 3 transformation matrix. 变换矩阵
    dsize – size of the output image.
"""
dst = cv2.warpAffine(img, M, (cols,rows));

cv2.imshow("dst",dst);
cv2.waitKey(3000);
cv2.destroyAllWindows();

#函数cv2.warpAffine() 的第三个参数的是输出图像的大小，它的格式应该是图像的（宽，高）。应该记住的是图像的宽对应的是列数，高对应的是行数。


import subprocess
import json

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from pandas import DataFrame, read_csv
from numpy import log
#from statistics import mean

from datetime import datetime
import time

from keras.models import Sequential, load_model
from keras.layers import LSTM, Dense, Dropout
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping, TensorBoard, ReduceLROnPlateau, CSVLogger

log_dir = "2-layer-32-32-LeakyReLU-update"

early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto')
#logger = TensorBoard(log_dir='log/{}'.format(log_dir), write_graph=True, histogram_freq=1, batch_size=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.001)
csv_logger = CSVLogger('training-2-layer-32-32-LeakyReLU-update.csv')

previous_df = read_csv('memoryUsed9-final.csv')
previous_df.columns = ['timestamp', 'value']
previous_df.set_index('timestamp', inplace=True)
previous_df.dropna(inplace=True)
print(previous_df.head())
print(previous_df.shape)
previous_values = previous_df.values

last_input_value = previous_values[-2]
prev_value = previous_values[-1]

def difference_2(dataset, interval=1):
    print(dataset.ndim)
    print(dataset.shape)
    diff = np.ndarray(shape=(dataset.shape[0]-interval, dataset.shape[1]), dtype=float)
    for i in range(interval, len(dataset)):
        for j in range(dataset.shape[1]):
            diff[i-1][j] = dataset[i][j] - dataset[i - interval][j]
    return diff

diff_values = difference_2(previous_values)
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(diff_values)

min_val = scaler.data_min_
max_val = scaler.data_max_

def mean(nums):
    return sum(nums, 0.0) / len(nums)

def streaming():
    counter = 0
    total_instance = 0
    total_mem = list()
    mem_max = 0
    while(counter != 5):
        print("in streaming loop with counter and time in minutes", counter, datetime.now().minute%5)
        instance_stat = subprocess.check_output("cf curl v2/apps/3bbffff8-6834-4383-8140-cd223d9927ee/stats | jq 'with_entries(.value = .value.stats)'", shell=True)
        instance_stat_json = json.loads(instance_stat)
        mem_quota = list()
        instances = list()
        mem_usages = list()
        #mem = 0
        for instance in instance_stat_json:
            instances.append(int(instance))
            mem_usages.append(instance_stat_json[instance]['usage']['mem'])
            mem_quota.append(instance_stat_json[instance]['mem_quota'])
            #mem = mem + int(instance)*instance_stat_json[instance]['usage']['mem']
        total_mem.append(sum(mem_usages))
        total_instance = len(instances)
        mem_max = max(mem_quota)
        # Wait for 60 seconds
        time.sleep(60)
        counter = counter+1
        if datetime.now().minute % 5 == 0:
            break
    return mean(total_mem), total_instance

def difference(data, prev_val):
    return data-prev_value

def scale_dataset(data, min_val, max_val):
    return (data - min_val)/(max_val - min_val)

def inverse_scale(data, min_val, max_val):
    return data*(max_val - min_val) + min_val

def inverse_diff(data, last_val):
    return data + last_val

def predict(data):
    print("in prediction function with data", data)
    diff_data = difference(data)
    scaled_data = scale_dataset(diff_data)
    train_X = scaled_data.reshape(scaled_data.shape[0], 1, 1)
    yhat= model.predict(train_X, batch_size=1)
    yhat = yhat.reshape(yhat.shape[0], yhat.shape[2])
    yhat_inv = inverse_scale(yhat)
    yhat_final = inverse_diff(yhat_inv, last_value)
    return yhat_final

def scale_decision(memory, instance):
    print("scaling decision function with memory and instance", memory, instance)

def update(data):
    print("update model with next data", data)

#model = load_model('mem0_5-stack-2-32-LeakyReLU.h5')

flag=1
# date_time = datetime.now()
# year, month, day = date_time.year, date_time.month, date_time.day
# hour, minute = date_time.hour, date_time.minute
# rem = minute % 5
# sleep 5-rem
print("streaming starting from : ", datetime.now())
c=0
while(flag):
    print("in infinite loop with iteration: ", c)
    total_mem, instance_count =  streaming()
    predict(total_mem)
    scale_decision(total_mem, instance_count)
    update(total_mem)
    c = c+1

# history = model.fit(train_X, train_y, epochs=100, batch_size=1, validation_data=(test_X, test_y), verbose=2, shuffle=False, callbacks=[logger, reduce_lr, csv_logger])

# mem = subprocess.check_output("cf curl v2/apps/3bbffff8-6834-4383-8140-cd223d9927ee/stats | jq 'with_entries(.value = .value.stats.usage.mem)'", shell=True)
# json_mem = json.loads(mem)
# print(json_mem)
# instances = list()
# mem_usages = list()
# for instance in json_mem:
#     print(instance, json_mem[instance])
#     instances.append(int(instance))
#     mem_usages.append(json_mem[instance])

# print(instances)
# print(mem_usages)

# total_instance = len(instances)
# total_mem = sum(mem_usages)
# mem_per_instance = total_mem/total_instance

# print(total_instance)
# print(total_mem)
# print(mem_per_instance)

# instance_stat = subprocess.check_output("cf curl v2/apps/3bbffff8-6834-4383-8140-cd223d9927ee/stats | jq 'with_entries(.value = .value.stats)'", shell=True)
# instance_stat_json = json.loads(instance_stat)
# mem_quota = list()
# instances = list()
# mem_usages = list()
# for instance in instance_stat_json:
#     #print("instance", instance)
#     #print("content", json_mem[instance])
#     #print("memory quota", instance_stat_json[instance]['mem_quota'])
#     #print("memory usage of the instance", instance_stat_json[instance]['usage']['mem'])
#     instances.append(int(instance))
#     mem_usages.append(instance_stat_json[instance]['usage']['mem'])
#     mem_quota.append(instance_stat_json[instance]['mem_quota'])

# print("instances list", instances)
# print("memory quota list", mem_quota)
# print("memory usage of the instances", mem_usages)

# total_instance = len(instances)
# total_mem = sum(mem_usages)
# mem_per_instance = total_mem/total_instance

# print("toatal number of instances of the app", total_instance)
# print("total memory consumption by all the app instances", total_mem)
# print("average memory consumption per app instance", mem_per_instance)


"""Common code to keep api handler DRY."""
import functools
import inspect
import re
from typing import Any, Callable, ParamSpec, TypeVar, cast

import schematics.exceptions

import db.conn
import journaling
from controllers.auth import AuthUser
from controllers.models import APIBaseError, HttpCode
from journaling import log
from pretty_ns import time_ns

Func = TypeVar("Func", bound=Callable[..., Any])

Param = ParamSpec("Param")
Result = TypeVar("Result")

def transaction(handler: Callable[Param, Result]) -> Callable[Param, Result]:
    """Decorate api handler into try-except to handle DB transaction."""

    @functools.wraps(handler)
    def transaction_wrapper(*args: Param.args, **kwargs: Param.kwargs) -> Result:
        try:
            return handler(*args, **kwargs)
        except APIBaseError as e:
            db.conn.session.rollback()
            log.error(f"{e}")
            return f"API error {e}", e.status  # type: ignore
        except schematics.exceptions.BaseError as e:
            db.conn.session.rollback()
            messages = []
            for field in e.errors:
                error_messsage = str(e.errors[field]).replace("Rogue", "Unknown")
                messages.append(f"{field} - {error_messsage}")
            log.error(f"Model validation error: {e}")
            return f'Wrong request parameters: {", ".join(messages)}', HttpCode.wrong_request  # type: ignore
        except TypeError as e:
            db.conn.session.rollback()
            missing_args = re.match(
                r"(.+)missing \d+ required positional argument(s)?: (.+)", str(e)
            )
            if missing_args:
                return f"Missing arguments: {missing_args[3]}", HttpCode.wrong_request  # type: ignore
            log.error(f"{e}", exc_info=True)
            return "Server internal error", HttpCode.unhandled_exception  # type: ignore
        except Exception as e:
            db.conn.session.rollback()
            log.error(f"{e}", exc_info=True)
            return "Server internal error", HttpCode.unhandled_exception  # type: ignore
        finally:
            db.conn.session.close()

    return transaction_wrapper

def api_result(handler: Func) -> Func:
    """Decorate api handler result.

    Expects from handler tuple with result object and optional code (default 200).
    Formats result as tuple (<result object>, <http result code>)
    """

    @functools.wraps(handler)
    def api_result_wrapper(*args, **kwargs):  # type: ignore
        journaling.request_start_time = time_ns()
        try:
            result = handler(*args, **kwargs)
            if isinstance(result, tuple):
                code = result[1]
                result = result[0]
            else:
                code = HttpCode.success
            return result, code
        finally:
            journaling.request_start_time = None

    # Removing our calculated argument auth_user from decorated function
    signature = inspect.signature(handler)
    params = []
    for param in signature.parameters.values():
        if param.name != "auth_user":
            params.append(param)
    # suppress "Callable[[VarArg(Any), KwArg(Any)], Any]" has no attribute "__signature__"
    api_result_wrapper.__signature__ = signature.replace(parameters=params)  # type: ignore[attr-defined]
    return cast(Func, api_result_wrapper)

def token_to_auth_user(handler: Callable[Param, Result]) -> Callable[Param, Result]:
    """Create auth_user parameter from token."""

    @functools.wraps(handler)  # preserve initial function signature
    def token_to_auth_user_wrapper(*args, **kwargs) -> Result:  # type: ignore
        if "auth_token" in kwargs:
            if kwargs["auth_token"] is not None:
                log.debug(f'Add auth_user to {handler.__name__}\n{kwargs["auth_token"]}')
                auth_user = AuthUser(kwargs["auth_token"])
                args = (auth_user,) + args
                journaling.user = auth_user.email
            else:
                log.warning("No or wrong user token in request")
                return "No or wrong user token in request", HttpCode.no_token  # type: ignore
            del kwargs["auth_token"]
        try:
            return handler(*args, **kwargs)
        finally:
            journaling.user = None

    # suppress "Callable[[VarArg(Any), KwArg(Any)], Any]" has no attribute "__signature__"
    token_to_auth_user_wrapper.__signature__ = inspect.signature(handler)  # type: ignore[attr-defined]
    return token_to_auth_user_wrapper


import logging
import pdb
import random
import sys

from graphwalker import codeloader
from graphwalker import halting

# some ghetto enums
COST, PATH = 0, 1

# approximation of infinity(tm)
inf = 2 ** 31
log = logging.getLogger(__name__)

class Planner(object):
    randcls = random.Random

    def __init__(self, *al, **kw):
        self.al, self.kw = al, kw
        self.rng = self.randcls(self.kw.get('seed'))

    def _setup(self, g, stop, start, context):
        for ident, vert in g.V.items():
            if vert.name == start:
                self.vert = vert
                break
        else:
            if start in g.V:
                self.vert = g.V[start]
            else:
                raise RuntimeError("Could not find start vertex")

        stop.add(vert)

        self.g, self.plan, self.stop = g, [], stop

        return self.g.V, self.g.E, self.plan, self.vert

    def forced_plan(self, plan=None):
        """Enter forced steps from Start source vertex.

        The start node is normally a source, but sometimes the source is a
        larger string of single-edge vertices. This enters them into the plan.
        """
        plan = plan if plan is not None else self.plan
        I, O = self.g.vert_degrees()

        while len(self.vert.outgoing) == 1 and I[self.vert.id] == 0:
            self.g.del_vert(self.vert)
            self.vert = self.step(self.vert, self.vert.outgoing[0], plan)
            I, O = self.g.vert_degrees()

    def visit(self, it, plan=None):
        plan = self.plan if plan is None else plan
        self.stop.add(it)
        plan.append(it)

    def step(self, vert, edge, plan=None):
        plan = self.plan if plan is None else plan

        assert edge.src == vert.id, 'Edge not from this vertex'

        dest = self.g.V[edge.tgt]

        self.visit(edge, plan)
        self.visit(dest, plan)
        return dest

class EvenRandom(Planner):
    def __call__(self, g, stop, start, context):
        """Walk through the graph by random edges until done."""
        self._setup(g, stop, start, context)
        return iter(self)

    def choose_edge(self, edges):
            return self.rng.choice(edges)

    def __iter__(self):
        while not self.stop:
            edge = self.choose_edge(self.vert.outgoing)
            self.stop.add(edge)
            yield edge
            self.vert = self.g.V[edge.tgt]
            self.stop.add(self.vert)
            yield self.vert

class Random(EvenRandom):
    """Walk through the graph by random edges until done."""

    def choose_edge(self, edges):
        naive, weighted = [], []
        for e in edges:
            if e.weight is None:
                naive.append(e)
            elif e.weight[-1:] == '%':
                weighted.append((e, float(e.weight[:-1]) / 100))
            else:
                weighted.append((e, float(e.weight)))

        if not weighted:
            return self.rng.choice(edges)

        total_given_probability = sum(w for e, w in weighted)
        remaining = 1.0 - total_given_probability

        if total_given_probability > 1.001:
            log.warn("Probalities supplied exceed unity")

        if len(naive) > 0:
            if remaining <= 0:
                log.warn("Unweighted edges get zero probability")
            else:
                weighted.extend((e, remaining / len(naive)) for e in naive)
        else:
            if remaining >= 0.01:
                log.warn("Weighted edges sum to less than unity")

        x, X = 0, self.rng.uniform(0.0, sum(w for e, w in weighted))
        for e, w in weighted:
            x += w
            if x >= X:
                return e

        return e

class Euler(Planner):
    """Walk through the graph by ordered edges until done."""

    def __call__(self, g, stop, start, context):
        self._setup(g, stop, start, context)

        self.g = self.g.copy()
        self.forced_plan()
        self.g.eulerize()

        self.stop = halting.Never().start(g, None)
        vert = self.vert
        seen = set()

        def loop(vert):
            subplan = []
            begin = vert
            while len(seen) < len(self.g.E):
                for edge in vert.outgoing:
                    if edge.id not in seen:
                        seen.add(edge.id)
                        vert = self.step(vert, edge, subplan)
                        break
                else:
                    assert vert.id == begin.id, "Graph is not Eulerian"
                    break

            return subplan

        plan = loop(vert)  # initial plan

        while len(seen) < len(self.g.E):
            j = 0
            for i in range(len(plan)):
                if self.g.V.get(plan[i].id) is not plan[i]:
                    continue

                vert = plan[i]
                for edge in vert.outgoing:
                    if edge.id not in seen:
                        # splice new loop into plan
                        plan[i + 1: i + 1] = loop(vert)
                        j += 1
                        break
            else:
                assert j, "Graph is not connected"

        self.plan, plan = [], self.plan + plan
        for step in plan:
            if stop:
                break
            self.visit(step)

        return self.plan

class Goto(Planner):
    """Plan direct path to goal state(s), repeating [repeat] times."""

    def __init__(self, *al, **kw):
        self.al, self.kw = al, kw
        self.goals = self.al
        self.repeat = kw.pop('repeat', 1)

    def __call__(self, g, stop, start, context):
        self._setup(g, stop, start, context)
        self.d = d = self.g.all_pairs_shortest_path()

        for i in xrange(self.repeat or inf):
            for goal in self.goals:
                if self.g.is_stuck(self.vert) or self.stop:
                    break
                if goal == 'random':
                    goal = self.rng.choice(self.g.V.keys())

                try:
                    cost, path = min(d[(self.vert.id, v.id)]
                                     for v in self.g.V.values()
                                     if ((v.name == goal or
                                          v.id == goal) and
                                         v is not self.vert))
                    plan = path
                except ValueError:
                    continue

                for item in plan:
                    edge = [e for e in self.vert.outgoing
                            if e.tgt == item][0]
                    self.vert = self.step(self.vert, edge)

        return self.plan

class Interactive(Planner):
    """Planner that yields steps (or not) from user interaction.

    The protocol between choose and iterplan is deliberately kept simple to
    keep it simple to replace the choose method.
    """

    raw_input = raw_input
    out = sys.stderr
    debugger = pdb.Pdb('\t', sys.stdin, sys.stderr)

    help = """\
0-n:    Traverse edge
h(elp)  This message
d(ebug) Enter Pdb
g(oto)  Use Goto planner to go some vertex
f(orce) Forcibly insert some words into the plan
j(ump)  Forcibly set the vert where the planner believes it is at
q(uit)  End the interactive session
"""

    def choose(self, planner, alts):
        while True:
            for i in range(len(alts)):
                print >>self.out, i, alts[i]
            try:
                self.out.write('> ')
                return self.raw_input()
            except KeyboardInterrupt:
                return None
            except EOFError:
                return None
            except Exception as e:
                print >>self.out, 'huh? %r' % e

    choose_fn = choose

    def __init__(self, *al, **kw):
        self.al, self.kw = al, kw

    def goto(self, goals):
        stop = halting.Never().start(self.g, self.context)
        return Goto(*goals)(self.g, stop, self.vert.id, self.context)

    def choose_vert(self, name):
        if name in self.g.V:
            return self.g.V[name]

        candidates = [v for v in self.g.V.values() if v.name == name]
        if len(candidates) == 1:
            return candidates[0]
        else:
            i = self.choose_fn(self, self.format_vert_list(candidates))
            return candidates[int(i)]

    def __call__(self, g, stop, start, context):
        self._setup(g, stop, start, context)
        self.context = context
        return iter(self)

    def format_vert_list(self, verts):
        V = self.g.V
        alts = []
        for v in verts:
            outs = set(['[%s]\t--(%s)-->%s' % (e.id, e.name, V[e.tgt].name)
                        for e in v.outgoing])
            alts.append(v.name + '\n  ' + '\n  '.join(outs))
        return alts

    def format_edge_list(self, edges):
        V = self.g.V
        return ["[%s]\t%s--(%s)-->%s" %
                (e.id, V[e.src].name, e.name, V[e.tgt].name)
                for e in edges]

    def __iter__(self):
        while True:
            edges = self.vert.outgoing

            if not edges:
                raise StopIteration()

            where = "== Currently at: %s [%s]" % (self.vert.name, self.vert.id)
            print >>self.out, where
            if self.stop:
                print >>self.out, "According to end conditions, we're done"

            i = self.choose_fn(self, self.format_edge_list(edges))
            self.out.flush()

            if i in ('q', None):  # quit
                raise StopIteration()

            elif i == '':
                print >>self.out, 'huh?'
                continue

            elif i[0] == 'd':  # debugger
                self.debugger.set_trace()

            elif i[0] == 'f':  # force
                for s in i.split()[1:]:
                    yield (s, s, ())

            elif i[0] == 'g':  # goto
                for name in i.split()[1:]:
                    for step in self.goto([self.choose_vert(name).id]):
                        yield step
                        self.vert = step

            elif i[0] == 'j':  # jump
                while True:
                    try:
                        self.vert = self.choose_vert(i.split()[-1])
                        break
                    except Exception as e:
                        print >>self.out, 'huh? %r' % e

            elif i[0] in 'h?':
                print >>self.out, self.help

            elif i.strip().isdigit():
                index = int(i.strip())
                if index >= len(edges):
                    print >>self.out, 'huh?'
                else:
                    edge = edges[index]

                    self.vert = self.step(self.vert, edge)
                    yield edge
                    yield self.vert

class MasterPlan(Planner):
    def __init__(self, plans):
        self.plans = plans
        self.i = 0

    def __call__(self, g, stop, start, context):
        self.step = (None, start)
        while self.i < len(self.plans):
            planner = self.plans[self.i]
            start = self.step[0] or self.step[1]
            for step in planner(g, stop, start, context):
                self.step = step
                yield step
            self.i += 1

def build(specs):
    """Import, construct and aggregate requested reporters."""
    planners = []

    for spec in specs:
        planners.append(
            codeloader.construct(
                spec,
                default_module=__name__,
                call_by_default=True))

    if len(planners) == 1:
        return planners[0]
    else:
        return MasterPlan(planners)

planners = [cls
            for cls in locals().values()
            if type(cls) is type
            and issubclass(cls, Planner)
            and cls.__doc__
            and cls is not Planner]


import os
import numpy as np
import math
import json
import imageio
import torch
from data_util.face3d_helper import Face3DHelper

from utils.commons.euler2rot import euler_trans_2_c2w, c2w_to_euler_trans
from tasks.audio2motion.dataset_utils.euler2quaterion import euler2quaterion, quaterion2euler
import tqdm

from utils.commons.hparams import hparams, set_hparams
set_hparams()

face3d_helper = Face3DHelper()

audio_cond_win_size = 16 # hparams['cond_win_size'] for ad_nerf/radnerf
audio_smo_win_size = 8 # hparams['smo_win_size'] for ad_nerf/radnerf
exp_cond_win_size = 1 # hparams['cond_win_size'] for lm3d_nerf/lm3d_radnerf
exp_smo_win_size = 5 # hparams['smo_win_size'] for lm3d_nerf/lm3d_radnerf

def get_win_conds(conds, idx, smo_win_size=8, pad_option='zero'):
    """
    conds: [b, t=16, h=29]
    idx: long, time index of the selected frame
    """
    idx = max(0, idx)
    idx = min(idx, conds.shape[0]-1)
    smo_half_win_size = smo_win_size//2
    left_i = idx - smo_half_win_size
    right_i = idx + (smo_win_size - smo_half_win_size)
    pad_left, pad_right = 0, 0
    if left_i < 0:
        pad_left = -left_i
        left_i = 0
    if right_i > conds.shape[0]:
        pad_right = right_i - conds.shape[0]
        right_i = conds.shape[0]
    conds_win = conds[left_i:right_i]
    if pad_left > 0:
        if pad_option == 'zero':
            conds_win = np.concatenate([np.zeros_like(conds_win)[:pad_left], conds_win], axis=0)
        elif pad_option == 'edge':
            edge_value = conds[0][np.newaxis, ...]
            conds_win = np.concatenate([edge_value] * pad_left + [conds_win], axis=0)
        else:
            raise NotImplementedError
    if pad_right > 0:
        if pad_option == 'zero':
            conds_win = np.concatenate([conds_win, np.zeros_like(conds_win)[:pad_right]], axis=0)
        elif pad_option == 'edge':
            edge_value = conds[-1][np.newaxis, ...]
            conds_win = np.concatenate([conds_win] + [edge_value] * pad_right , axis=0)
        else:
            raise NotImplementedError
    assert conds_win.shape[0] == smo_win_size
    return conds_win

def load_processed_data(processed_dir):
    # images required by AD-NeRF
    head_img_dir = os.path.join(processed_dir, "head_imgs")
    ori_img_dir = os.path.join(processed_dir, "ori_imgs")
    parsing_dir = os.path.join(processed_dir, "parsing")
    # images required by RAD-NeRF
    torso_img_dir = os.path.join(processed_dir, "torso_imgs")
    gt_img_dir = os.path.join(processed_dir, "gt_imgs")

    background_img_name = os.path.join(processed_dir, "bc.jpg")
    train_json_name = os.path.join(processed_dir, "transforms_train.json")
    val_json_name = os.path.join(processed_dir, "transforms_val.json")
    track_params_name = os.path.join(processed_dir, "track_params.pt")
    deepspeech_npy_name = os.path.join(processed_dir, "aud_deepspeech.npy")
    esperanto_npy_name = os.path.join(processed_dir, "aud_esperanto.npy")
    coeff_npy_name = os.path.join(processed_dir, "vid_coeff.npy")
    hubert_npy_name = os.path.join(processed_dir, "aud_hubert.npy")
    mel_f0_npy_name = os.path.join(processed_dir, "aud_mel_f0.npy")

    # required by RAD-NeRF

    ret_dict = {}

    print("loading deepspeech ...")
    deepspeech_features = np.load(deepspeech_npy_name)
    print("loading Esperanto ...")
    esperanto_features = np.load(esperanto_npy_name)
    print("loading hubert ...")
    hubert_features = np.load(hubert_npy_name)
    ret_dict['hubert'] = hubert_features
    print("loading Mel and F0 ...")
    mel_f0_features = np.load(mel_f0_npy_name, allow_pickle=True).tolist()
    ret_dict['mel'] = mel_f0_features['mel']
    ret_dict['f0'] = mel_f0_features['f0']

    print("loading 3dmm coeff ...")
    coeff_dict = np.load(coeff_npy_name, allow_pickle=True).tolist()
    coeff_arr = coeff_dict['coeff'][:]

    identity_arr = coeff_arr[:, 0:80]
    exp_arr = coeff_arr[:, 80:144]

    print("calculating lm3d ...")
    idexp_lm3d_arr = face3d_helper.reconstruct_idexp_lm3d(torch.from_numpy(identity_arr), torch.from_numpy(exp_arr)).cpu().numpy()

    video_idexp_lm3d_mean = idexp_lm3d_arr.mean(axis=0).reshape([1,68,3])
    video_idexp_lm3d_std = idexp_lm3d_arr.std(axis=0).reshape([1,68,3])
    ret_dict['idexp_lm3d_mean'] = video_idexp_lm3d_mean
    ret_dict['idexp_lm3d_std'] = video_idexp_lm3d_std
    idexp_lm3d_arr_normalized = (idexp_lm3d_arr - video_idexp_lm3d_mean) / video_idexp_lm3d_std

    if deepspeech_features.shape[0] < coeff_arr.shape[0]:
        num_to_pad = coeff_arr.shape[0] - deepspeech_features.shape[0]
        tmp = np.zeros([num_to_pad, 16, 29])
        deepspeech_features = np.concatenate([deepspeech_features, tmp], axis=0)
    elif deepspeech_features.shape[0] > coeff_arr.shape[0]:
        deepspeech_features = deepspeech_features[:coeff_arr.shape[0]]

    if esperanto_features.shape[0] < coeff_arr.shape[0]:
        num_to_pad = coeff_arr.shape[0] - esperanto_features.shape[0]
        tmp = np.zeros([num_to_pad, 16, 44])
        esperanto_features = np.concatenate([esperanto_features, tmp], axis=0)
    elif esperanto_features.shape[0] > coeff_arr.shape[0]:
        esperanto_features = esperanto_features[:coeff_arr.shape[0]]

    translation = coeff_arr[:, 254:257] # [T_y, c=3]
    angles = euler2quaterion(coeff_arr[:, 224:227]) # # [T_y, c=4]
    pose_deep3drecon = np.concatenate([translation, angles], axis=1)

    print("loading train_val.json ...")
    with open(train_json_name) as f:
        train_meta = json.load(f)
    with open(val_json_name) as f:
        val_meta = json.load(f)
    bg_img = imageio.imread(background_img_name)
    ret_dict['bg_img'] = bg_img
    ret_dict['H'], ret_dict['W'] = bg_img.shape[:2]
    ret_dict['focal'], ret_dict['cx'], ret_dict['cy'] = float(train_meta['focal_len']), float(train_meta['cx']), float(train_meta['cy'])

    idexp_lm3d_normalized_win_lst = []
    # hubert_win_lst = []
    for frame in train_meta['frames'] + val_meta['frames'] :
        idx = frame['aud_id']
        idexp_lm3d_normalized_win = get_win_conds(idexp_lm3d_arr_normalized, idx, smo_win_size=exp_cond_win_size, pad_option='zero')
        idexp_lm3d_normalized_win_lst.append(idexp_lm3d_normalized_win)
        # hubert_win = get_win_conds(hubert_features, idx, smo_win_size=16)
        # hubert_win_lst.append(hubert_win)
    idexp_lm3d_normalized_wins_arr = np.stack(idexp_lm3d_normalized_win_lst, axis=0) # [T, t_w, 204]
    # hubert_win_arr = np.stack(hubert_win_lst, axis=0) # [T, t_w, 204]

    # obtaining train samples
    train_samples = []
    for i_frame, frame in tqdm.tqdm(enumerate(train_meta['frames']), desc="Binarizing train set", total=len(train_meta['frames'])):
        assert frame['aud_id'] == frame['img_id']
        idx = frame['aud_id']
        ori_img_fname = os.path.join(ori_img_dir,f"{idx}.jpg")
        head_img_fname = os.path.join(head_img_dir,f"{idx}.jpg")
        torso_img_fname = os.path.join(torso_img_dir,f"{idx}.png")
        gt_img_fname = os.path.join(gt_img_dir,f"{idx}.jpg")
        parsing_fname = os.path.join(parsing_dir,f"{idx}.png")

        camera2world_matrix = np.array(frame['transform_matrix'])
        euler, trans = c2w_to_euler_trans(camera2world_matrix)
        face_rect = np.array(frame['face_rect'])
        deepspeech_wins = get_win_conds(deepspeech_features, idx, smo_win_size=audio_smo_win_size, pad_option='zero')
        esperanto_wins = get_win_conds(esperanto_features, idx, smo_win_size=audio_smo_win_size, pad_option='zero')

        idexp_lm3d_normalized_win = get_win_conds(idexp_lm3d_arr_normalized, idx, smo_win_size=exp_cond_win_size, pad_option='zero') # [cond_win_size, 68, 3]
        idexp_lm3d_normalized_wins = get_win_conds(idexp_lm3d_normalized_wins_arr, idx, smo_win_size=exp_smo_win_size, pad_option='zero') # [smo_win_size, cond_win_size, 68, 3]

        # hubert_win = hubert_win_arr[idx]
        # hubert_wins = get_win_conds(hubert_win_arr, idx, smo_win_size=8, pad_option='zero')

        sample = {
            'idx': idx,
            'face_rect': face_rect,
            'ori_img_fname': ori_img_fname,
            'head_img_fname': head_img_fname,
            'torso_img_fname': torso_img_fname,
            'gt_img_fname': gt_img_fname,
            'parsing_fname': parsing_fname,
            'c2w': camera2world_matrix,
            'euler': euler,
            'trans': trans,
            'exp': exp_arr[idx],
            'identity': identity_arr[idx],
            'pose_deep3drecon': pose_deep3drecon[idx],
            'idexp_lm3d': idexp_lm3d_arr[idx],
            'idexp_lm3d_normalized': idexp_lm3d_arr_normalized[idx],
            'idexp_lm3d_normalized_win': idexp_lm3d_normalized_win,
            'idexp_lm3d_normalized_wins': idexp_lm3d_normalized_wins,
            'deepspeech_win': deepspeech_features[idx],
            'deepspeech_wins': deepspeech_wins,
            'esperanto_win': esperanto_features[idx],
            'esperanto_wins': esperanto_wins,
            # 'hubert_win': hubert_win,
            # 'hubert_wins': hubert_wins,
        }
        train_samples.append(sample)
    ret_dict['train_samples'] = train_samples

    # obtaining val samples
    val_samples = []
    for i_frame, frame in tqdm.tqdm(enumerate(val_meta['frames']), desc="Binarizing val set", total=len(val_meta['frames'])):
        assert frame['aud_id'] == frame['img_id']
        idx = frame['aud_id']
        ori_img_fname = os.path.join(ori_img_dir,f"{idx}.jpg")
        head_img_fname = os.path.join(head_img_dir,f"{idx}.jpg")
        torso_img_fname = os.path.join(torso_img_dir,f"{idx}.png")
        gt_img_fname = os.path.join(gt_img_dir,f"{idx}.jpg")
        parsing_fname = os.path.join(parsing_dir,f"{idx}.png")

        face_rect = np.array(frame['face_rect'])
        camera2world_matrix = np.array(frame['transform_matrix'])
        euler, trans = c2w_to_euler_trans(camera2world_matrix)
        deepspeech_wins = get_win_conds(deepspeech_features, idx, smo_win_size=audio_smo_win_size, pad_option='zero')
        esperanto_wins = get_win_conds(esperanto_features, idx, smo_win_size=audio_smo_win_size, pad_option='zero')

        idexp_lm3d_normalized_win = get_win_conds(idexp_lm3d_arr_normalized, idx, smo_win_size=exp_cond_win_size, pad_option='zero')
        idexp_lm3d_normalized_wins = get_win_conds(idexp_lm3d_normalized_wins_arr, idx, smo_win_size=exp_smo_win_size, pad_option='zero')

        # hubert_win = hubert_win_arr[idx]
        # hubert_wins = get_win_conds(hubert_win_arr, idx, smo_win_size=8, pad_option='zero')

        sample = {
            'idx': idx,
            'face_rect': face_rect,
            'ori_img_fname': ori_img_fname,
            'head_img_fname': head_img_fname,
            'torso_img_fname': torso_img_fname,
            'gt_img_fname': gt_img_fname,
            'parsing_fname': parsing_fname,
            'c2w': camera2world_matrix,
            'euler': euler,
            'trans': trans,
            'exp':  exp_arr[idx], # [64]
            'identity': identity_arr[idx],
            'pose_deep3drecon': pose_deep3drecon[idx],
            'idexp_lm3d': idexp_lm3d_arr[idx],
            'idexp_lm3d_normalized': idexp_lm3d_arr_normalized[idx],
            'idexp_lm3d_normalized_win': idexp_lm3d_normalized_win,
            'idexp_lm3d_normalized_wins': idexp_lm3d_normalized_wins,
            'deepspeech_win': deepspeech_features[idx],
            'deepspeech_wins': deepspeech_wins,
            'esperanto_win': esperanto_features[idx],
            'esperanto_wins': esperanto_wins,
            # 'hubert_win': hubert_win,
            # 'hubert_wins': hubert_wins,
        }

        val_samples.append(sample)
    ret_dict['val_samples'] = val_samples

    return ret_dict

class Binarizer:
    def __init__(self):
        self.data_dir = 'data/'

    def parse(self, video_id):
        processed_dir = os.path.join(self.data_dir, 'processed/videos', video_id)
        binary_dir = os.path.join(self.data_dir, 'binary/videos', video_id)
        out_fname = os.path.join(binary_dir, "trainval_dataset.npy")
        os.makedirs(binary_dir, exist_ok=True)
        ret = load_processed_data(processed_dir)
        mel_name = os.path.join(processed_dir, 'aud_mel_f0.npy')
        mel_f0_dict = np.load(mel_name, allow_pickle=True).tolist()
        ret.update(mel_f0_dict)
        np.save(out_fname, ret, allow_pickle=True)

if __name__ == '__main__':
    binarizer = Binarizer()
    binarizer.parse(hparams['video_id'])
    print(f"Binarization for {hparams['video_id']} Done!")


#!/usr/bin/python3
''' main cli for spotiplay rooms '''
from __future__ import absolute_import, unicode_literals
import socket
import json
import sys
import time
from multiprocessing import Process
from Crypto.Cipher import AES

from spotiplay.strategy import get_pytify_class_by_platform
from spotiplay.song_list import SongList
from spotiplay.prompt import custom_prompt
from spotiplay.commander import Commander

from spotiplay.room import Room

PORT = 8080
HOST = str(AES.new('1n1dklmnAMADKENM', AES.MODE_ECB).decrypt(b'\xd8\x85\x11\xa85P$\x91\xee\x87\x05>\x9e\x89\xba\xb0\xa5\x14\xfa\xbdu\xe5F\xf6\xa7\xa2\x1d\x92\x1e\x91}\x1f\x96e\x91\x8b\x14\xf6O,&\x16\xd1\xdb\x91\xc4\x98"\xd3\xd2\x1b\x19(\x9f\xa4G[\x18\x8d\\\x06\x81!\x83').strip())[2:-1]

class App:
    ''' cli app '''
    def __init__(self):
        self.pytify = get_pytify_class_by_platform()()
        self.room = None
        self.command = Commander(self.pytify, self.room)

        self.parent = Process(target=self.watch_songs)
        self.parent.start() #run in background
        try:
            self.interaction()
        except EOFError:
            self.parent.terminate()
        except KeyboardInterrupt:
            self.parent.terminate()

    def list_songs(self, song_list):
        ''' display the search results '''
        SongList(song_list, self.room)

    def host_room(self):
        ''' host a room '''
        print('Sending host request...')
        room_addr = create_room(HOST, PORT)
        print('Joined room: %s' % room_addr)
        self.room = Room(self.pytify, room_addr) # no queue
        self.command = Commander(self.pytify, self.room)

    def leave_room(self):
        ''' leave a room '''
        if self.room:
            print('Sending leave request...')
            ret = leave_room(HOST, PORT, self.room.get_addr())
            if ret[0] != '@':
                self.room = None
                print(ret)
                self.parent.terminate()
                sys.exit()
        else:
            print("You're not in a room!")

    def join_room(self, addr):
        print('Sending join request...')
        room_data = join_room(HOST, PORT, addr)
        if room_data != {}:
            self.room = Room(self.pytify, addr, room_data) # could be a queue
            self.command = Commander(self.pytify, self.room)
        else:
            print('Problem joining room.')

    def interaction(self):
        print('spotiplay [pytify] 0.2.1')

        while 1:
            search_input = custom_prompt()

            if search_input == '/create_room':
                if not self.room:
                    try:
                        self.host_room()
                    except ConnectionRefusedError:
                        print('Spotiplay server is down.')
                continue

            elif search_input == '/leave':
                self.leave_room()
                continue

            elif search_input.split(' ')[0] == '/join':
                self.join_room(search_input.split(' ')[1])
                continue

            if self.command.run(search_input):
                continue

            if self.room:
                search = self.pytify.query(search_input)
                if search:
                    self.list_songs(self.pytify.list())
            else:
                print('You need a room to add songs to!')

    def watch_songs(self):
        ''' watch for unauthorized + authorized song changes'''
        print('Monitoring Spotify...')

        while 1:
            time.sleep(2) #check every two seconds
            self.pytify.get_current_playing()

def create_room(host, port):
    ''' actually send the create room request to the server '''
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((host, port))

    sock.send(('/CREATE_ROOM').encode('utf-8'))
    room_key = sock.recv(1024).decode('utf-8')
    sock.close()

    return room_key

def leave_room(host, port, addr):
    ''' actually send the leave room request to the server '''
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((host, port))

    sock.send(('/LEAVE_ROOM').encode('utf-8'))
    sock.send(addr.encode('utf-8'))

    ret_val = sock.recv(1024).decode('utf-8')
    sock.close()

    return ret_val

def join_room(host, port, room_addr):
    ''' actually send the join room request to the server '''
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((host, port))

    sock.send(('/JOIN_ROOM').encode('utf-8'))
    sock.send((str(room_addr)))
    ret_val = sock.recv(1024).decode('utf-8')
    room_data = {}
    if ret_val[0] != '@':
        room_data = json.loads(ret_val)
    else:
        print(ret_val)

    sock.close()
    return room_data

def main():
    ''' main '''
    try:
        App()
    except EOFError:
        print('\n Closing application...\n')
    except KeyboardInterrupt:
        print('\n Closing application...\n')


# -*- coding: utf-8 -*-

"""
    author : youfaNi
    date : 2016-07-18
"""

import pdb
import tornado
from dhuicredit.app import get_options
from dhuicredit.urls import handlers
import dhuicredit.model.model as model
import dhuicredit.model.mongo as mongo
import dhuicredit.consts as consts
import dhuicredit.libs.utils as utils
from bson.son import SON

class AuthorityModel(model.BaseModel,model.Singleton):
    __name = "dhuicredit.authority"
    help = "用户权限管理"

    def __init__(self):
        model.BaseModel.__init__(self,AuthorityModel.__name)
        self.scope_manifest = [
            ("default",("font-api",["font-api"])),
            ("all",("font-api",["font-api","back-api","system-api"])),
        ]
        self.scope_api_dict = {
            "font-api":[
                "/api/user",
            ]
        }

    def parse(self):
        pass

    def process(self,request,access):
        request_path = request.path
        scopes = access["scopes"]
        available_api_list = self.get_scopes_api_list(scopes)
        if request_path in available_api_list or "font-api" in scopes:
            pass
        else:
            raise Exception("访问受限")

    def get_scopes(self):
        scope_manifest = self.scope_manifest
        scopes = {}
        for scope in self.scope_manifest:
            scopes[scope[0]] = scope[1]
        return scopes

    def get_api_list(self):
        global handlers
        api_list = []
        for handler in handlers:
            if isinstance(handler,tornado.web.URLSpec):
                api_list.append(handler._path)
            else:
                api_list.append(handler[0])
        return api_list

    def get_scopes_api_list(self,scopes):
        api_list = []
        for scope in scopes:
            if self.scope_api_dict.has_key(scope):
                api_list.extend(self.scope_api_dict[scope])
        return list(set(api_list))

authority = AuthorityModel()


import argparse

my_parser = argparse.ArgumentParser()

my_parser.add_argument(
    '-v',
    '--verbosity',
    action='store',
    type=int,
    metavar='LEVEL'
)

args = my_parser.parse_args()

print(vars(args))

"""
$ python metavar_example.py -h
usage: metavar_example.py [-h] [-v LEVEL]

optional arguments:
  -h, --help            show this help message and exit
  -v LEVEL, --verbosity LEVEL
"""


"""Home URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from . import views

urlpatterns = [
    path('', views.home_page, name='home'),
    path('About', views.About_Page, name='About'),
    path('Contact Us', views.Contact_Page, name='Contact Us'),

    # Admin------------
    path('Admin_Login', views.Admin_Login, name='Admin_Login'),
    path('Admin_', views.Admin_, name='Admin_'),
    path('Admin_Logout', views.Admin_Logout, name='Admin_Logout'),

    # Student--------
    path('Student_Signup', views.Save_signup, name='Student_Signup'),
    path('Student_Login', views.Student_Login, name='Student_Login'),
    path('Student_', views.Student_, name='Student_'),
    path('Student_Logout', views.Student_Logout, name='Student_Logout'),

    # Teacher-------
    path('Teacher_Signup', views.Teacher_signup, name='Teacher_Signup'),
    path('Teacher_Login', views.Teacher_Login, name='Teacher_Login'),
    path('Teacher_', views.Teacher_, name='Teacher_'),
    path('Teacher_Logout', views.Teacher_Logout, name='Teacher_Logout'),

]


from django.shortcuts import render, redirect
from django.views.decorators.http import require_safe, require_POST
from .models import Movie, Genre
from django.http import JsonResponse
from django.shortcuts import get_object_or_404

# Create your views here.
@require_safe
def index(request):
    movies = Movie.objects.all()
    context = {
        'movies':movies,
    }
    return render(request, 'movies/index.html', context)

@require_POST
def movie_like(request, movie_pk):
    if request.user.is_authenticated:
        movie = Movie.objects.get(pk=movie_pk)
        user = request.user

        if movie.like_users.filter(pk=user.pk).exists():
            movie.like_users.remove(user)
            is_liked = False
        else:
            movie.like_users.add(user)
            is_liked = True
        context = {
            'is_liked': is_liked,
        }
        return JsonResponse(context)
    return redirect('acctouns:login')

@require_safe
def detail(request, movie_pk):
    movie = Movie.objects.get(pk=movie_pk)
    context = {
        'movie': movie,
    }
    return render(request, 'movies/detail.html', context)

@require_safe
def recommended(request):
    # 상위 3개 장르 추출
    G = Genre.objects.all()
    G_count = {i.name:0 for i in G}
    if request.user.is_authenticated:
        user = request.user
        movies = user.like_movies.all()
        for movie in movies:
            genre = movie.genres.all()
            for i in genre:
                G_count[i.name] += 1
        new_G = sorted(G_count.items(), key=lambda x: x[1], reverse=True)[:3]
        preferences = [Genre.objects.get(name=i[0]).id for i in new_G]

        movies = Movie.objects.filter(genres__id__in=preferences).order_by('-vote_average')
        movies = list(set(movies))[:10]

    return render(request, 'movies/recommended.html', {'movies': movies})


import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np

class Model(nn.Module):
    def __init__(self, args):
        super(Model, self).__init__()
        self.input_layer = nn.Linear(1, 1000)
        self.activation = nn.GELU()
        self.output_layer = nn.Linear(1000, 1)

    def forward(
        self,
        x_enc,
        x_mark_enc,
        x_dec,
        x_mark_dec,
        enc_self_mask=None,
        dec_self_mask=None,
        dec_enc_mask=None,
    ):
        out = self.input_layer(x_enc)
        out = self.activation(out)
        out = self.output_layer(out)

        return out


from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from data.kicker_data import get_kicker_data

def log_regression():
    x, y = get_kicker_data()
    scaler = StandardScaler()
    x_scaled = scaler.fit_transform(x)

    lr = LogisticRegression(C=1)
    lr.fit(x_scaled, y)
    return lr


from pathlib import Path

import pandas as pd
import pysam
from tqdm import tqdm

def split_bam_file_by_cell_barcodes(bam_file_path: Path,
                                    cell_barcodes: set[str],
                                    output_folder: Path) -> None:
    """
    Split .bam file to multiple smaller ones, each containing reads from a single cell.
    :param bam_file_path: Path to a .bam file with reads.
    :param cell_barcodes: Set of cell barcodes. Reads with barcodes not present in cell_barcodes will be ignored.
    :param output_folder: Folder to which the resulting files will be written.
    :return: None.
    """
    output_folder.mkdir(parents=True, exist_ok=True)
    samfile_input = pysam.AlignmentFile(bam_file_path, "rb")
    output_file_names_by_barcode = {barcode: output_folder / f"{barcode}.bam" for barcode in cell_barcodes}
    output_samfiles_by_barcode = {barcode: pysam.AlignmentFile(file_path, "wb",
                                                               template=samfile_input)
                                  for barcode, file_path in output_file_names_by_barcode.items()}
    for read in tqdm(samfile_input, desc=f'Processing reads from the input .bam file {bam_file_path}'):
        if not (read.has_tag('CB') and read.has_tag('UB') and read.get_tag('UB') != '-'):
            continue

        read_barcode = read.get_tag('CB')
        if read_barcode in cell_barcodes:
            output_samfiles_by_barcode[read_barcode].write(read)

    for samfile_output in output_samfiles_by_barcode.values():
        samfile_output.close()
    for file_name in tqdm(output_file_names_by_barcode.values(), desc='Indexing .bam files'):
        pysam.index(str(file_name))

def split_star_solo_output_by_cell_barcodes(star_solo_output_path: Path,
                                            output_folder_path: Path) -> None:
    barcodes_df = pd.read_csv(
        star_solo_output_path / Path('Solo.out/GeneFull/filtered/barcodes.tsv'),
        delimiter='\t',
        names=['barcode'])
    split_bam_file_by_cell_barcodes(
        bam_file_path=star_solo_output_path / 'Aligned.sortedByCoord.out.bam',
        cell_barcodes=set(barcodes_df['barcode']),
        output_folder=output_folder_path / star_solo_output_path.name)

if __name__ == "__main__":
    for folder in Path(
            '/cellfile/datapublic/jkoubele/leibniz_institute_data/aligned/20201014_582_KLR/GEX').iterdir():
        split_star_solo_output_by_cell_barcodes(star_solo_output_path=folder,
                                                output_folder_path=Path(
                                                    '/cellfile/datapublic/jkoubele/leibniz_institute_data/splitted_by_cells/20201014_582_KLR/GEX/'))


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import os
import time

student_name = []
msg = ''

with open("msg.txt", "r") as j:
    msg = j.read()

with open("alunos.txt", "r") as f:
    student_name = f.readlines() # readlines() returns a list of items, each item is a line in your file

tam = len(student_name)
for i in range(tam):
    student_name[i] = student_name[i].rstrip()

print(msg)

class Wpp:
    def __init__(self, driver):
        self.driver = driver
        self.url = 'https://web.whatsapp.com/'
        self.match_text = 'matched-text' #class
        self.ctt_field = '/html/body/div[1]/div/div/div[3]/div/div[1]/div/label/input'
        self.input_msg = '/html/body/div[1]/div/div/div[4]/div/footer/div[1]/div[2]/div/div[2]'
    def navigate(self):
        self.driver.get(self.url)
        print('Insira o QR Code')
        print('')
        os.system('PAUSE')

    def search_ctt(self, std_nome):
        self.std_nome = std_nome
        self.driver.find_element(By.XPATH, self.ctt_field).clear()
        self.driver.find_element(By.XPATH, self.ctt_field).send_keys(self.std_nome)
        time.sleep(2)

    def check_ctt_name(self, std_nome):
        self.std_nome = std_nome
        self.match = self.driver.find_element_by_class_name(self.match_text)
        self.match_txt_upper = self.match.text

        if (self.std_nome.upper() == self.match_txt_upper.upper()):
            self.enter_key_ctt()
            print('Uma mensagem foi enviada para {}!'.format(self.std_nome))
        else:
            print('Nome procurado nao existe!')

    def write_message(self, msg):
        self.msg = msg
        self.driver.find_element(By.XPATH, self.input_msg).send_keys(self.msg)
        self.enter_key_message()

    def enter_key_ctt(self):
        self.driver.find_element(By.XPATH, self.ctt_field).send_keys(Keys.ENTER)

    def enter_key_message(self):
        self.driver.find_element(By.XPATH, self.input_msg).send_keys(Keys.ENTER)

ff = webdriver.Firefox()
wpp = Wpp(ff)
wpp.navigate()

for i in range(len(student_name)):

    try:
        wpp.search_ctt(student_name[i])
        wpp.check_ctt_name(student_name[i])
        wpp.write_message('{} {}'.format(student_name[i], msg))
    except:
        print('O contato {} não consta na sua lista'.format(student_name[i]))


import functools
import logging
import threading
import time
from collections import deque
from resource import getrusage, RUSAGE_SELF

from django.conf import settings
from django.db.backends.utils import CursorWrapper

from .collector import statsd

PROCESSING_TIME_METRIC_PREFIX = getattr(
    settings, 'PROCESSING_TIME_METRIC_PREFIX', 'processing_time'
)
REQUESTS_COUNTER_METRIC_PREFIX = getattr(
    settings, 'REQUESTS_COUNTER_METRIC_PREFIX', 'requests_count'
)

METRIC_NAME_TMPL = getattr(
    settings,
    'REQUESTS_METRIC_NAME_TMPL',
    '{prefix}.{url_name}.{request_method}.{status_code}'
)
ALL_URLS_METRIC_NAME_TMPL = getattr(
    settings,
    'REQUESTS_ALL_URLS_METRIC_NAME_TMPL',
    '{prefix}_all_urls.{request_method}'
)
UNKNOWN_URL_NAME = 'unknown'

REQUESTS_METRICS_ENABLED = getattr(
    settings,
    'ENABLE_REQUESTS_AND_QUERIES_METRICS',
    True
)
LARGE_NUMBER_OF_QUERIES_THRESHOLD = getattr(
    settings,
    'LARGE_NUMBER_OF_QUERIES_THRESHOLD',
    25
)
LONG_QUERIES_THRESHOLD_MS = getattr(
    settings,
    'LONG_QUERIES_THRESHOLD_MS',
    250
)

logger = logging.getLogger(__name__)

queries_data = threading.local()
old_execute = CursorWrapper.execute
old_executemany = CursorWrapper.executemany

class QueryLogEntry:
    sql: str
    duration: float

    def __init__(self, sql: str, duration: float):
        self.sql = sql,
        self.duration = duration

    def __str__(self):
        return "'{}' took {} ms".format(self.sql, self.duration * 1000)

def get_queries_log() -> deque:
    if not hasattr(queries_data, 'queries'):
        queries_data.queries = deque(maxlen=9000)
    return queries_data.queries

def add_query_log(duration: float, sql: str) -> None:
    get_queries_log().append(QueryLogEntry(sql=sql, duration=duration))

@functools.wraps(old_execute)
def new_execute(self, sql, params=None):  # type: ignore
    """
    Wraps original cursor execute method and adds thread safe timing to it.
    """
    start = time.monotonic()
    try:
        return old_execute(self, sql, params)
    finally:
        duration = time.monotonic() - start
        add_query_log(sql=sql, duration=duration)

@functools.wraps(old_executemany)
def new_executemany(self, sql, param_list):  # type: ignore
    """
    Wraps original cursor executemany method and adds thread safe timing to it.
    """
    start = time.monotonic()
    try:
        return old_executemany(self, sql, param_list)
    finally:
        duration = time.monotonic() - start
        add_query_log(sql=sql, duration=duration)

# Monkey patch
def patch_cursor() -> None:
    CursorWrapper.execute = new_execute
    CursorWrapper.executemany = new_executemany

def get_per_query_stat(query_stats):
    return "\n".join(str(stat) for stat in query_stats)

class RequestMetricsMiddleware(object):
    """
    Middleware reporting request metrics (such as processing time) to statsd

    How to use it:
    * add this middleware at the beginning of MIDDLEWARE_CLASSES in your
      settings
    * configure statsd in your settings:
      http://statsd.readthedocs.io/en/v3.2.1/configure.html#in-django
    """

    IGNORED_PATHS = (
        'javascript-catalog',
        'microservice_contract-health',
        'microservice_contract-ping',
        'microservice_contract-ping-service-id',
        'microservice_contract-info',
        'microservice_public_endpoints',
    )

    def process_request(self, request):
        request._request_start_time = time.monotonic()
        request._start_resources = getrusage(RUSAGE_SELF)
        get_queries_log().clear()

    def _collect_metrics(self, request, response):
        if not REQUESTS_METRICS_ENABLED:
            return
        if not request.resolver_match:
            return

        try:
            url_name = request.resolver_match.url_name
        except AttributeError:
            logger.warning(
                'URL resolver not found', extra={'path': request.path}
            )
            url_name = UNKNOWN_URL_NAME

        view_name = request.resolver_match.view_name
        if view_name in self.IGNORED_PATHS:
            return

        common_log_params = {
            'request_method': request.method,
            'url_name': url_name,
            'status_code': response.status_code,
        }

        query_stats = get_queries_log()

        # processing time
        end_resources, end_time = getrusage(RUSAGE_SELF), time.monotonic()
        real_time = (end_time - request._request_start_time) * 1000
        sys_time = (end_resources.ru_stime - request._start_resources.ru_stime) * 1000
        user_time = (end_resources.ru_utime - request._start_resources.ru_utime) * 1000
        cpu_time = sys_time + user_time
        queries_time = sum(stat.duration for stat in query_stats) * 1000

        processing_time_metric_name = METRIC_NAME_TMPL.format(
            prefix=PROCESSING_TIME_METRIC_PREFIX, **common_log_params
        )
        processing_time_all_urls_metric_name = ALL_URLS_METRIC_NAME_TMPL.format(
            prefix=PROCESSING_TIME_METRIC_PREFIX, **common_log_params
        )
        processing_time = int(real_time)
        statsd.timing(processing_time_metric_name, processing_time)
        statsd.timing(processing_time_all_urls_metric_name, processing_time)

        # requests counter
        requests_counter_metric_name = METRIC_NAME_TMPL.format(
            prefix=REQUESTS_COUNTER_METRIC_PREFIX, **common_log_params
        )
        statsd.incr(requests_counter_metric_name)

        queries_count = len(query_stats)

        data = {
            'queries_time': queries_time,
            'queries_count': queries_count,
            'real_time': real_time,
            'sys_time': sys_time,
            'user_time': user_time,
            'cpu_time': cpu_time,
            'cpu_real_ratio': cpu_time / real_time * 100,
            'queries_real_ratio': queries_time / real_time * 100,
        }
        metadata = {
            'path': request.get_full_path(),
            'url_name': url_name,
            'view_name': view_name,
            'method': request.method,
            'http_user_agent': request.META.get('HTTP_USER_AGENT'),
            'username': request.user.username,
        }

        logger.info(
            'Request time:\n'
            '\treal: {real_time}\n'
            '\tsys: {sys_time}\n'
            '\tuser: {user_time}\n'
            '\ttotal CPU time: {cpu_time}\n'
            '\tCPU to real time ratio: {cpu_real_ratio}\n'

            'Queries:\n'
            '\tcount: {queries_count}\n'
            '\ttotal time: {queries_time}\n'
            '\tqueries time to real time ratio: {queries_real_ratio}'.format(
                **data
            ), extra={
                **data,
                **metadata
            }
        )

        if queries_count > LARGE_NUMBER_OF_QUERIES_THRESHOLD:
            logger.warning("A lot of queries ({}) in {}: {}".format(
                queries_count, view_name, get_per_query_stat(query_stats)),
                extra={**metadata}
            )
        elif queries_time > LONG_QUERIES_THRESHOLD_MS:
            logger.warning("Long queries (total of {} ms) in {}: {}".format(
                queries_time, view_name, get_per_query_stat(query_stats)),
                extra={**metadata}
            )

    def process_response(self, request, response):
        try:
            self._collect_metrics(request, response)
        except Exception:
            logger.exception('Exception during collecting metrics')
        return response


import pygame, pytmx, pyscroll
import mapscript
import transition
import ClassPG as PG
import valeurs
import Ennemis
import random
import fight

class Map:
    def __init__(self, name, walls, objecte, group, tmx_data, mapdata, mapobject, ennemis=[]):
        self.name = name
        self.walls = walls
        self.object = objecte
        self.group = group
        self.tmx_data = tmx_data
        self.map_data = mapdata
        self.mapobject = mapobject
        self.ennemis = ennemis

class Tuile(pygame.sprite.Sprite):
    def __init__(self, x, y, image, tile_size):
        super(Tuile, self).__init__()
        self.name = "tile"
        self.rect = pygame.Rect(x * tile_size, y * tile_size, tile_size, tile_size)
        self.image = image
        self.type = "tile"

class Mapmanager:
    def __init__(self, screen, player):
        self.mapobject = None
        self.maps = {}
        self.basemap = "Lobby"
        self.load_localtion()
        self.current_map = self.basemap
        self.screen = screen
        self.player = player

        self.loading = False  # Load from the file
        self.seecollision = False

        self.maps_dico = {}

        self.all_register()

        self.current_map = self.basemap

        self.changemap(self.basemap, "PlayerPos")

        self.Animation = transition.Animation(3)

        self.song = {}
        self.song['door'] = PG.son("../song/Ingame/door.mp3", "song")
        self.song['chest'] = PG.son("../song/Ingame/chest.mp3", "song")
        self.song['aie'] = PG.son("../song/Ingame/aie.mp3", "song")

        self.teleport_player("PlayerPos")

    def all_register(self):
        self.register_map("Lobby", mapscript.lobby(self))
        self.register_map("Grotte", mapscript.Grotte(self))
        self.register_map("Village", mapscript.Village(self))
        self.register_map("Maison", mapscript.Maison(self))
        self.register_map("Puit", mapscript.Puit(self))
        self.register_map("Foret", mapscript.Foret(self))
        self.register_map("Maison_Foret", mapscript.Maison_Foret(self))
        self.register_map("Plaine", mapscript.Plaine(self))
        self.register_map("MaisonPêcheur", mapscript.MaisonPêcheur(self))
        self.register_map("Plage", mapscript.Plage(self))
        self.register_map("MaisonPlage", mapscript.MaisonPlage(self))

        self.zoomIO(1.6*(self.screen.get_size()[0] / 1280))

    def reload_maps(self):
        self.register_map(self.current_map, self.maps_dico[self.current_map])
        self.zoomIO(1.6 * (self.screen.get_size()[0] / 1280))
        print('reload maps')

    def alltiles(self):
        self.alltiles_d = {}
        self.alltiles_d["Base"] = pytmx.util_pygame.load_pygame("../img/tile/alltiles.tmx", load_all_tiles=True)
        self.alltiles_d["Base2"] = pytmx.util_pygame.load_pygame("../img/tile/Base2Tiles.tmx", load_all_tiles=True)
        self.alltiles_d["Base3"] = pytmx.util_pygame.load_pygame("../img/tile/TX Props.tmx", load_all_tiles=True)

    def register_map(self, name, mapobject):
        self.current_map = name
        self.maps_dico[name] = mapobject
        tmx = pytmx.util_pygame.load_pygame(f"../img/tile/{name}.tmx", load_all_tiles=True)
        self.alltiles()
        mapdata = pyscroll.data.TiledMapData(tmx)
        map_layer = pyscroll.orthographic.BufferedRenderer(mapdata, self.screen.get_size())
        map_layer.zoom = 1.6 * (self.screen.get_size()[0] / 1280)
        self.walls = []
        objects_input = {}

        player_layer = tmx.layers
        liste = []
        for element in range(len(player_layer)):
            liste.append(player_layer[element].name)
            if player_layer[element].name == "PlayerCalque":
                u = element
        group = pyscroll.PyscrollGroup(map_layer=map_layer, default_layer=int(u))

        for obj in tmx.objects:  # Ajout des collsion dans des listes
            if obj.type == "Collision":
                self.walls.append(pygame.Rect(obj.x, obj.y, obj.width, obj.height))
                if self.seecollision: group.add(
                    PG.img("../img/menu/background.jpg", obj.x, obj.y, obj.width, obj.height, False))
            if obj.type != "Collision":
                objects_input[obj.id] = [obj, pygame.Rect(obj.x, obj.y, obj.width, obj.height), obj.type]
                if self.seecollision: group.add(
                    PG.img("../img/img.inv/case.png", obj.x, obj.y, obj.width, obj.height, False))

                # Créer un nombre aléatoire d'ennemis dans un emplacement aléatoire dans une zone donnée et les ajoutes dans le group de la map.
        ennemis = []
        for element in tmx.objects:
            if element.type == "Ennemi":
                for i in range(element.properties['nbennemis']):
                    id = int(random.choice(element.properties['Ennemis'].split(",")))
                    lvl = random.randint(int(element.properties['level'].split(",")[0]),
                                         int(element.properties['level'].split(",")[-1]))
                    x = int(random.randint(int(element.x), int(element.x + element.width)))
                    y = int(random.randint(int(element.y), int(element.y + element.height)))
                    a = Ennemis.Ennemy(id, lvl, x, y)
                    group.add(a)
                    ennemis.append(a)

        # groupe contenant le joueur et la map
        group.add(self.player)
        group.change_layer(self.player, u)

        self.maps[name] = Map(name, self.walls, objects_input, group, tmx, mapdata, mapobject, ennemis)
        mapobject.load()
        self.actionnb = mapobject.actionb

    def reloadmap(self):

        pygame.display.flip()

    def zoomIO(self, level):
        self.maps[self.current_map].zoom = level

    def save_all(self, name):
        self.maps[name].allmap.save_player_info()

    def save(self, name):
        """
        Il écrit la carte actuelle et le dictionnaire actionb dans un fichier

        :param name: Le nom du fichier de sauvegarde
        """
        print("savegarde de la map : ", name)
        w = open(valeurs.valeur.save_l + f"/save/{name}.txt", "w")
        w.write(self.current_map + "\n")

        for i in self.maps[self.current_map].mapobject.actionb.keys():
            w.write(str(i) + ": " + str(self.maps[self.current_map].mapobject.actionb[i]) + "\n")
        w.close()

    def load(self, name):
        try:
            w = open(valeurs.valeur.save_l + f"/save/{name}.txt", "r")
        except FileNotFoundError:
            open(valeurs.valeur.save_l + f"/save/{name}.txt", "w")
            w = open(valeurs.valeur.save_l + f"/save/{name}.txt", "r")
        liste = w.readlines()
        a = {}
        for i in liste:
            try:
                a[i.split(":")[0]] = eval(i.split(":")[1])
            except IndexError:
                pass

        if a != {}:
            return a
        else:
            return None

    def load_localtion(self):
        """ Fonction qui change la location du spawnpoint"""
        a = open(valeurs.valeur.save_l + "/save/sauvegarde.txt", "r")
        liste = a.readlines()
        try:
            self.basemap = liste[0].split(": ")[1].strip()
        except IndexError:
            self.basemap = "Lobby"
        return self.basemap

    def get_map(self):
        return self.maps[self.current_map]

    def get_group(self):
        return self.get_map().group

    def get_walls(self):
        return self.get_map().walls

    def get_objectinp(self, type, returntype="all"):
        """
        Fonction qui renvoie une liste qui renvoie les objet d'un certain type
        """
        l = []
        if returntype == "rect":
            for i in self.get_map().object.keys():
                if self.get_map().object[i][2] == type:
                    l.append(self.get_map().object[i][1])
        if returntype == "object":
            for i in self.get_map().object.keys():
                if self.get_map().object[i][2] == type:
                    l.append(self.get_map().object[i][0])
        if returntype == "all":
            for i in self.get_map().object.keys():
                if self.get_map().object[i][2] == type:
                    l.append(self.get_map().object[i])
        return l

    def get_allobject(self, returntype="all"):
        """
        Fonction qui renvoie tout les objects
        :param returntype:
        :return:
            returntype = rect : return pygame.Rect
            returntype = object : return object
            returntype = all : [object, pygame.Rect]
        """
        l = []
        if returntype == "rect":
            for i in self.get_map().object.keys():
                l.append(self.get_map().object[i][1])
        if returntype == "object":
            for i in self.get_map().object.keys():
                l.append(self.get_map().object[i][0])
        if returntype == "all":
            for i in self.get_map().object.keys():
                l.append(self.get_map().object[i])
        return l

    def get_object(self, id):
        return self.get_map().tmx_data.get_object_by_name(id)

    def teleport_player(self, name):
        point = self.get_object(name)
        self.player.rect.centerx, self.player.rect.centery = point.x, point.y
        self.player.save_old_location()

    def draw(self):
        self.get_group().draw(self.screen)
        self.get_group().center(self.player.rect.center)

    def updating(self):
        self.maps[self.current_map].group.update()
        self.collision()

        try:
            self.maps[self.current_map].mapobject.update()
        except KeyError:
            pass

    def collision(self):
        for i in self.get_group().sprites():
            if i.type == 'Joueur':
                if i.feet.collidelist(
                        self.get_walls()) > -1:  # si les pieds du joueurs entre en collision avec un objet
                    i.moveback()
                player = i
                self.maps[self.current_map].mapobject.collision(i)

        for i3 in self.maps[self.current_map].ennemis:
            if i3.rect.colliderect(player.rect):
                fightt = fight.fight(self.player, i3.id,i3.lvl)
                fightt.image(self.screen)
                self.player.allinputoff(False)
                if fightt.FightScreen() in ["Player", "Fuite"]:
                    self.maps[self.current_map].ennemis.remove(i3)
                    self.get_group().remove(i3)
                else:
                    self.changemap(self.load_localtion(), "PlayerPos")
                    self.player.hp = self.player.hpmax
                self.player.allinputoff(True)
                self.player.pressed[pygame.K_l] = False

    def checkcollision(self, x, y):
        for element in self.get_walls():
            if element.collidepoint((x, y)):
                return True

    def changemap(self, name, teleportpoint):
        """
        Change la map actuel
        :param name: str
        :param teleportpoint: str
        :return: Nothing
        """
        self.current_map = name
        self.mapobject = self.maps[name].mapobject
        self.teleport_player(teleportpoint)

    def remove_collision(self, obj):
        self.maps[self.current_map].walls.remove(pygame.Rect(obj.x, obj.y, obj.width, obj.height))

    """def remove_inputaction(self, obj):
        self.maps[self.current_map].object"""

    def add_collision_from_obj(self, obj):
        self.maps[self.current_map].walls.append(pygame.Rect(obj.x, obj.y, obj.width, obj.height))

    def add_collision_from_rect(self, rect):
        self.maps[self.current_map].walls.append(rect)

    def add_image_to_draw(self, file, x, y, w, h):
        img = PG.img(file, x, y, w, h, True)
        self.maps[self.current_map].group.add(img)

    def add_element_to_draw_obj(self, name: str, gid: int, x: int, y: int, layer: int):
        img = self.alltiles_d[name].get_tile_image_by_gid(gid + 1)
        self.maps[self.current_map].group.add(Tuile(x, y, img, 32), layer=layer)

    def add_class_to_draw_obj(self, object):
        self.maps[self.current_map].group

    def remove_element_to_draw_obj(self, x, y):
        try:
            for element in self.maps[self.current_map].group:
                if element.name == "tile":
                    if element.rect.x == x * 32 and element.rect.y == y * 32:
                        self.maps[self.current_map].group.remove(element)
        except AttributeError:
            pass

    def set_layer_visible(self, layer, Visible):
        """
        Met le layer en visible ou non
        :param layer: str
        :param Visible: bool
        :return: Nothing
        """
        self.maps[self.current_map].tmx_data.get_layer_by_name(layer).visible = Visible

    def get_layer_by_name(self, name):
        return self.get_map().tmx_data.get_layer_by_name(name)