# Basic data wranging and plotting
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot
# Creating the modeling dataset
from sklearn.datasets import make_classification
# Data processing
import pandas as pd
import numpy as np
# Data visualization
import matplotlib.pyplot as plt
import seaborn as sns
# Model and performance
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import precision_recall_curve, accuracy_score, classification_report, confusion_matrix, roc_auc_score, roc_curve, precision_recall_fscore_support
# Oversampling and under sampling
from imblearn.over_sampling import RandomOverSampler, SMOTE
from imblearn.under_sampling import RandomUnderSampler, NearMiss
from collections import Counter
# Processing time
import time
# Statistic
import statistics as s
# A little try for Reloading Utils when we update
from importlib import reload  # Python 3.4+ :: Just call the u = u.reload(u) on the main notebook
import sys

def model_run(models,resamplers,X,y,X_bbt,y_bbt,random_state,test_size,verbose,print_report):
    """

    INPUT
    models = Pandas List with model names
    resamples = Pandas List with resampler names
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    random_state = INT random state number
    verbose = STR to switch execution log on or off
    print_report = STR to switch result printing on or off

    OUTPUT
    """

    start_time = time.time() #Start processing time
    print('Model training and evaluation routine started.')

    #Create list to hold firs run's trained models
    model_prediction = []

    #Collect dataframe to collect scores during first run
    Scores = ['Model',
            'Model Name',
            'Resampler Name',
            'TN',
            'FP',
            'FN',
            'TP',
            'Precision 0',
            'Precision 1',
            'Recall 0',
            'Recall 1',
            'F1-Score 0',
            'F1-Score 1',
            'Support 0',
            'Support 1',
            'FPR',
            'TPR',
            'Thresholds']
    model_scores_table = pd.DataFrame()
    model_scores_table['Scores'] = Scores

    #Create Dataframe to collect first run model's coeficient and feature importance measures
    model_coef_table = pd.DataFrame()
    model_coef_table['Features'] = X.columns

    for model_name in models:
        for resampler_name in resamplers:

            #Split, train and predict on models
            model,cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds =  model_predict(model_name,
                                                                                resampler_name,
                                                                                X,
                                                                                y,
                                                                                random_state,
                                                                                test_size,
                                                                                verbose,
                                                                                print_report)
            model_prediction.append((model,model_name,resampler_name))
            if verbose != 'off':print(model_name+' '+resampler_name+' Split, Training and Test Prediction ready.')

            #Collect Scores
            models_scores_append = [model,
                                    model_name,
                                    resampler_name,
                                    cm[0][0],
                                    cm[0][1],
                                    cm[1][0],
                                    cm[1][1],
                                    precision[0],
                                    precision[1],
                                    recall[0],
                                    recall[1],
                                    fbeta_score[0],
                                    fbeta_score[1],
                                    support[0],
                                    support[1],
                                    fpr,
                                    tpr,
                                    thresholds]

            model_scores_table[model_name+' '+resampler_name] = models_scores_append
            if verbose != 'off':print(model_name+' '+resampler_name+' Scores collected.')

            #Collect Coeficients and Feature Importance
            if model_name == 'Random Forest': model_coef_table_append = model.feature_importances_
            elif model_name == 'Logistic Regression': model_coef_table_append = model.coef_[0]

            model_coef_table[model_name+' '+resampler_name] = model_coef_table_append
            if verbose != 'off':print(model_name+' '+resampler_name+' Coeficients and Features Importance collected.')

            print('           '+model_name+' '+resampler_name+' routine finished')

    # Let's transpose the Model Scores Table to get a better look
    model_scores_table = transpose_model_scores(model_scores_table)

    print('Model training and evaluation finished.')
    print('---------------------------------------')

    # ---------------------------------------------------------------------------------
    #Run Big Blind Test routines:
    print('Starting Big Blind Test routine.')

    #Create dataframe to collect Big Blind Test performance scores
    BBT_model_scores_table = pd.DataFrame()
    BBT_model_scores_table['Scores'] = Scores

    #Create scaled version of X to use on Logistic Regression Model
    scaler = StandardScaler().fit(X_bbt)
    X_bbt_scaled = scaler.transform(X_bbt)
    X_bbt_scaled

    for model in model_prediction:
        BBT_model = model[0]
        BBT_model_name = model[1]
        BBT_resampler_name = model[2]

        #Predict on new Big Blind Test data
        if BBT_model_name == 'Logistic Regression':
            bbt_prediction = model[0].predict(X_bbt_scaled)
            bbt_model_proba = model[0].predict_proba(X_bbt_scaled)
        elif BBT_model_name == 'Random Forest':
            bbt_prediction = model[0].predict(X_bbt)
            bbt_model_proba = model[0].predict_proba(X_bbt)
        if verbose != 'off':print(BBT_model_name+' '+BBT_resampler_name+' Big Blind Data prediction ready.')

        #Evaluate perfomance of traind models on the new Big Blind Test data

        cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds = model_performance('Big Blind Test',
                                                                                BBT_model_name,
                                                                                BBT_resampler_name,
                                                                                y_bbt,
                                                                                bbt_prediction,
                                                                                bbt_model_proba,
                                                                                verbose,
                                                                                print_report)
        if verbose != 'off':print(model_name+' '+resampler_name+' Big Blind Test Score evaluation finished')

        #Collect Scores
        BBT_model_scores_table_append = [BBT_model,
                                         BBT_model_name,
                                         BBT_resampler_name,
                                         cm[0][0],
                                         cm[0][1],
                                         cm[1][0],
                                         cm[1][1],
                                         precision[0],
                                         precision[1],
                                         recall[0],
                                         recall[1],
                                         fbeta_score[0],
                                         fbeta_score[1],
                                         support[0],
                                         support[1],
                                         fpr,
                                         tpr,
                                         thresholds]

        BBT_model_scores_table[BBT_model_name+' '+BBT_resampler_name] = BBT_model_scores_table_append
        if verbose != 'off':print(BBT_model_name+' '+BBT_resampler_name+' Big Blind Data Scores collected.')

        print('           Big Blind Test '+BBT_model_name+' '+BBT_resampler_name+' routine finished')

    # Let's transpose the BBT Model Scores Table to get a better look
    BBT_model_scores_table = transpose_model_scores(BBT_model_scores_table)

    print("\nTotal Run processing time: --- %s seconds ---" % (time.time() - start_time))

    return model_prediction, model_scores_table, model_coef_table, BBT_model_scores_table

def model_predict(model_name,resampling_name,X,y,random_state,test_size,verbose,print_report):
    """
    Trains and Fits Models with different Resampling Techniques, than predicts, scores and measures coeficient and feature importance.

    INPUT
    model_name = Str with model name
    resampling_name = Str with resampling technique name
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    random_state = INT random state number
    verbose = STR to switch execution log on or off
    print_report = STR to switch result printing on or off

    OUTPUT
    model_prediction = Vector with chosen model with resampling predictions on the test set
    """
    start_time = time.time() #Count processing time
    if verbose != 'off':print('\n--------------------------------------------------------------------------------\n')
    if verbose != 'off': print('\nStarting new sequence:',model_name,'with',resampling_name)
    model_prediction = []
    y_train_resampled = []

    #Define model
    model = define_model(model_name,random_state,verbose)
    #Split train and test sets + Apply scaling when need + Apply resampling
    X_train, X_test, y_train, y_test = split_resample_sets(model_name,
                                                           resampling_name,
                                                           X,
                                                           y,
                                                           test_size,
                                                           random_state,
                                                           verbose)
    #Train model
    if verbose != 'off': print('\nFitting model.')
    model = model.fit(X_train, y_train)
    if verbose != 'off': print("\nFitting model processing time: --- %s seconds ---" % (time.time() - start_time))
    #Predict on trained model
    if verbose != 'off': print('\nPredicting on model.')
    model_prediction = model.predict(X_test)
    if verbose != 'off': print("\nModel prediction processing time: --- %s seconds ---" % (time.time() - start_time))
    #Evaluate model performance
    model_proba = model.predict_proba(X_test)[:, 1]
    cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds = model_performance('First Run',
                                                                       model_name,
                                                                       resampling_name,
                                                                       y_test,
                                                                       model_prediction,
                                                                       model_proba,
                                                                       verbose,
                                                                       print_report)

    if verbose != 'off': print("\nTotal Model processing time: --- %s seconds ---" % (time.time() - start_time))

    #pyplot.show()

    return model,cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds

def define_model(model_name,random_state,verbose):
    """
    Instantiates and defines Classifier models.

    INPUT
    model_name = Str with model name
    random_state = INT random state number
    verbose = STR to switch execution log on or off

    OUTPUT
    Returns intantiated model according to users choice.
    """
    start_time = time.time() #Count processing time
    if verbose != 'off': print('\nInstantiating',model_name,'model.')

    if model_name == 'Random Forest':
        model = RandomForestClassifier(random_state = random_state,
#                                        class_weight='balanced_subsample',
#                                        max_depth = 2,
#                                        n_estimators=40,
#                                        max_leaf_nodes=10
                                       )
        if verbose != 'off': print('\nModel ready:',model)
        if verbose != 'off': print("Model Instatiating processing time: --- %s seconds ---" % (time.time() - start_time))

    elif model_name == 'Logistic Regression':
        model =  LogisticRegression(random_state = random_state,
#                                      class_weight='balanced',
#                                      penalty="l2", # l1_ratio = 0.5, only necessary with penalty="elasticnet"
#                                      tol=0.01,
#                                      solver="lbfgs",
#                                      C=0.01
                                    )
        if verbose != 'off': print('\nModel ready:',lr)
        if verbose != 'off': print("Model Instatiating processing time: --- %s seconds ---" % (time.time() - start_time))

    else:
        print('\nNo compatible model.')

    return model

def split_resample_sets(model_name,resampling_name,X,y,test_size,random_state,verbose):
    """
    Splits and resamples X dataset and y vector.

    INPUT
    model_name = Str with model name
    resampling_name = Str with resampling method
    X = DataFrame with independent variables
    y = Vector with dependent (response) variable
    test_size = Float (0 to 1) for test size porcentage of the train/test split
    random_state = INT random state number
    verbose = STR to switch execution log on or off

    OUTPUT
    X_train = DataFrame with independent variables splitted for train set
    X_test = DataFrame with independent variables splitted for test set
    y_train = Vector with dependent variable splitted for train set
    y_test = Vector with dependent variable splitted for test set
    """
    start_time = time.time() #Count processing time

    if verbose != 'off': print('\nIniatialing split for train and test sets. \nAnalyzing need for variable rescaling.')
    if model_name == 'Logistic Regression':
        if verbose != 'off': print('\nLogistic Regression requies scaling. \nVariable rescaling necessary.')
        scaler = StandardScaler().fit(X)
        X_scaled = scaler.transform(X)
        X_scaled

        # Split into train and test
        X_train, X_test, y_train, y_test = train_test_split(X_scaled,
                                                            y,test_size = test_size,
                                                            random_state = random_state)
    elif model_name == 'Random Forest':

        if verbose != 'off': print('\nRandom Forest does not require rescalling.')
        # Split into train and test
        X_train, X_test, y_train, y_test = train_test_split(X,
                                                            y,
                                                            test_size = test_size,
                                                            random_state = random_state)

    if verbose != 'off': print('\nApplying resampling technique choosen.')

    if resampling_name == 'Baseline':
        if verbose != 'off': print("\nBaseline doesn't require resampling.")

    elif resampling_name == 'Random Over Sampling':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = RandomOverSampler(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'SMOTE':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = SMOTE(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'NearMiss KNN':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = NearMiss(version=3,random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    elif resampling_name == 'Random Under Sampling':
        if verbose != 'off': print('\nApplying',resampling_name)
        resampler = RandomUnderSampler(random_state=random_state)
        X_train, y_train= resampler.fit_resample(X_train, y_train)

    if verbose != 'off': print("\nResampling processing time: --- %s seconds ---" % (time.time() - start_time))

    return X_train, X_test, y_train, y_test

def model_performance(run,model_name,resampling_name,y_test,model_prediction,model_proba,verbose,print_report):
    """
    Prints the performance reports: Classification report and Confusion Matrix

    Inputs
    run = Str indicating which run is being executed. Accepts only 'First Run' and 'Big Blind Test'
    model_name = Str with model name
    resampling_name = Str with resampling method
    y_test = Test vector
    model_prediction = Prediction vector
    verbose = STR to switch execution log on or off
    print_report = STR to switch resulting printing on or off

    Returns print with the reports
    """
    start_time = time.time() #Count processing time
    cr = classification_report(y_test, model_prediction,zero_division=0)
    cm = confusion_matrix(y_test, model_prediction)
    precision, recall, fbeta_score, support = precision_recall_fscore_support(y_test, model_prediction)
    if print_report != 'off': print('\n',model_name,'with',resampling_name,' Classification Report:')
    if print_report != 'off': print(cr)
    if print_report != 'off': print(cm)

    if verbose != 'off': print("\nModel Performane processing time: --- %s seconds ---" % (time.time() - start_time))

    # Plot the ROC Curve
    #Implementation derived from :
    ## https://scikit-learn.org/stable/modules/model_evaluation.html#roc-metrics
    ## https://stackoverflow.com/questions/33208897/how-to-interpret-this-triangular-shape-roc-auc-curve

    if run == 'First Run':
        fpr, tpr, thresholds = roc_curve(y_test, model_proba)
        pyplot.plot(fpr, tpr,label=run+' '+model_name+' '+resampling_name)
    elif run == 'Big Blind Test':
        fpr, tpr, thresholds = roc_curve(y_test, model_proba[:, 1])
        pyplot.plot(fpr, tpr,label=run+' '+model_name+' '+resampling_name,linestyle='--')
    pyplot.title('ROC and AUC Plot for tested models')
    pyplot.xlabel('False Positive Rate')
    pyplot.ylabel('True Positive Rate')
    pyplot.legend(bbox_to_anchor=(1,1), loc="upper left")

    return cr,cm, precision, recall, fbeta_score, support, fpr, tpr, thresholds

def Model_Coef_Table(models, X):
    """
    Function for analyzing the different importance coefients of both Logistic Regression and Random Forest models.
    Prints out a table and bar graph of the top 10 coeficients (simple summ)

    INPUT
    model_prediction = List containing the outputs of model_predict function:
                            model = Class Fitted Classfier model instances (Logistic Regression or Ranfom Forest)
                            cr = Str Classification Report
                            cm = NP.Array Confusion Matrix
                            precision =
                            recall =
                            fbeta-score =
                            support =
                            model_name = Str with model name
                            resampling_name = Str with resampling technique name
    X = pd.DataFrame with predictor (independent) variables

    OUTPUT
    Model_Coef_Table = Pandas Dataframe with Features and Coefients

    """
    Model_Coef_Table = pd.DataFrame()
    Model_Coef_Table['Features'] = X.columns

#     Model_Score_Table['Model'] = model_prediction[7] + model_prediction[8]

    for model in models:

        model_ = model[0] # Model
        model_name = model[1]# model_name
        resampler_name = model[2]# resampler_name

        print(model_)

        if model_name == 'Random Forest': Model_Coef_Table['Feature Importance '+model_name+resampler_name] = model_.feature_importances_
        elif model_name == 'Logistic Regression': Model_Coef_Table['Coeficient '+model_name+resampler_name] = model_.coef_[0]

    return Model_Coef_Table

def transpose_model_scores(model_scores_table):
    """
    Transposes Model Scores Table for easier analysis

    INPUT
    model_scores_table = pd.DataFrame() with model scores

    OUTPUT
    model_scores_table_T = pd.DataFrame() of transposed model scores table
    """
    # Let's transpose the Model Scores Table to get a better look
    model_scores_table_T = model_scores_table.T
    # Now let's promote the first row as header and drop the index
    model_scores_table_T = model_scores_table_T.rename(columns=model_scores_table_T.iloc[0]).drop(model_scores_table_T.index[0]).reset_index(drop=True)
    # Let's clean out the score related to predicting the majority class (1) and focus only on the minority
    #model_scores_table_T = model_scores_table_T.drop(columns=['Precision 1','Recall 1','F1-Score 1'])
    return model_scores_table_T


# -*- coding: utf-8 -*-
"""
Created on Wed Feb  5 10:29:04 2020

@author: tengjunwan
"""
import numpy as np

def adjust_anchors(anchors, locs):
    """这个函数是将anchors通过rpn得到的locs进行校正，注意，这个校正是对所有的anchor
    进行的，不论输入的input的图片的anchor是否为正样本或者其他。
        anchors(array):维度为(#anchors, 4),4为(xmin,ymin,xmax,ymax)
        locs(array):维度为(#anchors, 4),4为rpn网络的输出(tx,ty,tw,th)
    """
    if anchors.shape[0] == 0:  # 如果传进来的anchors数量为0
        return np.zeros((0, 4), dtype=locs.dtype)  # 返回一个(0,4)维的array

    # 转换为（w,h,ctr_x,ctr_y）的形式
    anchors_width = anchors[:, 2] - anchors[:, 0]
    anchors_height = anchors[:, 3] - anchors[:, 1]
    anchors_ctr_x = anchors[:, 0] + 0.5 * anchors_width
    anchors_ctr_y = anchors[:, 1] + 0.5 * anchors_height

    tx = locs[:, 0]
    ty = locs[:, 1]
    tw = locs[:, 2]
    th = locs[:, 3]

    ctr_x = tx * anchors_width + anchors_ctr_x
    ctr_y = ty * anchors_height + anchors_ctr_y
    width = np.exp(tw) * anchors_width
    height =  np.exp(th) * anchors_height

    # 再将adjust后的boxes改为(xmin,ymin,xmax,ymax)的形式,取名为roi,region of
    # interst
    roi = np.zeros(locs.shape, dtype=locs.dtype)  # 即(#anchors, 4)
    roi[:, 0] = ctr_x - 0.5 * width  # xmin
    roi[:, 2] = ctr_x + 0.5 * width  # xmax
    roi[:, 1] = ctr_y - 0.5 * height # ymin
    roi[:, 3] = ctr_y + 0.5 * height # ymax
    #print(roi.shape)
    return roi

def non_maximum_suppression(roi, thresh):
    """他原来的代码写的好鸡儿复杂，似乎涉及到了cuda代码(cupy)的编写，我这里的代码
    主要是网页上的代码的编写，不涉及到gpu计算
    参数:
        roi(array):维度(m,4),m表示在调节anchor之后进行loc后的剪切以及去除过小边的
            box的个数,m<=#anchors
        thresh(int):nms算法中去除"重叠一起"的roi的阈值
    """
    xmin = roi[:, 0]
    ymin = roi[:, 1]
    xmax = roi[:, 2]
    ymax = roi[:, 3]

    areas = (xmax - xmin) * (ymax - ymin)
    keep = []  # 初始化保留的roi的index的list
    order = np.arange(roi.shape[0])  # [0,1,2...,m-1]
    while order.size > 0:
        i = order[0]  # 取最大的score的roi的index,第一次遍历i=0
        keep.append(i)  # 收入keep中
        xx1 = np.maximum(xmin[i], xmin[order[1:]]) # (m-1,),m-1为剩下的roi
        xx2 = np.minimum(xmax[i], xmax[order[1:]])
        yy1 = np.maximum(ymin[i], ymin[order[1:]])
        yy2 = np.minimum(ymax[i], ymax[order[1:]])

        width = np.maximum(0.0, xx2 - xx1)
        height =  np.maximum(0.0, yy2 - yy1)
        inter = width * height  # (m-1,)挑出的roi和剩下的roi的重叠面积
        iou = inter / (areas[i] + areas[order[1:]] - inter)  # (m-1,)

        idx = np.where(iou <= thresh)[0]  # 表示剩下的roi(不满足一定重叠率)
        order = order[1 + idx]  # 剔除score最大以及iou重叠率的roi剩下roi的index

    roi_after_nms = roi[keep]
    return roi_after_nms, keep

class ProposalCreator():
    """
    通过调用内置函数__call__来通过rpn网络的locs值来校正anchors的位置,主要是完成
    rpn网络之后的nms工作(non maximum supression)
    参数:
        nms_thresh:
        n_train_pre_nums(int):在train阶段，进行nms之前保留的top anchors的数量/。
        n_train_post_nms(int):在test阶段，进行nms之后保留的top boxes的数量(基于
            anchor的校正)。
        n_test_pre_nms(int):同理，test阶段不同的设置。
        n_test_post_nms(int):同理，test阶段不同的设置。
        min_size(int):如果修正后的boxes的最小边小于该值,舍弃,防止roipool层切割得到
           某个维度为0.
    """
    def __init__(self,
                 parent_model,  # 传入rpn网络
                 nms_thresh=0.7,
                 n_train_pre_nms=12000,
                 n_train_post_nms=2000,
                 n_test_pre_nms=6000,
                 n_test_post_nms=300,
                 min_size=16
                 ):
        self.parent_model = parent_model
        self.nms_thresh = nms_thresh
        self.n_train_pre_nms = n_train_pre_nms
        self.n_train_post_nms = n_train_post_nms
        self.n_test_pre_nms = n_test_pre_nms
        self.n_test_post_nms = n_test_post_nms
        self.min_size = min_size

    def __call__(self, locs, scores, anchors, img_size):
        """
        参数:
            locs(array):维度(#anchors, 4),从rpn网络的输出rpn_locs传入
            scores(array):维度(#anchors, ),从rpn网络的输出rpn_scores传入
            anchors(array):维度(#anchors, 4),从数据中读取
            img_size(tuple of ints):(宽，高),从数据中读取
        返回:
            roi(array):维度为(#roi, 4),4为(xmin,ymin,xmax,ymax),如果是在training
                阶段,#roi=2000,如果是eval阶段,#roi=300
        """
        # self.parent_model是rpn,如果rpn.eval()会使得rpn.training = False
        if self.parent_model.training:
            n_pre_nms = self.n_train_pre_nms  # 12000
            n_post_nms = self.n_train_post_nms  # 2000
        else:
            n_pre_nms = self.n_test_pre_nms  # 6000
            n_post_nms = self.n_test_post_nms # 300

        # 先默认训练模式(v.10该回了上面的判断)
#        n_pre_nms = self.n_train_pre_nms  # 12000
#        n_post_nms = self.n_train_post_nms  # 2000

        roi = adjust_anchors(anchors, locs)  # (#anchors, 4),(xmin,ymin,xmax,ymax)

        # 剪切,因为校正的关系，会出现出界的情况
        roi[:, [0, 2]] = np.clip(roi[:, [0, 2]], 0, img_size[0])  # x轴剪切
        roi[:, [1, 3]] = np.clip(roi[:, [1, 3]], 0, img_size[1])  # y轴剪切
        #print("check1", roi.shape)
        # 去除 高 或 宽 < min_size的roi
        min_size = self.min_size
        roi_width = roi[:, 2] - roi[:, 0]
        roi_height = roi[:, 3] - roi[:, 1]
        keep = np.where((roi_width >= min_size) &
                        (roi_height >= min_size))[0]  # 满足条件的行index
        roi = roi[keep, :]

        #print("check2", roi.shape)

        # 对roi通过rpn的cls scores进行排序
        scores = scores[:, 1]  # 取第二列为“是object”的评分
        scores = scores[keep]  # (#keep,) 这里 #keep <= #anchors
        order = scores.argsort()[::-1]  # 得到scores的下降排列的坐标
        order = order[: n_pre_nms]   # 保留固定数量 训练 12000
        #print(order.shape)
        roi = roi[order]  # 从scores高到低的排序的roi
        #print("check3", roi.shape)  # 这里还是12000
        # nms
        roi_after_nms, _ = non_maximum_suppression(roi, thresh=self.nms_thresh)
        roi_after_nms = roi_after_nms[:n_post_nms]  # 保留固定数量 训练2000
        return roi_after_nms  # roi

if __name__ == "__main__":
    path = 'pretrained_model/checkpoints/vgg16-397923af.pth'
    from vgg16_to_rpn import Vgg16ToRPN
    vgg16torpn = Vgg16ToRPN(path)
    import torch
    x = torch.rand((1, 3, 600, 850))  # 第一张图片的大小

    rpn_locs, rpn_scores, anchors = vgg16torpn(x)  # tensor,tensor,numpy
    # 除去第一个batch维度
    # 因为我这里propoal的大部分操作是numpy操作
    rpn_locs = rpn_locs[0].detach().numpy()
    rpn_scores = rpn_scores[0].detach().numpy()

    proposalcreator = ProposalCreator()
    size = (600, 850)
    roi = proposalcreator(rpn_locs, rpn_scores, anchors, size)

    print(roi)  # (1653, 4)实际上不足2000，这种不足是在nms中导致的


'''
----------------------------------------------
Project: The Office
File: data_quality_check.py
Description:

    performs the following checks:-
    - row count check
    - null values check

-----------------------------------------------
'''

from doctest import DocFileTest
import pandas as pd
from helper import load_yaml_data

DATA_PATH = '../data/'

def dq_on_imdb(df):
    check_row_count(df)
    check_null_values(df)

    print("Data Quality Check is completed for IMDb data")

def dq_on_dialogues(df):
    check_row_count(df)
    check_null_values(df)

    print("Data Quality Check is completed for Transcripts data")

def check_row_count(df):
    if len(df) > 0:
        print("Row count check is successful")

    else:
        raise Exception("Row count check has failed")

def check_null_values(df):
    if df.isnull().values.any() == False:
        print("No null values found")

    else:
        raise Exception("There are null values in data")

if __name__ == '__main__':

    data = load_yaml_data()

    imdb_file_name = data['imdb_file_name']
    dialogue_file_name = data['dialogue_file_name']

    df = pd.read_csv(DATA_PATH + imdb_file_name)

    dq_on_imdb(df)

    df = pd.read_csv(DATA_PATH + dialogue_file_name)

    dq_on_dialogues(df)


# -*- coding: utf-8 -*-
"""
Created on Sun Feb 21 09:18:20 2021

@author: hailey
"""
import math
import pandas as pd
from csv import writer
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt
import seaborn as sns

def euclidean_distance(point1, point2):
    sum_sqrd_dist = 0
    for i in range(len(point1)):
        sum_sqrd_dist += math.pow(point1[i]-point2[i], 2)
    return math.sqrt(sum_sqrd_dist)

def getNeighbors(patient, df):

    # calc euclidean distance between patient and database points
    dist_list = []
    dfPatient = []
    for i in range (len(df['ratio 2'])):
        dist = euclidean_distance([patient.loc[0]['ratio 2'], patient.loc[0]['ratio 5']], [df.loc[i]['ratio 2'], df.loc[i]['ratio 5']])
        dfPatient.append(df.loc[i]['patient'])
        dist_list.append(dist)

    data = pd.DataFrame({'patient': dfPatient, 'distance': dist_list})
    data = data.sort_values('distance')

    return data.head(3)

df = pd.read_csv('database.csv')

# normalize data set
"""

norm_r1 = pd.DataFrame((preprocessing.normalize([np.array(df['ratio 1'])])).transpose())
norm_r2 = pd.DataFrame((preprocessing.normalize([np.array(df['ratio 2'])])).transpose())
norm_r3 = pd.DataFrame((preprocessing.normalize([np.array(df['ratio 3'])])).transpose())
norm_r4 = pd.DataFrame((preprocessing.normalize([np.array(df['ratio 4'])])).transpose())
norm_r5 = pd.DataFrame((preprocessing.normalize([np.array(df['ratio 5'])])).transpose())

norm_df = pd.concat([norm_r1, norm_r2, norm_r3, norm_r4, norm_r5], axis = 1)

norm_df.to_csv('norm_database.csv', index = False)

norm_df = pd.read_csv('norm_database.csv')
"""

df.head()
df['class'].unique()
df.isnull().values.any()

df['class'] = df['class'].map({'faces-1' :0, 'faces-2' :1, 'faces-3' :2}).astype(int) #mapping numbers
df.head()

plt.close()
sns.set_style('whitegrid');
sns.pairplot(df, hue = 'class', height = 3);
plt.show()

sns.set_style('whitegrid');
sns.FacetGrid(df, hue='class', size=5) \
.map(plt.scatter, 'ratio 2', 'ratio 5') \
.add_legend();
plt.show()

x_data = df.drop(['class', 'patient', 'ratio 4', 'ratio 3', 'ratio 1'], axis=1)

y_data = df['class']

# normalize data
MinMaxScaler = preprocessing.MinMaxScaler()
X_data_minmax = MinMaxScaler.fit_transform(x_data)

data = pd.DataFrame(x_data, columns = ['ratio 2', 'ratio 5'])
df.head()

X_train, X_test, y_train, y_test = train_test_split(data, y_data, test_size=0.2, random_state = 1)

knn_clf = KNeighborsClassifier()
knn_clf.fit(X_train,y_train)
ypred = knn_clf.predict(X_test) #These are the predicted output values

result = confusion_matrix(y_test, ypred)
print('Confusion Matrix:')
print(result)
result1 = classification_report(y_test, ypred)
print('Classification Report:',)
print (result1)
result2 = accuracy_score(y_test,ypred)
print('Accuracy:',result2)

#some kind of action that connects with button
curPatient = pd.read_csv('curPatient.csv')

kNeighbors = getNeighbors(curPatient, df)


"""Logic for when this module is called from the command line.
"""

import importlib
import sys

import soonq as sq

# Import example modules.
EXAMPLE_FUNCTIONS = (
    '.clear_queue', '.clear_work',
    '.error_task', '.error_worker',
    '.sim_task', '.sim_worker',
    '.timer_task', '.timer_worker',
    '.view_queue', '.view_work',
)
EXAMPLE_MODULES = [importlib.import_module(func, package='examples')
    for func in EXAMPLE_FUNCTIONS]
EXAMPLE_MODULE_MAP = {mod.__name__.rsplit('.', maxsplit=1)[-1]: mod
    for mod in EXAMPLE_MODULES}
EXAMPLE_MODULE_DESCRIPTIONS = [mod.__doc__.rstrip('\n')
    for mod in EXAMPLE_MODULES]

HELP_OPTIONS = ('-h', '--help')
OPTIONS = [(HELP_OPTIONS, "Show this message and exit.")]

def execute(command):
    try:
        cmd = getattr(EXAMPLE_MODULE_MAP[command], command)
    except AttributeError:
        sq.echo(f"Example {command!r} not recognized.")
        return
    cmd()

def print_help():
    sq.echo(f"Usage: python -m examples [OPTIONS] COMMAND [ARGS]...\n")
    sq.echo("Options:")
    for opts, desc in OPTIONS:
        sq.echo(f"  {', '.join(opts)}  {desc}")
    sq.echo()
    sq.echo("Commands:")
    max_modname_len = max(len(mod) for mod in EXAMPLE_MODULE_MAP)
    for cmd, descr in zip(
            EXAMPLE_MODULE_MAP.keys(), EXAMPLE_MODULE_DESCRIPTIONS):
        sq.echo(f"  {cmd:<{max_modname_len+2}}{descr}")

command = sys.argv[1]
if command in HELP_OPTIONS:
    print_help()
else:
    execute(command)


from tornado import gen

from tornadowebapi import exceptions
from tornadowebapi.resource import Resource
from tornadowebapi.resource_handler import ResourceHandler
from tornadowebapi.traitlets import Unicode

from remoteappmanager.webapi.decorators import authenticated
from remoteappmanager.db import exceptions as db_exceptions

class User(Resource):
    name = Unicode(allow_empty=False, strip=True)

class UserHandler(ResourceHandler):
    resource_class = User

    @gen.coroutine
    @authenticated
    def delete(self, resource, **kwargs):
        db = self.application.db
        try:
            identifier = int(resource.identifier)
        except ValueError:
            raise exceptions.NotFound()

        try:
            db.remove_user(id=identifier)
            self.log.info("Removed user with id {}".format(identifier))
        except db_exceptions.NotFound:
            raise exceptions.NotFound()
        except db_exceptions.UnsupportedOperation:
            raise exceptions.Unable()

    @gen.coroutine
    @authenticated
    def create(self, resource, **kwargs):
        name = resource.name

        db = self.application.db
        try:
            resource.identifier = str(db.create_user(name))
        except db_exceptions.Exists:
            raise exceptions.Exists()
        except db_exceptions.UnsupportedOperation:
            raise exceptions.Unable()

    @gen.coroutine
    @authenticated
    def items(self, items_response, **kwargs):
        """Produces a list of User items in the items_response object.

        Parameters
        ----------
        items_response: ItemsResponse
            an object to be filled with the appropriate information
        """
        users = self.application.db.list_users()

        items_response.set([
            User(identifier=str(u.id), name=u.name)
            for u in users])


from typing import List

class Solution:
    """
    Task:
    Given an integer array nums, move all the even integers at the beginning of the array followed by all the odd integers.

    Return any array that satisfies this condition.
    """
    def sortArrayByParity(self, nums: List[int]) -> List[int]:
        if len(nums) == 0:
            return nums
        writeIdx = 0
        for readIdx in range(len(nums)):
            if nums[readIdx] % 2 == 0:
                nums[readIdx], nums[writeIdx] = nums[writeIdx], nums[readIdx]
                writeIdx += 1
        return nums


import numpy as np
import torch
from torch import nn
import torch.nn.functional as F

from coop_marl.utils.nn import ortho_layer_init

class FCLayers(nn.Module):
    def __init__(self, input_dim, hidden_size, num_hidden, output_size,
                 activation_fn=nn.ELU, base_std=np.sqrt(2), head_std=0.01,
                last_linear=True, layer_init_fn=ortho_layer_init):
        assert num_hidden > 0
        super().__init__()
        layer_construct = nn.Linear

        pre_hidden = [layer_init_fn(layer_construct(input_dim, hidden_size), base_std),
                                activation_fn()
                    ]
        base = pre_hidden
        if num_hidden>1:
            hiddens = []
            for i in range(num_hidden-1):
                hiddens.append(layer_init_fn(layer_construct(hidden_size, hidden_size), base_std))
                hiddens.append(activation_fn())
            base.extend(hiddens)
        out = [layer_init_fn(layer_construct(hidden_size,output_size), head_std)]
        if not last_linear:
            out.append(activation_fn())
        self.layers = nn.Sequential(*base, *out)

    def forward(self, x):
        return self.layers(x)

# taken from: https://github.com/keynans/HypeRL/blob/main/PEARL/torch/sac/hyper_network.py
class HyperHead(nn.Module):
    def __init__(self, base_inp_dim, meta_hidden_dim, output_size, stddev=0.05):
        super().__init__()
        self.output_size = output_size
        self.base_inp_dim = base_inp_dim
        self.w = nn.Linear(meta_hidden_dim, base_inp_dim * output_size)
        self.b = nn.Linear(meta_hidden_dim, output_size)
        self.init_layers(stddev)

    def forward(self, x):
        w = self.w(x).view(-1, self.output_size, self.base_inp_dim)
        b = self.b(x).view(-1, self.output_size, 1)
        return w, b

    def init_layers(self,stddev):
        ortho_layer_init(self.w, stddev)
        ortho_layer_init(self.b, stddev)

class ResBlock(nn.Module):

    def __init__(self, in_size, out_size):
        super(ResBlock, self).__init__()
        self.layers = nn.Sequential(
                            nn.ELU(),
                            ortho_layer_init(nn.Linear(in_size, out_size)),
                            nn.ELU(),
                            ortho_layer_init(nn.Linear(out_size, out_size)),
                            )

    def forward(self, x):
        h = self.layers(x)
        return x + h

class MetaResNet(nn.Module):
    def __init__(self, meta_dim, hidden_size):
        super(MetaResNet, self).__init__()

        self.hidden_size = hidden_size
        self.layers = nn.Sequential(
            nn.Linear(meta_dim, hidden_size),
            ResBlock(hidden_size, hidden_size),
            nn.ELU()
        )

        self.init_layers()

    def forward(self, meta_v):
        return self.layers(meta_v)

    def init_layers(self):
        for module in self.layers.modules():
            if isinstance(module, (nn.Conv1d, nn.Conv2d, nn.Linear)):
                ortho_layer_init(module)

class HyperNet(nn.Module):
    def __init__(self, base_inp_dim, meta_inp_dim, hidden_size, num_heads=1, head_sizes=[256], head_std=[0.05]):
        super().__init__()
        assert isinstance(head_sizes, list)
        assert isinstance(head_std, list)
        self.encoder = MetaResNet(meta_inp_dim, hidden_size)
        self.heads = nn.ModuleList()
        self.heads.append(HyperHead(base_inp_dim, hidden_size, head_sizes[0], stddev=head_std[0]))
        for i in range(1,num_heads):
            self.heads.append(HyperHead(head_sizes[i-1], hidden_size, head_sizes[i], stddev=head_std[i]))

    def produce_wb(self, z):
        if len(z.shape)==3:
            z = z.view(-1,z.shape[2])
        meta_h = self.encoder(z)
        wb = []
        for head in self.heads:
            w,b = head(meta_h)
            wb.append((w,b))
        return wb

    def forward(self, base_inp, meta_inp):
        wb = self.produce_wb(meta_inp)
        # base_inp for ff net -> [bs, feature] -> [bs, feature ,1]
        # base_inp for rnn -> [bs, ts, feature] -> [bs * ts, feature, 1]
        if len(base_inp.shape)==2:
            # ff net
            h = base_inp.unsqueeze(-1)
        elif len(base_inp.shape)==3:
            # rnn
            h = base_inp.reshape(base_inp.shape[0]*base_inp.shape[1],base_inp.shape[2],1)

        for w,b in wb[:-1]:
            h = F.elu(torch.bmm(w, h) + b)
        w,b = wb[-1]
        out = torch.bmm(w,h) + b
        if len(base_inp.shape)==3:
            out = out.view(*base_inp.shape[:2],-1)
            return out, wb
        return out.squeeze(-1), wb


# This sample uses a UFF MNIST model to create a TensorRT Inference Engine
from random import randint
from PIL import Image
import numpy as np

import pycuda.driver as cuda
# This import causes pycuda to automatically manage CUDA context creation and cleanup.
import pycuda.autoinit

import tensorrt as trt
import sys, os

# You can set the logger severity higher to suppress messages (or lower to display more messages).
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)

def GiB(val):
    return val * 1 << 30

class HostDeviceMem(object):
    def __init__(self, host_mem, device_mem):
        self.host = host_mem
        self.device = device_mem

    def __str__(self):
        return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)

    def __repr__(self):
        return self.__str__()

def allocate_buffers(engine):
    inputs = []
    outputs = []
    bindings = []
    stream = cuda.Stream()
    for binding in engine:
        size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size
        dtype = trt.nptype(engine.get_binding_dtype(binding))
        # Allocate host and device buffers
        host_mem = cuda.pagelocked_empty(size, dtype)
        device_mem = cuda.mem_alloc(host_mem.nbytes)
        # Append the device buffer to device bindings.
        bindings.append(int(device_mem))
        # Append to the appropriate list.
        if engine.binding_is_input(binding):
            inputs.append(HostDeviceMem(host_mem, device_mem))
        else:
            outputs.append(HostDeviceMem(host_mem, device_mem))
    return inputs, outputs, bindings, stream

class ModelData(object):
    MODEL_FILE = "lenet5.uff"
    INPUT_NAME ="input_1"
    INPUT_SHAPE = (1, 28, 28)
    OUTPUT_NAME = "dense_1/Softmax"

def build_engine(model_file):
    # For more information on TRT basics, refer to the introductory samples.
    with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network, trt.UffParser() as parser:
        builder.max_workspace_size = GiB(1)
        # Parse the Uff Network
        parser.register_input(ModelData.INPUT_NAME, ModelData.INPUT_SHAPE)
        parser.register_output(ModelData.OUTPUT_NAME)
        parser.parse(model_file, network)
        # Build and return an engine.
        return builder.build_cuda_engine(network)

# Loads a test case into the provided pagelocked_buffer.
def load_normalized_case(pagelocked_buffer):
    path = '/home/yinliang/TensorRT-7.0.0.11/samples/python/end_to_end_tensorflow_mnist/data'
    #case_num = randint(0, 1)
    case_num = 0
    test_case_path = os.path.join(path, str(case_num) + '.pgm')
    img = np.array(Image.open(test_case_path)).ravel()
    np.copyto(pagelocked_buffer, 1.0 - img / 255.0)
    return case_num

# This function is generalized for multiple inputs/outputs.
# inputs and outputs are expected to be lists of HostDeviceMem objects.
def do_inference(context, bindings, inputs, outputs, stream, batch_size=1):
    # Transfer input data to the GPU.
    [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]
    # Run inference.
    context.execute_async(batch_size=batch_size, bindings=bindings, stream_handle=stream.handle)
    # Transfer predictions back from the GPU.
    [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]
    # Synchronize the stream
    stream.synchronize()
    # Return only the host outputs.
    return [out.host for out in outputs]

# This function is generalized for multiple inputs/outputs for full dimension networks.
# inputs and outputs are expected to be lists of HostDeviceMem objects.
def do_inference_v2(context, bindings, inputs, outputs, stream):
    # Transfer input data to the GPU.
    [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]
    # Run inference.
    context.execute_async_v2(bindings=bindings, stream_handle=stream.handle)
    # Transfer predictions back from the GPU.
    [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]
    # Synchronize the stream
    stream.synchronize()
    # Return only the host outputs.
    return [out.host for out in outputs]

def main():

    model_file = './models/lenet5.uff'
    with build_engine(model_file) as engine:
        # Build an engine, allocate buffers and create a stream.
        # For more information on buffer allocation, refer to the introductory samples.
        inputs, outputs, bindings, stream = allocate_buffers(engine)
        with engine.create_execution_context() as context:
            #case_num = load_normalized_test_case(data_paths, pagelocked_buffer=inputs[0].host)
            case_num = load_normalized_case(pagelocked_buffer=inputs[0].host)
            # For more information on performing inference, refer to the introductory samples.
            # The common.do_inference function will return a list of outputs - we only have one in this case.
            [output] = do_inference(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream)
            pred = np.argmax(output)
            print("Test Case: " + str(case_num))
            print("Prediction: " + str(pred))

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 22 18:36:01 2019

@author: lillian
"""

import numpy as np
import matplotlib.pyplot as plt
import time

# =============================================================================
# Excercise 1 - Set up
# =============================================================================
#Given data by PS4
theta = 0.679
beta = 0.988
delta = 0.013
h=1
kappa = 5.24
vue = 2

# Functions
def y_func(k, h):
    return k**(1-theta)*h**theta

def c(k,kk,h):
    return k**(1-theta)*h**theta+(1-delta)*k-kk

def u(c):
    return  np.log(c)
# =============================================================================
# 1.4 Value Function - Local Search
# =============================================================================
# 1. Step: Setting the grid
N_k = 100

k_grid = np.linspace(0.5, 70, N_k)

# 2. Step: Guess the Value function that we improve by iteration
V = np.zeros((N_k))
V_guess=np.zeros ((N_k))
V_result=np.empty ((N_k))
V_next = np.zeros((N_k))

### Empty policy functions
k_policy = np.zeros((N_k))
c_policy = np.zeros((N_k))
V_next = np.zeros((N_k))
m = np.zeros((N_k,N_k))
Chi = np.zeros((N_k,N_k))
negativ=-10000
k_local=0
i = 1
error=5 #seems to work for several error typs

#3. Step: Creating the m-Matrix
for ik, k in enumerate(k_grid): #ik is index number for the rows running from 0-->k_grid, while k in k_grid is an actual value
    for jk, kk in enumerate(k_grid): #jk is the index number for the columns
        if kk<y_func(k,h)+(1-delta)*k:
            m[ik,jk] = u(y_func(k,h) +(1-delta)*k - kk) #checking consumption >0
        else:
            m[ik,jk] = negativ

#4. Step: Finding the Value Function
while i<10000:
    start_time = time.time()
    if i<=1:
        V=V_guess
    else:
        V=V_next
        V_next=np.zeros((N_k))
    for ik, k in enumerate(k_grid): #ik is index number for the rows running from 0-->k_grid, while k in k_grid is an actual value
        if ik<1:
            for jk, kk in enumerate(k_grid): #jk is the index number for the columns
                Chi[ik,jk]=m[ik,jk]+beta*V[jk] #checking consumption >0
        else:                   #Creating for all ik>=1 Chi-Values in a certain radius(=error)
            low=k_local
            high=k_local+error
            if low<0:   #Controlling for k-values in k_grid
                low=0
            if high>70:
                high=70
            for jk in range (low, high): #only considering right side due to monotonicity
                Chi[ik,jk]=m[ik,jk]+beta*V[jk]
        V_next[ik] = np.nanmax(Chi[ik,:]) #value
        k_policy[ik] = np.argmax(Chi[ik,:])   #position
        c_policy[ik] = y_func(k,1) +(1-delta)*k - k_policy[ik]
        k_local=int(k_policy[ik])
    if np.allclose(V_next,V):
        V_result=V_next
        break
    else:
        i+=1
end_time = time.time()

print("total time taken this loop: ", end_time - start_time)
print('total number of iterations: ', i)

#Plotting Value Function
fig, ax = plt.subplots()
ax.plot(k_grid,V_result, label='V(k)')
plt.title ('Value Function Iteration')
plt.legend()
plt.xlabel('Time')
plt.show()

### Ploting Policy Functions
fig, ax = plt.subplots()
ax.plot(k_grid,c_policy, label='c(k)')
ax.plot(k_grid,k_policy, label='k(k)')
plt.title ('Policy Functions')
plt.legend()
plt.xlabel('Time')
plt.show()


import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable

class Generator(nn.Module):
    def __init__(self, class_dim, latent_dim, image_size):
        """ Initializes Generator Class with latent_dim and class_dim."""
        super(Generator, self).__init__()

        self.latent_dim = latent_dim
        self.class_dim = class_dim

        # self.label_emb = nn.Embedding(self.class_dim, self.latent_dim)

        self.init_size = image_size // 4  # Initial size before upsampling
        self.l1 = nn.Sequential(nn.Linear(self.latent_dim, 128 * self.init_size ** 2))

        self.conv_blocks = nn.Sequential(
            nn.BatchNorm2d(128),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(128, 128, 3, stride=1, padding=1),
            nn.BatchNorm2d(128, 0.8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(128, 64, 3, stride=1, padding=1),
            nn.BatchNorm2d(64, 0.8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(64, 3, 3, stride=1, padding=1),
            nn.Tanh(),
        )
        return

    def forward(self, _input):
        _input = _input.view(-1, self.latent_dim)
        out = self.l1(_input)
        out = out.view(out.shape[0], 128, self.init_size, self.init_size)
        img = self.conv_blocks(out)
        return img

class Discriminator(nn.Module):

    def __init__(self, class_dim, image_size):
        super(Discriminator, self).__init__()

        def discriminator_block(in_filters, out_filters, bn=True):
            """Returns layers of each discriminator block"""
            block = [nn.Conv2d(in_filters, out_filters, 3, 2, 1), nn.LeakyReLU(0.2, inplace=True), nn.Dropout2d(0.25)]
            if bn:
                block.append(nn.BatchNorm2d(out_filters, 0.8))
            return block

        self.conv_blocks = nn.Sequential(
            *discriminator_block(3, 16, bn=False),
            *discriminator_block(16, 32),
            *discriminator_block(32, 64),
            *discriminator_block(64, 128),
        )

        # The height and width of downsampled image
        ds_size = image_size // 2 ** 4

        # Output layers
        self.adv_layer = nn.Sequential(nn.Linear(128 * ds_size ** 2, 1), nn.Sigmoid())
        self.aux_layer = nn.Sequential(nn.Linear(128 * ds_size ** 2, class_dim), nn.Softmax(dim=1))

    def forward(self, _input):
        out = self.conv_blocks(_input)
        out = out.view(out.shape[0], -1)
        validity = self.adv_layer(out)
        label = self.aux_layer(out)

        return validity, label

if __name__ == '__main__':
    latent_dim = 100
    class_dim = 2
    batch_size = 128
    img_size = 64

    cuda = False if torch.cuda.is_available() else False
    FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
    LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor

    # z = Variable(FloatTensor(np.random.normal(0, 1, (batch_size, latent_dim))))
    # c = Variable(LongTensor(np.random.randint(0, class_dim, batch_size)))
    label = np.random.randint(0, class_dim, batch_size)
    noise = Variable(FloatTensor(batch_size, latent_dim, 1, 1))
    aux_label = Variable(LongTensor(batch_size))

    noise_ = np.random.normal(0, 1, (batch_size, latent_dim))
    class_onehot = np.zeros((batch_size, class_dim))
    class_onehot[np.arange(batch_size), label] = 1
    noise_[np.arange(batch_size), :class_dim] = class_onehot[np.arange(batch_size)]
    noise_ = (torch.from_numpy(noise_))
    noise.data.copy_(noise_.view(batch_size, latent_dim, 1, 1))
    aux_label.data.resize_(batch_size).copy_(torch.from_numpy(label))

    print(f'z:{noise.shape}')
    print(f'c:{aux_label.shape}')

    G = Generator(class_dim, latent_dim, img_size)
    D = Discriminator(class_dim, img_size)
    o = G(noise)
    print(o.shape)
    x, y = D(o)
    print(x.shape, y.shape)


from gurobipy import *
import networkx as nx
import random as rd

def solveMinCostFlow(G: nx.DiGraph, source: int, sink: int, demand: int) -> dict:
    """
    Solves the minimum cost flow problem.
    :param G: directed graph
    :param source: Source node in G
    :param sink: Sink node in G
    :param demand: Demand value
    :return: Dict of edges and flow values
    """

    minCostFlow = Model('MinCostFlow')

    # Variable
    X = dict()

    for a in G.edges():
        X[a] = minCostFlow.addVar(vtype=GRB.CONTINUOUS, lb=0, ub=G.get_edge_data(*a)['capacity'], name=f'X_{a}')

    # Objective function
    minCostFlow.setObjective(quicksum(X[a]*G.get_edge_data(*a)['cost'] for a in G.edges()), sense=GRB.MINIMIZE)

    # Constraints
    for v in G.nodes():
        if v == source:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, demand)
        elif v == sink:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, -demand)
        else:
            minCostFlow.addConstr(quicksum(X[a] for a in G.out_edges(v)) - quicksum(X[a] for a in G.in_edges(v)),
                              GRB.EQUAL, 0)

    # Solve model
    minCostFlow.update()
    minCostFlow.optimize()

    if minCostFlow.status == GRB.OPTIMAL:
        flows = dict()
        for a in G.edges():
            if minCostFlow.getVarByName(f'X_{a}').x > 0:
                flows[a] = minCostFlow.getVarByName(f'X_{a}').x
        return flows

    else:
        return dict()

if __name__ == '__main__':
    # Example: Complete graph on 100 vertices
    G = nx.complete_graph(100).to_directed()
    for a in G.edges():
        G[a[0]][a[1]]['capacity'] = rd.randint(1, 14)
        G[a[0]][a[1]]['cost'] = rd.randint(1, 14)

    flows = solveMinCostFlow(G, 0, 99, 5)

    print('Min cost flow edges: {}'.format(flows))


from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import math

def calc(x):
    return str(math.log(abs(12 * math.sin(int(x)))))

try:
    link = "http://suninjuly.github.io/get_attribute.html"
    options = webdriver.ChromeOptions()
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    browser = webdriver.Chrome(options=options)
    browser.get(link)

    im = browser.find_element(By.CSS_SELECTOR, 'img')
    im_valuex = im.get_attribute("valuex")
    x = int(im_valuex)
    f = str(calc(x))

    field = browser.find_element(By.ID, 'answer')
    field.send_keys(f)
    time.sleep(2)

    check = browser.find_element(By.ID, 'robotCheckbox')
    check.click()
    time.sleep(2)

    radio = browser.find_element(By.ID, 'robotsRule')
    radio.click()
    time.sleep(2)

    button = browser.find_element(By.CSS_SELECTOR, 'button')
    button.click()
    time.sleep(10)
finally:
    browser.quit()


""" Post-migration "fixers", which are executed after an object has been migrated
"""

import logging

from plone.app.multilingual.api import get_translation_manager

from .config import LANGUAGES, TOP_LEVEL, AST_PATHS, FULL_PAGE_PATHS, SECTOR_POLICIES
from .utils import make_uid

logger = logging.getLogger()

def onpath(path):

    def decorator_factory(func):
        def decorator(context):
            if not context.absolute_url(relative=True).endswith(path):
                return
            return func(context)

        return decorator

    return decorator_factory

def inpath(path):

    def decorator_factory(func):
        def decorator(context):
            if path not in context.absolute_url(relative=True):
                return
            return func(context)

        return decorator

    return decorator_factory

@onpath('/knowledge/adaptation-information/climate-services/climate-services')
def fix_climate_services_toc(context):
    # in first column block, replace the first paragraph with a horizontal navigation table of contents

    col_block_id = context.blocks_layout['items'][1]    # [0] is title block
    col = context.blocks[col_block_id]
    column_ids = col['data']['blocks_layout']['items']
    first_col_id = column_ids[0]
    first_col = col['data']['blocks'][first_col_id]

    first_block_id = first_col['blocks_layout']['items'][0]
    new_data = {"@type": 'toc',
                "variation": "horizontalMenu"}
    first_col['blocks'][first_block_id] = new_data

@onpath('/help/Webinars')
def fix_webinars(context):
    blocks = context.blocks
    layout = context.blocks_layout

    videos = {
        1:  {
            'text': {
                'id': layout['items'][-1],
                'block': blocks[layout['items'][-1]]
            },
            'video': {
                'id': layout['items'][-2],
                'block': blocks[layout['items'][-2]]
            }
        },
        2:  {
            'text': {
                'id': layout['items'][-3],
                'block': blocks[layout['items'][-3]]
            },
            'video': {
                'id': layout['items'][-4],
                'block': blocks[layout['items'][-4]]
            }
        },
        3:  {
            'text': {
                'id': layout['items'][-5],
                'block': blocks[layout['items'][-5]]
            },
            'video': {
                'id': layout['items'][-6],
                'block': blocks[layout['items'][-6]]
            }
        },
        4:  {
            'text': {
                'id': layout['items'][-7],
                'block': blocks[layout['items'][-7]]
            },
            'video': {
                'id': layout['items'][-8],
                'block': blocks[layout['items'][-8]]
            }
        },
        5:  {
            'text': {
                'id': layout['items'][-9],
                'block': blocks[layout['items'][-9]]
            },
            'video': {
                'id': layout['items'][-10],
                'block': blocks[layout['items'][-10]]
            }
        },
    }

    col_1_id = make_uid()
    col_1_item_1_id = make_uid()

    col_2_id = make_uid()
    col_2_item_1_id = make_uid()
    col_2_item_2_id = make_uid()

    col_3_id = make_uid()
    col_3_item_1_id = make_uid()
    col_3_item_2_id = make_uid()

    blocks[col_1_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_1_item_1_id: {
                    "blocks": {
                        videos[1]['video']['id']: videos[1]['video']['block'],
                        videos[1]['text']['id']: videos[1]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[1]['video']['id'],
                            videos[1]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_1_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_2_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_2_item_1_id: {
                    "blocks": {
                        videos[2]['video']['id']: videos[2]['video']['block'],
                        videos[2]['text']['id']: videos[2]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[2]['video']['id'],
                            videos[2]['text']['id'],
                        ]
                    }
                },
                col_2_item_2_id: {
                    "blocks": {
                        videos[3]['video']['id']: videos[3]['video']['block'],
                        videos[3]['text']['id']: videos[3]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[3]['video']['id'],
                            videos[3]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_2_item_2_id,
                    col_2_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_3_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_3_item_1_id: {
                    "blocks": {
                        videos[4]['video']['id']: videos[4]['video']['block'],
                        videos[4]['text']['id']: videos[4]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[4]['video']['id'],
                            videos[4]['text']['id'],
                        ]
                    }
                },
                col_3_item_2_id: {
                    "blocks": {
                        videos[5]['video']['id']: videos[5]['video']['block'],
                        videos[5]['text']['id']: videos[5]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[5]['video']['id'],
                            videos[5]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_3_item_2_id,
                    col_3_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    for index in range(0, 10):
        layout['items'].pop()

    layout['items'].append(col_3_id)
    layout['items'].append(col_2_id)
    layout['items'].append(col_1_id)

    context.blocks = blocks
    context.blocks_layout = layout
    context._p_changed = True

@onpath('/help/tutorial-videos/index_html')
def fix_tutorial_videos(context):
    blocks = context.blocks
    layout = context.blocks_layout

    videos = {
        1:  {
            'text': {
                'id': layout['items'][-1],
                'block': blocks[layout['items'][-1]]
            },
            'video': {
                'id': layout['items'][-2],
                'block': blocks[layout['items'][-2]]
            }
        },
        2:  {
            'text': {
                'id': layout['items'][-3],
                'block': blocks[layout['items'][-3]]
            },
            'video': {
                'id': layout['items'][-4],
                'block': blocks[layout['items'][-4]]
            }
        },
        3:  {
            'text': {
                'id': layout['items'][-5],
                'block': blocks[layout['items'][-5]]
            },
            'video': {
                'id': layout['items'][-6],
                'block': blocks[layout['items'][-6]]
            }
        },
        4:  {
            'text': {
                'id': layout['items'][-7],
                'block': blocks[layout['items'][-7]]
            },
            'video': {
                'id': layout['items'][-8],
                'block': blocks[layout['items'][-8]]
            }
        },

    }

    col_1_id = make_uid()
    col_1_item_1_id = make_uid()
    col_1_item_2_id = make_uid()

    col_2_id = make_uid()
    col_2_item_1_id = make_uid()
    col_2_item_2_id = make_uid()

    blocks[col_1_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_1_item_1_id: {
                    "blocks": {
                        videos[1]['video']['id']: videos[1]['video']['block'],
                        videos[1]['text']['id']: videos[1]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[1]['video']['id'],
                            videos[1]['text']['id'],
                        ]
                    }
                },
                col_1_item_2_id: {
                    "blocks": {
                        videos[2]['video']['id']: videos[2]['video']['block'],
                        videos[2]['text']['id']: videos[2]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[2]['video']['id'],
                            videos[2]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_1_item_2_id,
                    col_1_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    blocks[col_2_id] = {
        "@type": "columnsBlock",
        "data": {
            "blocks": {
                col_2_item_1_id: {
                    "blocks": {
                        videos[3]['video']['id']: videos[3]['video']['block'],
                        videos[3]['text']['id']: videos[3]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[3]['video']['id'],
                            videos[3]['text']['id'],
                        ]
                    }
                },
                col_2_item_2_id: {
                    "blocks": {
                        videos[4]['video']['id']: videos[4]['video']['block'],
                        videos[4]['text']['id']: videos[4]['text']['block'],
                    },
                    "blocks_layout": {
                        "items": [
                            videos[4]['video']['id'],
                            videos[4]['text']['id'],
                        ]
                    }
                },
            },
            "blocks_layout": {
                "items": [
                    col_2_item_2_id,
                    col_2_item_1_id,
                ]
            }
        },
        "gridCols": [
            "halfWidth",
            "halfWidth"
        ],
        "gridSize": 12,
        "styles": {}
    }

    for index in range(0, 8):
        layout['items'].pop()

    layout['items'].append(col_2_id)
    layout['items'].append(col_1_id)

    context.blocks = blocks
    context.blocks_layout = layout
    context._p_changed = True

@onpath('/news-archive')
def fix_news_archive(context):

    current_lang = context.absolute_url(relative=True).split('/')[-2]

    listing_uid = make_uid()
    title_uid = make_uid()

    context.blocks = {
        title_uid: {
            "@type": "title",
            "hideContentType": True
        },
        listing_uid: {
            "@type": "listing",
            "headlineTag": "h2",
            "block": make_uid(),
            "itemModel": {
                "@type": "item",
                "hasDate": True,
                "hasDescription": True,
                "hasImage": False,
                "hasLink": True,
                "maxDescription": 2,
                "maxTitle": 2,
                "titleOnImage": False
            },
            "query": [],
            "querystring": {
                "query": [
                    {
                        "i": "portal_type",
                        "o": "plone.app.querystring.operation.selection.any",
                        "v": [
                            "News Item",
                            "Link"
                        ]
                    },
                    {
                        "i": "path",
                        "o": "plone.app.querystring.operation.string.absolutePath",
                        "v": "/" + current_lang + "/news-archive"
                    }
                ],
                "sort_on": "effective",
                "sort_order": "descending",
                "sort_order_boolean": True
            },
            "variation": "summary"
        }
    }

    context.blocks_layout = {"items": [title_uid, listing_uid]}
    context._p_changed = True

def are_on_path(url, paths):
    for path in paths:
        if url.endswith(path):
            return True

def are_in_path(url, paths):
    for path in paths:
        if path in url:
            return True

def fix_read_more(context):
    url = context.absolute_url(relative=True)

    PATHS = [
        '/knowledge/tools/urban-ast',
        '/knowledge/tools/adaptation-support-tool',
        '/knowledge/eu-vulnerability/eu-vulnerability-to-cc-impacts-occurring-outside',
        '/countries-regions/transnational-regions/baltic-sea-region/adaptation',
        '/countries-regions/transnational-regions/carpathian-mountains'
    ]

    def get_columns_block_id(blocks):
        columns_block = {k for k, v in blocks.items() if v['@type'] == 'columnsBlock'}
        col_id = list(columns_block)[0]
        return col_id

    def get_read_more_block_id(blocks):
        read_more_block = {k for k, v in blocks.items() if v['@type'] == 'readMoreBlock'}
        if read_more_block:
            read_more_block_id = list(read_more_block)[0]
            return read_more_block_id
        else:
            return None

    if are_in_path(url, SECTOR_POLICIES):
        col_id = get_columns_block_id(context.blocks)
        col = context.blocks[col_id]
        first_col_id = col['data']['blocks_layout']['items'][0]
        first_col = col['data']['blocks'][first_col_id]
        col_items = first_col['blocks_layout']['items']
        read_more_block_id = get_read_more_block_id(first_col['blocks'])
        tiles = {k for k, v in first_col['blocks'].items()
                if v['@type'] == 'relevantAceContent' or v['@type'] == 'filterAceContent'}
        if read_more_block_id:
            read_more_index = col_items.index(read_more_block_id)
            col_items.pop(read_more_index)
            col_items.insert(len(col_items)-len(tiles), read_more_block_id) # insert before acecontent blocks
            first_col['blocks_layout']['items'] = col_items

    elif are_in_path(url, PATHS):
        items = context.blocks_layout['items']
        read_more_block_id = get_read_more_block_id(context.blocks)
        if read_more_block_id:
            read_more_index = items.index(read_more_block_id)
            items.pop(read_more_index)
            items.insert(len(items) -1, read_more_block_id) # insert before last block
            context.blocks_layout['items'] = items

    else:
        items = context.blocks_layout['items']
        read_more_block_id = get_read_more_block_id(context.blocks)
        if read_more_block_id:
            read_more_index = items.index(read_more_block_id)
            items.pop(read_more_index)
            items.append(read_more_block_id) # insert as last one
            context.blocks_layout['items'] = items

    context._p_changed = True

def fix_images_in_slate(content):
    # for links like "resolveuid/231312"
    pass

def extract_first_column(context):
    # pull out the content from the column, we have a different layout for these pages
    title_block_id = context.blocks_layout['items'][0]
    title_block = context.blocks[title_block_id]

    if len(context.blocks_layout['items']) == 1:
        # no content
        return

    column_block_id = context.blocks_layout['items'][1]
    column_block = context.blocks[column_block_id]

    if not column_block['@type'] == 'columnsBlock':
        return

    second_column_id = column_block['data']['blocks_layout']['items'][1]
    second_column = column_block['data']['blocks'][second_column_id]

    blocks = second_column['blocks']
    blocks[title_block_id] = title_block

    context.blocks = second_column['blocks']
    block_ids = [title_block_id] + second_column['blocks_layout']['items']
    context.blocks_layout = {"items": block_ids}

@inpath('knowledge/tools/urban-ast/')
def fix_uast(context):

    if not context.blocks:  # unmigrated content?
        return

    return extract_first_column(context)

@inpath('knowledge/tools/adaptation-support-tool')
def fix_ast(context):
    return extract_first_column(context)

def fix_layout_size(context):
    url = context.absolute_url(relative=True)

    if are_on_path(url, FULL_PAGE_PATHS):
        return

    if are_in_path(url, AST_PATHS):
        return

    layout_uid = make_uid()
    page_blocks = context.blocks
    page_blocks_layout = context.blocks_layout
    uids = page_blocks_layout['items'] + [layout_uid]
    blocks_layout = {"items":  uids}
    _blocks = {}
    _blocks[layout_uid] = {"@type": "layoutSettings", "layout_size": "narrow_view"}

    for (uid, block) in page_blocks.items():
        _blocks[uid] = block

    context.blocks = _blocks
    context.blocks_layout = blocks_layout
    context._p_changed = True

def fix_field_encoding(context):

    title = context.title
    if isinstance(title, str):
        title = title.decode('utf-8')
        context.title = title

    description = context.description
    if isinstance(description, str):
        description = description.decode('utf-8')
        context.description = description

content_fixers = [fix_field_encoding, fix_images_in_slate,
                  fix_climate_services_toc, fix_tutorial_videos, fix_uast,
                  fix_ast, fix_webinars, fix_read_more]
folder_fixers = [fix_field_encoding, fix_news_archive]

def fix_content(content):
    for fixer in content_fixers:
        fixer(content)

def fix_folder(context):
    for fixer in folder_fixers:
        fixer(context)

languages = [lang for lang in LANGUAGES if lang != 'en']

def getpath(obj):
    return "/" + obj.absolute_url(relative=1)

def exclude(obj):
    obj.exclude_from_nav = True
    obj.reindexObject()     # update_metadata=True - only on p6

def include(obj):
    obj.exclude_from_nav = False
    obj.reindexObject()     # update_metadata=True - only on p6

def is_top_level_path(path):
    for test_path in TOP_LEVEL:
        if path == test_path or path == '/cca' + test_path:
            return True
    return False

def exclude_content_from_navigation(site):
    main = site.restrictedTraverse('en')
    for oid, obj in main.contentItems():
        path = getpath(obj)
        if not is_top_level_path(path):
            exclude(obj)

            logger.debug("Excluded from nav: %s", path)

            intl_mgr = get_translation_manager(obj)
            for lang in languages:
                trans = intl_mgr.get_translation(lang)

                if trans is None:
                    continue

                exclude(trans)
                logger.debug("Excluded from nav: %s", getpath(trans))
        else:
            include(obj)

            # children
            # for child_path in top_level[path]:
            #     child = obj[child_path]
            #     exclude(child)

site_fixers = [
    exclude_content_from_navigation
]

def fix_site(site):
    for fixer in site_fixers:
        fixer(site)


# GRADIENT BOOSTING (XGBOOST)

from base_model import BaseModel

import numpy as np
from sklearn import metrics
from xgboost import XGBClassifier

class Model (BaseModel):
	def __init__ (self):
		super().__init__('XGB')

	def train (self, Xt, Xv, Yt, Yv, verbose=False):
		model = XGBClassifier(verbose=verbose)
		model.fit(Xt, Yt)
		if verbose: print('fit model')

		pred = model.predict(Xv)
		acc = metrics.accuracy_score(Yv, pred)
		if verbose: print('Acc: ' + str(acc))

		self.src = model
		self.acc = acc

	def predict (self, X):
		return self.onehot_from_cat(self.src.predict(X))


import dbus, re

def fetch_metadata():
    try:
        bus = dbus.SessionBus()
        spotify_bus = bus.get_object("org.mpris.MediaPlayer2.spotify", "/org/mpris/MediaPlayer2")
        spotify_properties = dbus.Interface(spotify_bus, "org.freedesktop.DBus.Properties")
        metadata = spotify_properties.Get("org.mpris.MediaPlayer2.Player", "Metadata")

        artist = metadata['xesam:artist'][0]
        title = metadata['xesam:title']

        return artist, re.split("[,\\!-(?:]+", title)[0]

    except Exception as e:
        print(e)


from tqdm import tqdm
import os
import numpy as np
import librosa
from . import preprocess as pps
from .vad_utils import read_label_from_file, prediction_to_vad_label

def getmelFeature(frameData, sampleRate=16000,
                   frame_size: float=0.032, frame_shift: float=0.008,
                   NFFT=512, NMELS=40):
    '''
    Input:
        frameData(shape=(frameSize, frameNum))
    Return: melData(shape=(frameNum, NMELS))
    '''
    fftData = np.fft.rfft(frameData.T)
    periodogram = np.abs(fftData) ** 2 / NFFT
    melFBank = librosa.filters.mel(sr=sampleRate, n_fft=NFFT, n_mels=NMELS)
    melData = np.dot(periodogram, melFBank.T)
    return melData

def getMFCC(melData, n_mfcc=40):
    '''
    Input:
                            shape=(NMELS, frameNum)
    return: mfcc(np.ndarray[shape=(n_mfcc, frameNum)])
    '''
    mfcc = librosa.feature.mfcc(S=librosa.power_to_db(melData), n_mfcc=n_mfcc)
    return mfcc

# since the implementation in `preprocess.py` ocupying too much memory
# We use another way to read training data
def spectralData(waveDirPath: str, frame_size: float=0.032, frame_shift: float=0.008,
                 NFFT=512, NMELS=40, NMFCC=20, labelDirPath=None, featType='mfcc'):
    '''Generate data with spectral feature (and its coresponding label)

    Return:
        spectralData: np.ndarray[shape=(frameNum, NFFT/2 + 1)]
            or mfcc with shape=(N_MFCC=40, frameNum)
        label
    '''
    assert featType == 'mfcc' or featType =='mel', 'No such kind of feat{}'.format(featType)

    tmpsavePath = 'tmpData'
    if (featType == 'mfcc'):
        if os.path.exists(tmpsavePath + '/' + featType +'_' + str(NMFCC) +'_specX.npy'):
            trainX = np.load(tmpsavePath + '/' + featType + '_' + str(NMFCC) +'_specX.npy')
            trainY = np.load(tmpsavePath + '/' + 'specY.npy')
            return trainX, trainY
    else:
        if os.path.exists(tmpsavePath + '/' + featType +'_' + str(NMELS)+ '_specX.npy'):
            trainX = np.load(tmpsavePath + '/' + featType + '_' + str(NMELS) +'_specX.npy')
            trainY = np.load(tmpsavePath + '/' + 'specY.npy')
            return trainX, trainY

    wavfiles = os.listdir(waveDirPath)
    specData = np.zeros((1, NMELS ))

    if labelDirPath != None:
        wavelabel = read_label_from_file(labelDirPath, frame_size, frame_shift)
        labelist = []

    for b, fileName in enumerate(tqdm(wavfiles)):
        waveID = fileName.split('.')[0]
        frameData, sampleRate = pps.enframe(waveDirPath + '/' + fileName, frame_size, frame_shift)
        spectralData = getmelFeature(frameData, sampleRate, frame_size, frame_shift, NFFT, NMELS)
        specData = np.concatenate((specData, spectralData), axis=0)

        '''
        if( b % 600 == 599):
            c = int(b / 600)
            np.save("tmpData/{}_mel_".format(c) + str(NMELS) + "_specX.npy", specData[1:])
            specData = np.zeros((1, NMELS ))
        '''

        if labelDirPath != None:
            frameNum = frameData.shape[1]
            label = wavelabel[waveID]
            labelPad = np.pad(label, (0, np.maximum(frameNum - len(label), 0)))[:frameNum]
            labelist.append(labelPad)

    label = np.concatenate(labelist)
    np.save("tmpData/specY.npy", label)
    '''
    specData = np.zeros((1, NMELS ))
    for c in range(ceil(b/600)):
        arr = np.load("tmpdata/{}_mel_".format(c) + str(nmels) + "_specx.npy", specData[1:])
        specData = np.concatenate((specData, arr), axis=0)

    np.save("tmpdata/mel_" + str(nmels) + "_specx.npy", specData)
    '''
    if (featType == 'mel'):
        return specData[1:], label
    else:
        mfcc = getMFCC(specData[1:].T, n_mfcc=NMFCC)
        np.save("tmpData/mfcc_" + str(NMFCC) + "_specX.npy", mfcc)
        return mfcc, label

if __name__ == '__main__':
   specdata, label = spectralData('../data/dev', labelDirPath='../data/dev_label.txt')
   print(specdata.shape, label.shape)


from itertools import accumulate

values = [1] + [int(a) if a[-1].isdigit() else 0 for a in open('data.in').read().split()]
part1, part2 = {}, '\n'
for i, x in enumerate(accumulate(values), start=1):
    part1[i] = i * x
    part2 += '█' if (i - 1) % 40 - x in [-1, 0, 1] else ' '
    if not i % 40:
        part2 += '\n'
print(sum(part1[index] for index in range(20, 221, 40)), *part2)


from aocd.models import Puzzle
from sympy import solve, sympify

from helpers import get_day_and_part, submit_answer

yell_dict = {}

def build_yell_dict(data):
    """Stores the input data into internal data structure for further reference"""
    input_list = data.split("\n")

    for line in input_list:
        line_list = line.split(': ')
        monkey_name = line_list[0]
        yell = line_list[1]

        # convert to int if it is a number
        if yell.isnumeric():
            yell_num = int(yell)
            yell_dict[monkey_name] = yell_num
        else:
            yell_list = yell.split(' ')
            yell_dict[monkey_name] = yell_list

    return yell_dict

def build_equation(monkey_name):
    """A recursive function to build a math expr, by resolving unknown monkey name"""

    # humn is the unknown variable x
    if monkey_name == 'humn':
        return 'x'

    # resolved an unknown monkey name. So it returns its value
    if type(yell_dict[monkey_name]) == int:
        return yell_dict[monkey_name]

    yell_list = yell_dict[monkey_name]
    operand_1 = yell_list[0]
    operator = yell_list[1]
    operand_2 = yell_list[2]

    return f'({build_equation(operand_1)} {operator} {build_equation(operand_2)})'

def main():
    day, part = get_day_and_part(__file__)

    puzzle = Puzzle(year=2022, day=day)
    data = puzzle.input_data

    # solution
    build_yell_dict(data)

    yell_list = yell_dict['root']
    yell_dict['root'] = [yell_list[0], yell_list[2]]
    yell_dict['humn'] = 'x'

    left_str = build_equation(yell_list[0])
    right_str = build_equation(yell_list[2])

    # convert a string to math expr
    left_eq = sympify(left_str)
    right_eq = sympify(right_str)

    expr = left_eq - right_eq

    sol = solve(expr)
    ans = sol[0]

    # submit
    submit_answer(ans, day, part)

if __name__ == '__main__':
    main()


from peewee import Model, SqliteDatabase, TextField

class DataStorage:
    def __init__(self, db_path):
        self.db = SqliteDatabase(db_path)

        class BaseModel(Model):
            class Meta:
                database = self.db

        class Record(BaseModel):
            chat_id = TextField()
            name = TextField()
            sheet_id = TextField()
            range = TextField()
            old_value_hash = TextField()

        self.Record = Record

        self.db.connect()
        self.db.create_tables([self.Record])
        self.db.commit()
        self.db.close()

    def find_by_props(self, chat_id, sheet_id, cell_range):
        """
        :return: name of the notifier if found
                 None if not found
        """
        self.db.connect()
        same_record = self.Record.select().where(
            (self.Record.chat_id == chat_id)
            & (self.Record.sheet_id == sheet_id)
            & (self.Record.range == cell_range)
        )

        res = None if len(same_record) == 0 else same_record.get().name
        self.db.close()
        return res

    def find_by_name(self, chat_id, name):
        """
        :return: True or False
        """
        self.db.connect()
        record = self.Record.select().where(
            (self.Record.chat_id == chat_id) & (self.Record.name == name)
        )
        res = len(record) > 0
        self.db.close()
        return res

    def delete_with_name(self, chat_id, name):
        """
        :return: True   if deleted
                 False  if not found
        """
        self.db.connect()
        record = self.Record.select().where(
            (self.Record.chat_id == chat_id) & (self.Record.name == name)
        )
        if len(record) == 0:
            return False
        else:
            record.get().delete_instance()
            self.db.commit()
            self.db.close()
            return True

    def add_record(self, chat_id, name, sheet_id, cell_range, old_hash_value="---"):
        """
        :return: True if successfully set
                 False otherwise
        """
        self.db.connect()
        # print('old_hash:\t', old_hash_value)
        new_record = self.Record.create(
            chat_id=chat_id,
            name=name,
            sheet_id=sheet_id,
            range=cell_range,
            old_value_hash=old_hash_value,
        )
        new_record.save()
        self.db.commit()
        self.db.close()

    @staticmethod
    def transform_to_dict(records):
        return [
            {
                "chat_id": rec.chat_id,
                "name": rec.name,
                "sheet_id": rec.sheet_id,
                "range": rec.range,
                "old_value_hash": rec.old_value_hash,
            }
            for rec in records
        ]

    def records_by_user(self, chat_id):
        self.db.connect()

        records = self.transform_to_dict(
            self.Record.select().where(self.Record.chat_id == chat_id)
        )
        self.db.close()
        return records

    def get_all_records(self):
        self.db.connect()
        all_records = self.transform_to_dict(self.Record.select())
        self.db.close()
        return all_records


import json
from pathlib import Path
from typing import List, Optional
from zipfile import ZipFile

import liblzfse
import numpy as np
import open3d as o3d
import tqdm
from PIL import Image
from quaternion import as_rotation_matrix, quaternion
from torch.utils.data import Dataset

import sys
sys.path.append('..')

from dataloaders.scannet_200_classes import CLASS_LABELS_200

class MyDataset(Dataset):

    def __init__(self, num, custom_classes: Optional[List[str]] = CLASS_LABELS_200):
        self.image_lis, self.depth_lis, self.world_lis, self.conf_lis = [], [], [], []
        self.num = num
        image = np.load('../data/image000.npy')
        self.image_size = (image.shape[1], image.shape[0])
        if custom_classes:
            self._classes = custom_classes
        else:
            self._classes = CLASS_LABELS_200

        self._id_to_name = {i: x for (i, x) in enumerate(self._classes)}

        for i in range(self.num+1):
            num = '000' + str(self.num)
            num = num[-3:]
            image = np.load('../data/image'+num+'.npy')
            depth = (np.load('../data/depth'+num+'.npy')*0.001).astype(np.uint8)
            world = np.load('../data/world'+num+'.npy')

            self.image_lis.append(image)
            self.depth_lis.append(depth)
            self.world_lis.append(world)
            self.conf_lis.append(np.ones_like(depth))

    def __len__(self):
        return self.num

    def __getitem__(self, idx):
        result = {
            "xyz_position": self.world_lis[idx],
            "rgb": self.image_lis[idx],
            "depth": self.depth_lis[idx],
            "conf": self.conf_lis[idx],
        }

        return result

CUSTOM_LABELS = [
    "kitchen counter",
    "kitchen cabinet",
    "stove",
    "cabinet",
    "bathroom counter",
    "refrigerator",
    "microwave",
    "oven",
    "fireplace",
    "door",
    "sink",
    "furniture",
    "dish rack",
    "dining table",
    "shelf",
    "bar",
    "dishwasher",
    "toaster oven",
    "toaster",
    "mini fridge",
    "soap dish",
    "coffee maker",
    "table",
    "bowl",
    "rack",
    "bulletin board",
    "water cooler",
    "coffee kettle",
    "lamp",
    "plate",
    "window",
    "dustpan",
    "trash bin",
    "ceiling",
    "doorframe",
    "trash can",
    "basket",
    "wall",
    "bottle",
    "broom",
    "bin",
    "paper",
    "storage container",
    "box",
    "tray",
    "whiteboard",
    "decoration",
    "board",
    "cup",
    "windowsill",
    "potted plant",
    "light",
    "machine",
    "fire extinguisher",
    "bag",
    "paper towel roll",
    "chair",
    "book",
    "fire alarm",
    "blinds",
    "crate",
    "tissue box",
    "towel",
    "paper bag",
    "column",
    "fan",
    "object",
    "range hood",
    "plant",
    "structure",
    "poster",
    "mat",
    "water bottle",
    "power outlet",
    "storage bin",
    "radiator",
    "picture",
    "water pitcher",
    "pillar",
    "light switch",
    "bucket",
    "storage organizer",
    "vent",
    "counter",
    "ceiling light",
    "case of water bottles",
    "pipe",
    "scale",
    "recycling bin",
    "clock",
    "sign",
    "folded chair",
    "power strip",
]

dataset = MyDataset(48)

from dataloaders import DeticDenseLabelledDataset

labelled_dataset = DeticDenseLabelledDataset(
    dataset,
    use_extra_classes=False,
    exclude_gt_images=False,
    use_lseg=False,
    subsample_prob=0.01,
    visualize_results=True,
    detic_threshold=0.4,
    visualization_path="detic_labelled_results",
)


from collections import deque

def solution(blocks):
    dic = dict()
    for i in range(len(blocks)):
        for j in range(i + 1):
            dic[(i, j)] = 0
        dic[(i, blocks[i][0])] = blocks[i][1]

    idx = 1
    for i in range(len(blocks)):
        q = deque(list(dic.keys())[idx - i - 1: idx])
        print(q)
        while q:
            x, y = q.popleft()
            if dic[(x, y)] == 0:
                if y - 1 >= 0 and dic[(x, y - 1)]\
                        != 0:
                    dic[(x, y)] = dic[(x - 1, y - 1)] - dic[(x, y - 1)]
                elif y + 1 <= i and dic[(x, y + 1)] != 0:
                    dic[(x, y)] = dic[(x - 1, y)] - dic[(x, y + 1)]
                else:
                    q.append((x, y))

        idx += i + 2

    return list(dic.values())

print(solution([[0, 50], [0, 22], [2, 10], [1, 4], [4, -13]]))