import numpy as np
import init_lattice as lat
import MH_algorithm as MH
import Wolff_algorithm as W
import autocorrelation_functions as acf
import importlib
importlib.reload(MH)
importlib.reload(W)
importlib.reload(lat)
importlib.reload(acf)
# Produces data of internal energy autocorrelation against sweeps and the autocorrelation time for use in the report

# Initialise temperature
T = 2

# Temporary data storage
MH_autocorr_temp = []
MH_sweeps_tau_f_temp = []
Wolff_autocorr_temp = []
Wolff_sweeps_tau_f_temp = []

#Repeat and average
for i in range(5):
    print(i)
    # Reset lattice
    lattice = lat.make_lattice(25,1)
    # Start by burning iterations to equilibrium
    burn = W.Wolff_evolve_and_compute_E(lattice, T**-1, 1, 1000)[0]
    # Evolve with Wolff
    Es, sweeps_Wolff = W.Wolff_evolve_and_compute_E(lattice, T**-1, 1, 1000)
    # Now find autocorrelation
    Wolff_autocorr_temp.append(acf.compute_autocorrelation(Es))
    print('Wolff done')
    # Repeat with MH
    # Reset lattice
    lattice = lat.make_lattice(25,1)
    # Start by burning iterations to equilibrium
    burn = MH.evolve_and_compute_E(lattice, T**-1, 1, 0, 100000)[0]
    # Evolve the lattice with MH
    Es, sweeps_MH = MH.evolve_and_compute_E(lattice,T**-1, 1, 0, 100000)
    # Now find autocorrelation
    MH_autocorr_temp.append(acf.compute_autocorrelation(Es))
    print('MH done')

# Take Averages
MH_autocorr = np.mean(MH_autocorr_temp, axis = 0)
MH_sweeps_tau_f = sweeps_MH[acf.estimate_correlation_time(Es)]
Wolff_autocorr = np.mean(Wolff_autocorr_temp, axis = 0)
Wolff_sweeps_tau_f = sweeps_Wolff[acf.estimate_correlation_time(Es)]

# Save data
np.save('MH_autocorr_evolution_sweeps_E.npy', sweeps_MH)
np.save('MH_autocorr_evolution_autocorr_E.npy', MH_autocorr)
np.save('MH_autocorr_evolution_sweeps_tau_f_E.npy', MH_sweeps_tau_f)
np.save('Wolff_autocorr_evolution_sweeps_E.npy', sweeps_Wolff)
np.save('Wolff_autocorr_evolution_autocorr_E.npy', Wolff_autocorr)
np.save('Wolff_auto_corr_evolution_sweeps_tau_f_E.npy', Wolff_sweeps_tau_f)


#Developed By: Tonumoy Mukherjee

import os
from scipy.io import wavfile
import scipy
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from keras.layers import Conv2D, MaxPool2D, Flatten, LSTM
from keras.layers import Dropout, Dense, TimeDistributed
from keras.models import Sequential
from keras.utils import to_categorical
from keras import optimizers
from sklearn.utils.class_weight import compute_class_weight
from tqdm import tqdm
from python_speech_features import mfcc
import pickle
from keras.callbacks import ModelCheckpoint
from cfg import Config
import random
import theano
from keras.utils import plot_model
#import pdb
from mpl_toolkits.axes_grid1 import make_axes_locatable

def check_data():
    if os.path.isfile(config.p_path):
        print('Loading exixting data for {} model' .format(config.mode))
        with open(config.p_path, 'rb') as handle:
            tmp = pickle.load(handle)
            return tmp
    else:
        return None
#%% Feature Extraction

def build_rand_feat():
    tmp = check_data()
    if tmp:
        return tmp.data[0], tmp.data[1]
    X = []
    y = []
    _min, _max = float('inf'), -float('inf')
    for _ in tqdm(range(n_samples)):
        rand_class = np.random.choice(class_dist.index, p=prob_dist)
        file = np.random.choice(df[df.label==rand_class].index)
        rate, wav = wavfile.read('clean-train/'+file)
        label = df.at[file, 'label']
        rand_index = np.random.randint(0, wav.shape[0]-config.step)
        sample = wav[rand_index:rand_index+config.step]
        X_sample = mfcc(sample, rate,
                        numcep=config.nfeat, nfilt=config.nfilt, nfft=config.nfft)
        _min = min(np.amin(X_sample), _min)
        _max = max(np.amax(X_sample), _max)
        X.append(X_sample)
        y.append(classes.index(label))
    config.min = _min
    config.max = _max
    X, y  = np.array(X), np.array(y)
    X = (X - _min) / (_max - _min)
    if config.mode == 'conv':
        X = X.reshape(X.shape[0], X.shape[1], X.shape[2],1)
    elif config.mode == 'time':
        X = X.reshape(X.shape[0], X.shape[1], X.shape[2])
    y = to_categorical(y, num_classes=2)
    config.data = (X, y)

    with open(config.p_path, 'wb') as handle:
        pickle.dump(config, handle, protocol=2)

    return X,y

#%% CNN Model

def get_conv_model():
    model = Sequential()
    model.add(Conv2D(16, (3, 3), activation='relu', strides=(1, 1),
                     padding='same', input_shape=input_shape))
    #pdb.set_trace()
    model.add(Conv2D(32, (3, 3), activation='relu', strides=(1, 1),

                     padding='same', input_shape=input_shape))
    model.add(Conv2D(64, (3, 3), activation='relu', strides=(1, 1),
                     padding='same', input_shape=input_shape))
    model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1),
                     padding='same', input_shape=input_shape))
    model.add(MaxPool2D((2, 2)))
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dense(64, activation='relu'))
    #model.add(Dense(32, activation='relu'))
    #model.add(Dense(16, activation='relu'))
    model.add(Dense(2, activation='softmax'))
    model.summary()
    #adam = optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-9, amsgrad=False)
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['acc'])
    #keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
    return model

#%% LSTM Model

def get_recurrent_model():
    #shape of data for RNN is (n, time, features)
    model = Sequential()
    model.add(LSTM(128, return_sequences=True, input_shape=input_shape))
    model.add(LSTM(128, return_sequences=True))
    model.add(LSTM(128, return_sequences=True))
    model.add(LSTM(128, return_sequences=True))
    model.add(Dropout(0.5))
    model.add(TimeDistributed(Dense(64, activation='relu')))
    model.add(TimeDistributed(Dense(32, activation='relu')))
    model.add(TimeDistributed(Dense(16, activation='relu')))
    model.add(TimeDistributed(Dense(8, activation='relu')))
    model.add(Flatten())
    model.add(Dense(2, activation='softmax'))
    model.summary()
    #sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    #adam = optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-9, amsgrad=False)
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['acc'])
#    model.contrib.layers.l2_regularizer(
#    scale=1 ,
#    scope=None
#)
    return model

#%% Data Management & Model Selection

df = pd.read_csv('Quake_mod.csv')
df.set_index('fname', inplace=True)

for f in df.index:
    rate, signal = wavfile.read('clean-train/'+f)
    signal =signal[0:int(0.2*rate)] #first 0.2 sec of signal
    df.at[f, 'length'] = signal.shape[0]/rate

classes = list(np.unique(df.label))
class_dist = df.groupby(['label'])['length'].mean()

n_samples = 2 * int(df['length'].sum()/0.1) #10th of a second
prob_dist = class_dist/class_dist.sum()
choices = np.random.choice(class_dist.index, p=prob_dist)

fig, ax = plt.subplots()
ax.set_title('Class Distribution', y=1.08,fontsize='large', fontweight='bold')
ax.pie(class_dist, labels=class_dist.index,autopct='%2.2f%%',
       shadow=False, startangle=90)
ax.axis('equal')
plt.show()

config = Config(mode='conv')

if config.mode == 'conv':
    X, y = build_rand_feat()
    y_flat = np.argmax(y, axis=1)
    input_shape = (X.shape[1], X.shape[2], 1)
    model = get_conv_model()

elif config.mode == 'time':
    X, y = build_rand_feat()
    y_flat = np.argmax(y, axis=1)
    input_shape = (X.shape[1], X.shape[2])
    model = get_recurrent_model()

#%%   Training
class_weight = compute_class_weight('balanced', np.unique(y_flat), y_flat)

checkpoint = ModelCheckpoint(config.model_path, monitor='val_acc', verbose=1, mode='max',
                             save_best_only=True, save_weights_only=False, period=1)

model.fit(X, y, epochs=100, batch_size=32, shuffle=True,
          class_weight=class_weight, validation_split=0.1,
          callbacks=[checkpoint])

model.save(config.model_path)
plot_model(model, to_file='convolutional_neural_network.png')

#%%

#def plot_filters(layer,X,y):
##
##
##    filters = layer.W.get_value()
#    filters, biases = layer.get_weights()
#    fig = plt.figure()
#    for j in range(len (filters)):
#        ax = fig.add_subplot(y,X,j+1)
#        ax.matshow(filters[j][0], cmap = cm.binary)
##
#        plt.xticks(np.array([]))
#        plt.yticks(np.array([]))
#    plt.tight_layout()
#    return plt
##
#plot_filters(model.layers[0],4,4) #first convolution layer filters
##
##%%
#for layer in model.layers:
#	# check for convolutional layer
#	if 'conv' not in layer.name:
#		continue
#	# get filter weights
#	filters, biases = layer.get_weights()
#	print(layer.name, filters.shape)

#%% Apda Cde Img Resize Nearest Neighbour

def my_resize(arr, f):
    newarr = np.ones((arr.shape[0]*f, arr.shape[1]*f, arr.shape[2], arr.shape[3]))
    for k1 in range(arr.shape[2]):
        for k2 in range(arr.shape[3]):
            temp = arr[:, :, k1, k2]
            temp = (temp-np.min(temp))/(np.max(temp)-np.min(temp))
            for i in range(arr.shape[0]):
                for j in range(arr.shape[1]):
                    newarr[i*f:(i+1)*f, j*f:(j+1)*f, k1, k2]=temp[i, j]
    return newarr

def plot_filter(arr, f, padd):
    up_arr = my_resize(arr, f)
    newarr = np.ones((arr.shape[2]*(up_arr.shape[0]+padd), arr.shape[3]*(up_arr.shape[1]+padd)))
    for i in range(arr.shape[2]):
        for j in range(arr.shape[3]):
            newarr[i*up_arr.shape[0]+i*padd:(i+1)*up_arr.shape[0]+i*padd, j*up_arr.shape[0]+j*padd:(j+1)*up_arr.shape[0]+j*padd]= \
            up_arr[:,:,i, j]
    return newarr

#%%    Filter output plots CNN

fig1, (ax1,ax2,ax3,ax4) = plt.subplots(nrows=4 , ncols=1)
ax1.set_title("Layer 1 - 16 Filters")
#ax1.set_xlabel("X-label for axis 1"
filters, bias = model.layers[0].get_weights() #1st layer 16 filters
#filters = filters.reshape(3, 3, 4,4)
#title_obj = plt.title('16 Filters of Layer - 1') #get the title property handler
#plt.getp(title_obj, 'text') #print out the properties of title
out = plot_filter(filters, 8, 1)
ax1.imshow(out, cmap=cm.gray)

filters, bias = model.layers[1].get_weights() #2nd layer 32 filters
out = random.sample(list(plot_filter(filters, 8, 1)),32)
ax2.imshow(out, cmap=cm.gray)
ax2.set_title("Layer 2 - 16 X 32 Filters")

filters, bias = model.layers[2].get_weights() #3rd layer 64 filters
out = random.sample(list(plot_filter(filters, 8, 1)),64)
ax3.imshow(out, cmap=cm.gray)
ax3.set_title("Layer 3 - 32 X 64 Filters")

filters, bias = model.layers[3].get_weights() #4thlayer 128 filters
out = random.sample(list(plot_filter(filters, 8, 1)),128)
ax4.imshow(out, cmap=cm.gray)
ax4.set_title("Layer 4 - 64 X 128 Filters")

#%%

fig2, axs = plt.subplots(nrows=2 , ncols=5)
axs[0,0].imshow(X[1,:,:,0]) #Positive Class I/P
axs[0,0].set_title("Positive Class I/P")
axs[1,0].imshow(X[0,:,:,0]) #Negative Class I/P
axs[1,0].set_title("Negative Class I/P")

axs[0,1].imshow(X[5,:,:,0]) #Positive Class I/P
axs[0,1].set_title("Positive Class I/P")
axs[1,1].imshow(X[6,:,:,0]) #Negative Class I/P
axs[1,1].set_title("Negative Class I/P")

axs[0,2].imshow(X[8,:,:,0]) #Positive Class I/P
axs[0,2].set_title("Positive Class I/P")
axs[1,2].imshow(X[9,:,:,0]) #Negative Class I/P
axs[1,2].set_title("Negative Class I/P")

axs[0,3].imshow(X[20,:,:,0]) #Positive Class I/P
axs[0,3].set_title("Positive Class I/P")
axs[1,3].imshow(X[21,:,:,0]) #Negative Class I/P
axs[1,3].set_title("Negative Class I/P")

axs[0,4].imshow(X[24,:,:,0]) #Positive Class I/P
axs[0,4].set_title("Positive Class I/P")
axs[1,4].imshow(X[25,:,:,0]) #Negative Class I/P
axs[1,4].set_title("Negative Class I/P")

#%%
#from keras import backend as K
#def get_activations(model, layer_idx, X_batch):
#    get_activations = K.function([model.layers[0].input, K.learning_phase()], [model.layers[layer_idx].output,])
#    activations = get_activations([X_batch,0])
#    return activations

# visualizing intermediate layers

#output_layer = model.layers[0].get_output()
#output_fn = theano.function([model.layers[0].get_input()], output_layer)
#
## the input image
#
#input_image=X[1,:,:,0]
#print(input_image.shape)
#
#plt.imshow(input_image[0,:,:,0],cmap ='gray')
#plt.imshow(input_image[0,0,:,0])
#
#
#output_image = output_fn(input_image)
#print(output_image.shape)
#
## Rearrange dimension so we can plot the result
#output_image = np.rollaxis(np.rollaxis(output_image, 3, 1), 3, 1)
#print(output_image.shape)

fig3, axs = plt.subplots(nrows=3 , ncols=5)

filters, bias = model.layers[3].get_weights()

filt1 = filters[:,:,0,0] # 1st filter
filt2 = filters[:,:,0,1] # 2nd filter
filt3 = filters[:,:,0,11] # 3rd filter
filt4 = filters[:,:,0,13] # 4th filter
filt5 = filters[:,:,0,14] # 5th filter

inp1 = X[8,:,:,0] # random input

fst_conv = scipy.signal.convolve2d(inp1, filt1, mode='same', boundary='fill', fillvalue=0) #first filter convolution
fst_conv[fst_conv<0] = 0 #relu

scnd_conv = scipy.signal.convolve2d(inp1, filt2, mode='same', boundary='fill', fillvalue=0) #second filter convolution
scnd_conv[scnd_conv<0] = 0 #relu

thrd_conv = scipy.signal.convolve2d(inp1, filt3, mode='same', boundary='fill', fillvalue=0) #third filter convolution
thrd_conv[thrd_conv<0] = 0 #relu

frth_conv = scipy.signal.convolve2d(inp1, filt4, mode='same', boundary='fill', fillvalue=0) #fourth filter convolution
frth_conv[frth_conv<0] = 0 #relu

ffth_conv = scipy.signal.convolve2d(inp1, filt5, mode='same', boundary='fill', fillvalue=0) #fifth filter convolution
ffth_conv[ffth_conv<0] = 0 #relu

axs[0,0].imshow(filt1, cmap =cm.gray)
axs[0,0].set_title("Layer 1, Filter 1")

axs[0,1].imshow(filt2, cmap =cm.gray)
axs[0,1].set_title("Layer 1, Filter 2")

axs[0,2].imshow(filt3, cmap =cm.gray)
axs[0,2].set_title("Layer 1, Filter 3")

axs[0,3].imshow(filt4, cmap =cm.gray)
axs[0,3].set_title("Layer 1, Filter 4")

axs[0,4].imshow(filt5, cmap =cm.gray)
axs[0,4].set_title("Layer 1, Filter 5")

axs[1,0].imshow(inp1, cmap =cm.gray)
axs[1,1].imshow(inp1, cmap =cm.gray)
axs[1,2].imshow(inp1, cmap =cm.gray)
axs[1,2].set_title("Identical Positive Input to the filters")
axs[1,3].imshow(inp1, cmap =cm.gray)
im5  = axs[1,4].imshow(inp1, cmap =cm.gray)
divider = make_axes_locatable(axs[1,4])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im5, cax=cax, orientation='vertical')

axs[2,0].imshow(fst_conv, cmap =cm.gray)
axs[2,0].set_title("Layer 1, Filter 1 Activation")

axs[2,1].imshow(scnd_conv, cmap =cm.gray)
axs[2,1].set_title("Layer 1, Filter 2 Activation")

axs[2,2].imshow(thrd_conv, cmap =cm.gray)
axs[2,2].set_title("Layer 1, Filter 3 Activation")

axs[2,3].imshow(frth_conv, cmap =cm.gray)
axs[2,3].set_title("Layer 1, Filter 4 Activation")

axs[2,4].imshow(ffth_conv, cmap =cm.gray)
axs[2,4].set_title("Layer 1, Filter 5 Activation")

#plt.imshow(conv, cmap = cm.gray) # activations


import argparse
import csv

class MergeDataset:

    def __call__(self, positive_handle, negative_handle, out_handle, delimiter=",", quote_character='"'):
        csv_writer = csv.writer(out_handle, delimiter=delimiter, quotechar=quote_character)

        # Write positive
        for r in positive_handle:
            csv_writer.writerow([r.strip("\n"), 1])

        # Write negative
        for r in negative_handle:
            csv_writer.writerow([r.strip("\n"), 0])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("positivefile",
                        help="The positive file to merge")

    parser.add_argument("negativefile",
                        help="The negativefile file to merge")

    parser.add_argument("outfile",
                        help="The output file")
    args = parser.parse_args()

    with open(args.positivefile, "r", encoding="latin") as p:
        with open(args.negativefile, "r", encoding="latin") as n:
            with open(args.outfile, "w", encoding="latin") as o:
                MergeDataset()(p, n, o)


from typing import List

import hikari

async def alert(event: hikari.GuildMessageCreateEvent, command: str, config, *args) -> None:
    guild: hikari.GatewayGuild = event.get_guild()
    roles: List[hikari.Role] = guild.get_roles().values()
    for role in roles:
        if role.mention == args[0] and role.name not in config['excluded_roles']:
            for member in guild.get_members().values():
                if role in member.get_roles():
                    await member.user.send(' '.join(args[1:]))


import os
import gym
import joblib
import cv2
import numpy as np
import tensorflow as tf

from collections import deque
from argparse import ArgumentParser
from gym import spaces
from tensorflow.python.training.moving_averages import assign_moving_average

cv2.ocl.setUseOpenCL(False)

try:
    import const
except:
    from . import const

const.DEBUG = 1
# DEBUG_PRINT函数，去除print时只需要将const.DEBUG=0
def DEBUG_PRINT(*kwargs):
    if const.DEBUG:
        print(*kwargs)

def common_arg_parser():

    argparser = ArgumentParser()
    argparser.add_argument(
        '--num_timesteps',
        type=float,
        default=1e8,
        dest='total_steps_num',
        help='the total steps for training')
    argparser.add_argument(
        '--params-file',
        metavar='params_file',
        default='dqn_parameters.json',
        help='path to parameters file.Default=dqn_parameters.json')
    argparser.add_argument(
        '--save-path',
        default="trained_models/",
        metavar="save_path",
        help="directory to save/load trained model. Default= ./trained_models/")
    argparser.add_argument(
        "--load-path",
        default=None,
        metavar='load_path',
        help="directory to load trained model. Default= ./trained_models/carla-dqn-model.ckpt")
    argparser.add_argument(
        '--images-to-disk',
        action='store_true',
        dest='save_images_to_disk',
        help='save images (and Lidar data if active) to disk')
    argparser.add_argument(
        '--gpu-id',
        type=int,
        default=0,
        metavar="GPU_ID",
        help='GPU device ID to use. Default:0')
    argparser.add_argument(
        '--play',
        default=False,
        action='store_true',
        help='play the trained model. Default:False')

    return argparser

class NoopResetEnv(gym.Wrapper):
    '''
    在reset后随机走若干steps, 以保证每次reset 返回的observation不一样
    '''
    def __init__(self, env, noop_max=30):
        """Sample initial states by taking random number of no-ops on reset.
        No-op is assumed to be action 0.
        """
        gym.Wrapper.__init__(self, env)
        self.noop_max = noop_max
        self.override_num_noops = None
        self.noop_action = 0
        assert env.unwrapped.get_action_meanings()[0] == 'NOOP'

    def reset(self, **kwargs):
        """ Do no-op action for a number of steps in [1, noop_max]."""
        self.env.reset(**kwargs)
        if self.override_num_noops is not None:
            noops = self.override_num_noops
        else:
            noops = self.unwrapped.np_random.randint(1, self.noop_max + 1) #pylint: disable=E1101
        assert noops > 0
        obs = None
        for _ in range(noops):
            obs, _, done, _ = self.env.step(self.noop_action)
            if done:
                obs = self.env.reset(**kwargs)
        return obs

    def step(self, ac):
        return self.env.step(ac)

class FireResetEnv(gym.Wrapper):
    '''
    reset 后 agent 必须执行某个step
    '''
    def __init__(self, env):
        """Take action on reset for environments that are fixed until firing."""
        gym.Wrapper.__init__(self, env)
        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'
        assert len(env.unwrapped.get_action_meanings()) >= 3

    def reset(self, **kwargs):
        self.env.reset(**kwargs)
        obs, _, done, _ = self.env.step(1)
        if done:
            self.env.reset(**kwargs)
        obs, _, done, _ = self.env.step(2)
        if done:
            self.env.reset(**kwargs)
        return obs

    def step(self, ac):
        return self.env.step(ac)

class EpisodicLifeEnv(gym.Wrapper):
    def __init__(self, env):
        """Make end-of-life == end-of-episode, but only reset on true game over.
        Done by DeepMind for the DQN and co. since it helps value estimation.
        """
        gym.Wrapper.__init__(self, env)
        self.lives = 0
        self.was_real_done  = True

    def step(self, action):
        obs, reward, done, info = self.env.step(action)
        self.was_real_done = done
        # check current lives, make loss of life terminal,
        # then update lives to handle bonus lives
        lives = self.env.unwrapped.ale.lives()
        if lives < self.lives and lives > 0:
            # for Qbert sometimes we stay in lives == 0 condition for a few frames
            # so it's important to keep lives > 0, so that we only reset once
            # the environment advertises done.
            done = True
        self.lives = lives
        return obs, reward, done, info

    def reset(self, **kwargs):
        """Reset only when lives are exhausted.
        This way all states are still reachable even though lives are episodic,
        and the learner need not know about any of this behind-the-scenes.
        """
        if self.was_real_done:
            obs = self.env.reset(**kwargs)
        else:
            # no-op step to advance from terminal/lost life state
            obs, _, _, _ = self.env.step(0)
        self.lives = self.env.unwrapped.ale.lives()
        return obs

class MaxAndSkipEnv(gym.Wrapper):
    '''
       skip 若干 frames, 并挑选这些frames中max_observation 和 total_reward 返回
    '''
    def __init__(self, env, skip=4, use_image_only_observation=True):
        """Return only every `skip`-th frame"""
        gym.Wrapper.__init__(self, env)
        # most recent raw observations (for max pooling across time steps)
        if use_image_only_observation:
            self._obs_image_buffer = np.zeros((2,)+env.observation_space.shape, dtype=np.uint8)
        else:
            self._obs_image_buffer = np.zeros((2,)+env.observation_space.spaces[0].shape, dtype=np.uint8)
            self._obs_measurement_buffer = np.zeros(env.observation_space.spaces[1].shape, dtype=np.float32)

        self._skip       = skip
        self._use_image_only_obs = use_image_only_observation

    def step(self, action):
        """Repeat action, sum reward, and max over last observations."""
        total_reward = 0.0
        done = None
        for i in range(self._skip):
            obs, reward, done, info = self.env.step(action)

            if i == self._skip - 2:
                if self._use_image_only_obs:
                    self._obs_image_buffer[0] = obs
                else:
                    self._obs_image_buffer[0] = obs[0]

            if i == self._skip - 1:
                if self._use_image_only_obs:
                    self._obs_image_buffer[1] = obs
                else:
                    self._obs_image_buffer[1] = obs[0]
                    self._obs_measurement_buffer = obs[1]

            total_reward += reward
            if done:
                break
        # Note that the observation on the done=True frame
        # doesn't matter
        max_frame = self._obs_image_buffer.max(axis=0)
        if self._use_image_only_obs:
            observation = max_frame
        else:
            observation = (max_frame, self._obs_measurement_buffer)

        return observation, total_reward, done, info

    def reset(self, **kwargs):
        return self.env.reset(**kwargs)

    def reset_env(self, **kwargs):
        return self.env.reset_env(**kwargs)

class ClipRewardEnv(gym.RewardWrapper):
    '''
       将reward 统一裁剪为 -1, 0, +1,
    '''
    def __init__(self, env):
        gym.RewardWrapper.__init__(self, env)

    def reward(self, reward):
        """Bin reward to {+1, 0, -1} by its sign."""
        return np.sign(reward)

class WarpFrame(gym.ObservationWrapper):
    '''
       裁剪 frames(images), 范围,存储格式,大小形状
    '''
    def __init__(self, env, width=84, height=84, grayscale=True):
        """Warp frames to 84x84 as done in the Nature paper and later work."""
        gym.ObservationWrapper.__init__(self, env)
        self.width = width
        self.height = height
        self.grayscale = grayscale
        if self.grayscale:
            self.observation_space = spaces.Box(low=0, high=255,
                shape=(self.height, self.width, 1), dtype=np.uint8)
        else:
            self.observation_space = spaces.Box(low=0, high=255,
                shape=(self.height, self.width, 3), dtype=np.uint8)

    def observation(self, frame):
        if self.grayscale:
            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
        frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_AREA)
        if self.grayscale:
            frame = np.expand_dims(frame, -1)
        return frame

class FrameStack(gym.Wrapper):
    def __init__(self, env, k):
        """Stack k last frames.

        Returns lazy array, which is much more memory efficient.

        See Also
        --------
        baselines.common.atari_wrappers.LazyFrames
        """
        gym.Wrapper.__init__(self, env)
        self.k = k
        self.frames = deque([], maxlen=k)
        shp = env.observation_space.shape
        self.observation_space = spaces.Box(low=0, high=255, shape=(shp[:-1] + (shp[-1] * k,)), dtype=env.observation_space.dtype)

    def reset(self):
        ob = self.env.reset()
        for _ in range(self.k):
            self.frames.append(ob)
        return self._get_ob()

    def step(self, action):
        ob, reward, done, info = self.env.step(action)
        self.frames.append(ob)
        return self._get_ob(), reward, done, info

    def _get_ob(self):
        assert len(self.frames) == self.k
        return LazyFrames(list(self.frames))

class ScaledFloatFrame(gym.ObservationWrapper):
    '''
       不建议使用， 因为返回 float32 类型的observation
    '''
    def __init__(self, env):
        gym.ObservationWrapper.__init__(self, env)
        self.observation_space = gym.spaces.Box(low=0, high=1, shape=env.observation_space.shape, dtype=np.float32)

    def observation(self, observation):
        # careful! This undoes the memory optimization, use
        # with smaller replay buffers only.
        return np.array(observation).astype(np.float32) / 255.0

class LazyFrames(object):
    def __init__(self, frames):
        """This object ensures that common frames between the observations are only stored once.
        It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay
        buffers.

        This object should only be converted to numpy array before being passed to the model.

        You'd not believe how complex the previous solution was."""
        self._frames = frames
        self._out = None

    def _force(self):
        if self._out is None:
            self._out = np.concatenate(self._frames, axis=-1)
            self._frames = None
        return self._out

    def __array__(self, dtype=None):
        out = self._force()
        if dtype is not None:
            out = out.astype(dtype)
        return out

    def __len__(self):
        return len(self._force())

    def __getitem__(self, i):
        return self._force()[i]

class NormalizedEnv(gym.ObservationWrapper):
    '''
       observation 归一化, 返回 float32 类型 observation
    '''
    def __init__(self, env=None):
        gym.ObservationWrapper.__init__(self, env)
        self.state_mean = 0
        self.state_std = 0
        self.alpha = 0.9999
        self.num_steps = 0

    def observation(self, observation):
        self.num_steps += 1
        self.state_mean = self.state_mean * self.alpha + observation.mean() * (1 - self.alpha)
        self.state_std = self.state_std * self.alpha + observation.std() * (1 - self.alpha)
        unbiased_mean = self.state_mean / (1 - pow(self.alpha, self.num_steps))
        unbiased_std = self.state_std / (1 - pow(self.alpha, self.num_steps))
        obs = (observation - unbiased_mean) / (unbiased_std + 1e-8)
        return obs

def make_atari(env_id, timelimit=True):
    # XXX(john): remove timelimit argument after gym is upgraded to allow double wrapping
    env = gym.make(env_id)
    if not timelimit:
        env = env.env
    assert 'NoFrameskip' in env.spec.id
    env = NoopResetEnv(env, noop_max=30)
    env = MaxAndSkipEnv(env, skip=4)
    return env

def wrap_carla(env, episode_life=False, clip_rewards=False, frame_stack=False, scale=False):
    """Configure environment for DeepMind-style Atari.
    """
    if episode_life:
        env = EpisodicLifeEnv(env)
    # if 'FIRE' in env.unwrapped.get_action_meanings():
    #     env = FireResetEnv(env)
    env = WarpFrame(env)
    if scale:
        env = ScaledFloatFrame(env)
    if clip_rewards:
        env = ClipRewardEnv(env)
    if frame_stack:
        env = FrameStack(env, 4)
    return env

def batch_norm(x, train, eps=1e-03, decay=0.99, affine=True, name=None):
    '''

    :param x: input tensor
    :param train: True/False, whether train or not
    :param eps:   epsilon cofficient used in divsion
    :param decay:
    :param affine:
    :param name:
    :return:
    '''
    with tf.variable_scope(name, default_name='BatchNorm2d', reuse=tf.AUTO_REUSE):
        params_shape = [x.shape[-1]]
        moving_mean = tf.get_variable('mean', shape=params_shape, initializer=tf.zeros_initializer, trainable=False)
        moving_variance = tf.get_variable('variance', shape=params_shape, initializer=tf.ones_initializer, trainable=False)

        def mean_var_with_update():
            axises = list(np.arange(len(x.shape) - 1))
            mean, variance = tf.nn.moments(x, axes=axises, name='moments')
            with tf.control_dependencies([assign_moving_average(moving_mean, mean, decay),
                                          assign_moving_average(moving_variance, variance, decay)]):
                return tf.identity(mean), tf.identity(variance)

        mean, variance = tf.cond(train, mean_var_with_update, lambda: (moving_mean, moving_variance))
        if affine:
            beta = tf.get_variable('beta', params_shape, initializer=tf.zeros_initializer)
            gamma = tf.get_variable('gamma', params_shape, initializer=tf.ones_initializer)
            x = tf.nn.batch_normalization(x, mean, variance, beta, gamma, eps)
            print("bn beta name : ", beta.name)
            print("bn gamma name : ", gamma.name)
        else:
            x = tf.nn.batch_normalization(x, mean, variance, None, None, eps)
        return x

def save_variables(save_path, variables=None, sess=None):
    """
     保存模型参数

     :param save_path: the path to the model file
     :param variables: the trainable variables in the graph
     :param sess: the session of the graph
     :return: None
     """
    sess = sess or tf.get_default_session()
    variables = variables or tf.trainable_variables()

    ps = sess.run(variables)
    save_dict = {v.name: value for v, value in zip(variables, ps)}
    dirname = os.path.dirname(save_path)
    if any(dirname):
        os.makedirs(dirname, exist_ok=True)
    joblib.dump(save_dict, save_path)

def load_variables(load_path, variables=None, sess=None):
    """
     加载模型参数

     :param load_path: the path to the model file
     :param variables: the trainable variables in the graph
     :param sess: the session of the graph
     :return: None
     """
    sess = sess or tf.get_default_session()
    variables = variables or tf.trainable_variables()

    loaded_params = joblib.load(os.path.expanduser(load_path))
    restores = []
    if isinstance(loaded_params, list):
        assert len(loaded_params) == len(variables), 'number of variables loaded mismatches len(variables)'
        for d, v in zip(loaded_params, variables):
            restores.append(v.assign(d))
    else:
        for v in variables:
            restores.append(v.assign(loaded_params[v.name]))

    sess.run(restores)

def get_vars(scope):
    '''
       获取命名空间scope内的变量
    :param scope:
    :return:
    '''
    return [x for x in tf.global_variables() if scope in x.name]

def count_vars(scope):
    '''
          返回命名空间scope内变量的个数
    :param scope:
    :return:
    '''
    v = get_vars(scope)
    return sum([np.prod(var.shape.as_list()) for var in v])

def huber_loss(x, delta=1.0):
    """Reference: https://en.wikipedia.org/wiki/Huber_loss"""
    return tf.where(
        tf.abs(x) < delta,
        tf.square(x) * 0.5,
        delta * (tf.abs(x) - 0.5 * delta)
    )


import hls4ml
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import tarfile
import shutil

PARSE = False
data = []
data_path = 'data_pickles/data6.pkl'
saved_dir = os.getcwd()
if PARSE:
    df = pd.read_pickle(data_path)
    os.chdir('/eos/home-n/nghielme/')
    ids = df['ID'].tolist()
    for dir in os.listdir('.'):
        if dir.startswith('enet-results-run'):
            os.chdir(dir)
        else:
            continue
        for model in os.listdir('.'):
            datum = {}
            if model.endswith('.tar.gz') and model[8:-7] not in ids:
                with tarfile.open(model) as tar:
                    subdir_and_files = [
                        tarinfo for tarinfo in tar.getmembers()
                        if tarinfo.name.startswith('hls')
                    ]
                    tar.extractall(members=subdir_and_files)
            else:
                continue

            model = model[8:-7]
            parsed = hls4ml.report.vivado_report.parse_vivado_report(model + '_FIFO_OPT')
            shutil.rmtree(model + '_FIFO_OPT')
            model_info = model.split('_')
            datum['ID'] = model
            datum['Run'] = dir.split('-')[-1]
            datum['Filters'] = int(model_info[1][1:])
            datum['Clock'] = int(model_info[2][3:])
            datum['ReuseFactor'] = int(model_info[3][2:])
            datum['Model'] = 'Clock: ' + str(datum['Clock']) + ' \n RF: ' + str(datum['ReuseFactor'])
            datum['Quantization'] = int(model_info[4][1:])
            datum['Precision'] = model_info[7].replace('-', ',')
            try:
                datum['LUTs%'] = int(round(parsed['ImplementationReport']['TotLUTs%']))
                datum['FFs%'] = int(round(parsed['ImplementationReport']['FFs%']))
                datum['RAM36Bs%'] = int(round(parsed['ImplementationReport']['RAMB36s%']))
                datum['RAM18s%'] = int(round(parsed['ImplementationReport']['RAMB18s%']))
                datum['DSPs%'] = int(round(parsed['ImplementationReport']['DSPs%']))
                datum['WNS'] = parsed['TimingReport']['WNS']
            except KeyError:
                datum['LUTs%'] = 'NA'
                datum['FFs%'] = 'NA'
                datum['RAM36Bs%'] = 'NA'
                datum['RAM18s%'] = 'NA'
                datum['DSPs%'] = 'NA'
                datum['WNS'] = 'NA'
            datum['MaxLatency'] = parsed['CosimReport']['LatencyMax']
            data.append(datum)
        os.chdir('..')
    os.chdir(saved_dir)
    df1 = pd.DataFrame(data)
    list_df = [df, df1]
    res = df.concat(list_df)
    res.to_pickle(data_path)
else:
    df = pd.read_pickle(data_path)

df_na = df[df['LUTs%'] == 'NA']
df_na.to_csv('NA_models.csv')
df = df[df['LUTs%'] != 'NA']

df['Max Latency [ms]'] = df['MaxLatency'] * 1e-5
df['10 x WNS [ns]'] = df['WNS'] * 10
df['Latency Overclock [ms]'] = df['MaxLatency'] * (10 - df['WNS']) * 1e-6
# df.to_csv('dataframe.csv')
ap_fixed_16_6_data = df[df['Precision'] == '16,6']
ap_fixed_8_4_data = df[df['Precision'] == '8,4']

ap_fixed_8_4_data = ap_fixed_8_4_data.sort_values(by=['Clock', 'ReuseFactor'], ascending=True)
ap_fixed_16_6_data = ap_fixed_16_6_data.sort_values(by=['Clock', 'ReuseFactor'], ascending=True)

def print_plot(data, title):
    def pointplot_with_outliers(*args, **kwargs):
        local_data = kwargs.pop('data')
        gt100ms = local_data.copy()
        gt100ms.loc[gt100ms['Max Latency [ms]'] >= 100, 'Max Latency [ms]'] = 100
        gt100ms[['LUTs%', 'FFs%', 'RAM36Bs%', 'RAM18s%', 'DSPs%', '10 x WNS [ns]', 'Latency Overclock [ms]']] = -10
        lt100ms = local_data.copy()
        lt100ms.loc[lt100ms['Max Latency [ms]'] >= 100, 'Max Latency [ms]'] = -10
        gt100ms = gt100ms.melt(id_vars=['Model', 'ReuseFactor', 'Clock', 'Filters', 'Quantization'],
                               value_vars=['LUTs%', 'FFs%', 'RAM36Bs%', 'RAM18s%', 'DSPs%',
                                           'Max Latency [ms]', '10 x WNS [ns]', 'Latency Overclock [ms]'])
        lt100ms = lt100ms.melt(id_vars=['Model', 'ReuseFactor', 'Clock', 'Filters', 'Quantization'],
                               value_vars=['LUTs%', 'FFs%', 'RAM36Bs%', 'RAM18s%', 'DSPs%',
                                           'Max Latency [ms]', '10 x WNS [ns]', 'Latency Overclock [ms]'])
        palette = kwargs['palette']
        if len(gt100ms) > 0:
            kwargs['palette'] = 'dark:brown'
            sns.pointplot(**kwargs, data=gt100ms, markers='x')
            kwargs['palette'] = palette

        sns.pointplot(**kwargs, data=lt100ms)

    sns.set_theme()

    g = sns.FacetGrid(data, col='Filters', row='Quantization', sharex=False, sharey=False, aspect=3.2,
                      ylim=(0, 110))
    g.map_dataframe(pointplot_with_outliers, join=False, x='Model', y='value', hue='variable', palette='tab10')
    g.add_legend()
    g.set_xticklabels(rotation=45)
    g.fig.suptitle(title)
    plt.show()

print_plot(ap_fixed_8_4_data, 'Default Quantization: ap_fixed<8,4>')
print_plot(ap_fixed_16_6_data, 'Default Quantization: ap_fixed<16,6>')


import asyncio
import ssl
from itertools import zip_longest

import click
from aiohttp import TCPConnector
from aiohttp.http import HeadersParser

from hls_get.downloader import HLSDownloader

async def download(links, path, names, coros, headers, timeout, clean_up, verify):
    headers_parser = HeadersParser()
    header_lines = [b'', *(line.encode('latin-1') for line in headers), b'']
    parsed_headers, raw_headers = headers_parser.parse_headers(header_lines)
    kwargs = dict()
    if not verify:
        kwargs['connector'] = TCPConnector(verify_ssl=False)
    for link, name in zip_longest(links, names):
        async with HLSDownloader(
            link, path, name, coros, timeout,
            headers=parsed_headers,
            clean_up=clean_up,
            **kwargs
        ) as downloader:
            await downloader.download(link)
            downloader.on_success()

@click.command(
    help='Download m3u8 links '
         '(like "http://www.example.domain/path/to/index.m3u8#Save name" '
         ' etc.) asynchronously, and merge into mp4 files.'
)
@click.argument('links', nargs=-1, required=True)
@click.option('-P', '--path', default='.', help='Save path')
@click.option('-N', '--names', multiple=True, help='Save name')
@click.option('-C', '--coros', default=5, help='Max coroutines')
@click.option('-H', '--headers', multiple=True, help='Headers parameters like curl\'s')
@click.option('-X', '--timeout', default=0, help='timeout in seconds')
@click.option('-c', '--clean-up', default=True, help='Clean up the cache directory when completed', is_flag=True)
@click.option('--verify', default=True, help='Verify certificate', is_flag=True)
@click.option('-D', '--delay', default=3, help='delay seconds before retrying')
@click.option('-R', '--retry-times', default=10, help='Max retry times')
def main(*args, delay=3, retry_times=10, **kwargs):
    try:
        import uvloop
        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
    except ImportError:
        pass

    loop = asyncio.get_event_loop()
    orig_handler = loop.get_exception_handler()

    def ignore_ssl_error(loop, context):
        if context.get('message') in {'SSL error in data received',
                                      'Fatal error on transport'}:
            # validate we have the right exception, transport and protocol
            exception = context.get('exception')
            if (isinstance(exception, ssl.SSLError) and
                exception.reason == 'KRB5_S_INIT'):
                if loop.get_debug():
                    asyncio.log.logger.debug('Ignoring SSL KRB5_S_INIT error')
                return
        if orig_handler is not None:
            orig_handler(loop, context)
        else:
            loop.default_exception_handler(context)

    loop.set_exception_handler(ignore_ssl_error)
    loop.run_until_complete(download(*args, **kwargs))

if __name__ == '__main__':
    main()


# -*- coding: utf_8 -*-
import sys
import time
import json
import re
import datetime
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

options = Options()

options.add_argument('--headless')

options.add_argument("--disable-infobars")
options.add_argument("--disable-extensions")
options.add_argument('--log-level=OFF')
options.add_argument('--no-sandbox')
options.add_argument('--disable-application-cache')
options.add_argument('--disable-gpu')
options.add_argument('--start-maximized')
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--incognito")
options.add_argument("--verbose")

options.add_argument('--disable-browser-side-navigation')
prefs = {"profile.managed_default_content_settings.images": 2}
options.add_experimental_option("prefs", prefs)

s_dt = input()

e_dt = input()

print("Scraping Date: " + s_dt + " ~ " + e_dt)

driver = webdriver.Chrome(ChromeDriverManager().install(), options = options)

start = datetime.datetime.strptime(s_dt, "%Y-%m-%d")
end = datetime.datetime.strptime(e_dt, "%Y-%m-%d")

date_generated = [start + datetime.timedelta(days=x) for x in range(1, (end-start).days+2)]

start_flag = False

for date in date_generated:
    drange = date.strftime("%Y%m%d")

    main_url = "https://info.jfx.co.jp/jfxphpapl/mnavi/mnavi_SwapPoint.php?stdate=P" + drange

    # print(main_url)

    driver.get(main_url)

    iframe = WebDriverWait(driver, 20).until(
        EC.presence_of_element_located((By.XPATH, "//iframe[@name='SWAPSCREEN']")))

    f1 = driver.find_element(By.XPATH, "//td[@class='f1']")
    dt = f1.text

    dt.replace("<","")
    dt.replace(">","")
    dt = dt.strip()
    real_dt = dt
    dt = ''.join([n for n in dt if n.isdigit()])
    # print(dt)

    if start_flag == False and dt != start.strftime("%Y%m%d"):
    	continue

    start_flag = True

    driver.switch_to.frame(iframe)

    # Getting individual cities url
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    trs = soup.findAll("tr", {"bgcolor" : "white"})

    print("===================================================================================")

    for tr in trs:
    	tds = tr.findAll('td')
    	currency = tds[0].getText()
    	buy = tds[4].getText()
    	sell = tds[5].getText()
    	print(real_dt + "   " + currency + "   " + buy + "   " + sell)


# normal libraries
from inspect import signature  # used in the method eval of the class

import numpy as np
import scipy.stats  # functions of statistics
# other files
from corai_error import Error_type_setter
from scipy.integrate import simps

# my libraries

np.random.seed(124)

# section ######################################################################
#  #############################################################################
# some information

# -------------------------------------------------------------------------------------------------------
# list of the possible kernels:
# fct_top_hat
# fct_plain
# fct_truncnorm
# fct_biweight
#
#

# the functions are correct, they scale and shift the way it is supposed.
# However they are written in the following way : f_t(t_i) = K( t_i - t )

# example of kernels:
# list_of_kernels =
#           [Kernel(fct_top_hat, name="wide top hat", a=-450, b=450),
#            Kernel(fct_top_hat, name="normal top hat", a=-200, b=200),
#            Kernel(fct_truncnorm, name="wide truncnorm", a=-500, b=500, sigma=350),
#            Kernel(fct_truncnorm, name="normal truncnorm", a=-350, b=350, sigma=250)]
# -------------------------------------------------------------------------------------------------------
# the functions only work for positive time. If one input negative times, it messes up the orientation.

# section ######################################################################
#  #############################################################################
# class
class Kernel:
    # kernel is a functor, used for weighting some computations.

    # the evaluation gives back a list of np.array

    # the function should hand in the list of np.arrays non scaled.

    # the parameters of the function (to be called) are gathered before:
    #   the weights do not change inside the estimation process.

    # the name is for identification in plots
    def __init__(self, fct_kernel, name=' no name ', **kwargs):
        self.fct_kernel = fct_kernel
        self.name = name
        self.__dict__.update(kwargs)

    def __repr__(self):
        return f"Function is {repr(self._fct_kernel)} and name {self.name}."

    def __call__(self, T_t, eval_point, T_max, debug=False):
        # getting the length over each dimensions for the kernel.
        shape_T_t = [len(T_t[i]) for i in range(len(T_t))]  # recall each dim has different nb of jumps
        # ans is the kernel evaluated on the jumps
        ans = self._fct_kernel(T_t=T_t, eval_point=eval_point, shape_T_t=shape_T_t,
                               **{k: self.__dict__[k] for k in self.__dict__ if
                                  k in signature(self._fct_kernel).parameters})
        # ans is a list of np arrays. It is normalized such that it is a kernel.
        # then I want to scale every vector.
        # The total integral should be T_max, so I multiply by T_max

        # If it isn't fct plain, then I have to scale.
        if self._fct_kernel.__name__ != 'fct_plain':

            # I want to rescale the results for the kernels that are not covering seen part. For that reason,
            # I compute the integral of the kernel, and scale accordingly.
            tt_integral = [np.linspace(0, T_max, int(5E5))] # in a list to respect the format list of list of T_t.
            yy = self._fct_kernel(T_t=tt_integral, eval_point=eval_point, shape_T_t=[1],
                                  **{k: self.__dict__[k] for k in self.__dict__ if
                                     k in signature(self._fct_kernel).parameters})
            integral = simps(yy[0], tt_integral[0])
            # yy[0] bc function gives back a list of arrays.

            for i in range(len(shape_T_t)):
                ans[i] = ans[i] / integral * T_max
                # *= do not work correctly since the vectors are not the same type (int/float).
                # I also divide by the sum, the vector is normalized, however,
                # possibly we're on the edge and we need to take that into account.

        if debug:
            print(f"inside kernel debug, "
                  f"that's my integral : "
                  f"{np.sum(ans[0][:-1]) * T_max / (len(ans[0]) - 1)}. "
                  f"Name : {self.fct_kernel.__name__}.")
        return ans

    # section ######################################################################
    #  #############################################################################
    # getters setters
    @property
    def fct_kernel(self):
        return self._fct_kernel

    @fct_kernel.setter
    def fct_kernel(self, new_fct_kernel):
        self._fct_kernel = new_fct_kernel

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, new_name):
        if isinstance(new_name, str):
            self._name = new_name
        else:
            raise Error_type_setter(f'Argument is not an string.')

# section ######################################################################
#  #############################################################################
# kernels' functions

def fct_top_hat(T_t, shape_T_t, eval_point, a=-200, b=200):
    output = []
    for i in range(len(shape_T_t)):
        vector = np.array(T_t[i])
        # -1 if x < 0, 0 if x==0, 1 if x > 0.
        output.append(1 / (2 * (b - a)) *
                      (np.sign(vector - eval_point - a) +
                       np.sign(b - vector + eval_point))
                      )
    return output

def fct_plain(T_t, shape_T_t, eval_point):
    # no scaling parameter, would be full to use scaling on plain.
    return [np.full(shape_T_t[i], 1) for i in range(len(shape_T_t))] # full of 1.

def fct_truncnorm(T_t, shape_T_t, eval_point, a=-300, b=300, sigma=200):
    output = []
    for i in range(len(shape_T_t)):
        output.append(scipy.stats.truncnorm.pdf(np.array(T_t[i]), a / sigma, b / sigma,
                                                loc=eval_point, scale=sigma))
    return output

def fct_truncnorm_test(T_t, shape_T_t, eval_point, a=-300, b=300, sigma=200):
    output = []
    i = 0  # for output[i] after, but there shouldn't be any problem.
    for i in range(len(shape_T_t)):
        output.append(2 * scipy.stats.truncnorm.pdf(np.array(T_t[i]), a / sigma, b / sigma,
                                                    loc=eval_point, scale=sigma))
    output[i][T_t[i] < eval_point] = 0
    return output

def fct_biweight(T_t, shape_T_t, eval_point, a=-300, b=300):
    #  if important, I can generalize biweight with function beta.
    #  Thus creating like 4 kernels with one function ( BETA(1), BETA(2)...)
    assert a == -b, "The kernel only accepts symmetrical bounds."
    output = []
    for i in range(len(shape_T_t)):
        xx = (np.array(T_t[i]) - (a + b) / 2 - eval_point) * 2 / (b - a)
        # the correct order is eval_point - T_t,
        # bc we evaluate at eval_point but translated by T_t,
        # if kernel not symmetric a != b, then we also need to translate by the mid of them.
        xx[(xx < -1) | (xx > 1)] = 1
        output.append(15 / 16 * np.power(1 - xx * xx, 2) * 2 / (b - a))
    return output

def fct_epa(T_t, shape_T_t, eval_point, a=-300, b=300):
    assert a == -b, "The kernel only accepts symmetrical bounds."
    output = []
    for i in range(len(shape_T_t)):
        xx = (np.array(T_t[i]) - (a + b) / 2 - eval_point) * 2 / (b - a)
        # the correct order is eval_point - T_t,
        # bc we evaluate at eval_point but translated by T_t,
        # if kernel not symmetric a != b, then we also need to translate by the mid of them.
        xx[(xx < -1) | (xx > 1)] = 1
        output.append(3 / 4 * (1 - xx * xx) * 2 / (b - a))
    return output


import werkzeug

def test_CVE_2019_14806():
    """
    CVE-2019-14806
    high severity
    Vulnerable versions: < 0.15.3
    Patched version: 0.15.3
    https://github.com/advisories/GHSA-gq9m-qvpx-68hc

    Pallets Werkzeug before 0.15.3, when used with Docker,
    has insufficient debugger PIN randomness because
    Docker containers share the same machine id.
    """
    werkzeug_version = tuple(map(int, werkzeug.__version__.split('.')))
    secure_version = (0, 15, 3)

    assert werkzeug_version >= secure_version


from scrapy import Spider
from scrapy.selector import Selector

from stack.items import StackItem

with open(r'C:\Users\amarciniak\AppData\Local\Programs\Python\Python35-32\Scripts\stack\stack\spiders\links.txt') as f:
    linkList = f.read().splitlines()

class StackSpider(Spider):
    name = "stack"
    allowed_domains = ["realcanadiansuperstore.ca"]
    start_urls = linkList

    def parse(self, response):

        name = Selector(response)
        calories = Selector(response)

        item = StackItem()
        item['ItemName'] = name.xpath('//h1/text()').extract()[1].strip(';\n\t ')
        itemTempCal =calories.xpath('//*[@id="nutrition"]/div/div[1]/div/div[1]/div[4]/span[2]/text()').extract()
        item['Length']= len(itemTempCal)
        tempLength = len(itemTempCal)

        item['Calories'] = ('').join(itemTempCal).strip(';\n\t ')

        yield item


#!/usr/bin/env python
# _*_ coding: utf-8 _*_
# @Time : 2021/1/3 21:23
# @Author : mafei0728
# @Version：V 0.1
# @File : bar.py
# @desc :
# 1）准备数据
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

movie_name = ['雷神3：诸神黄昏', '正义联盟', '寻梦环游记']

first_day = [10587.6, 10062.5, 1275.7]
first_weekend = [36224.9, 34479.6, 11830]

x = range(len(movie_name))

# 2）创建画布
plt.figure(figsize=(20, 8), dpi=100)

# 3）绘制柱状图
plt.bar(x, first_day, width=0.2, label="首日票房")
plt.bar([i + 0.2 for i in x], first_weekend, width=0.2, label="首周票房")

# 显示图例
plt.legend()

# 修改x轴刻度显示
plt.xticks([i + 0.1 for i in x], movie_name)

# 4）显示图像
plt.show()


import pandas as pd
import datetime
import pickle
import numpy as np
from sklearn.linear_model import SGDRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.externals import joblib
import random

# Codes des compagnies aériennes et l'équivalent du code interne après numérisation de la feature.
carrier_dict = {'AA':0, 'AS':1, 'B6':2, 'DL':3, 'EV':4, 'F9':5,  'HA':6, 'NK':7, 'OO':8, 'UA':9,  'VX':10,'WN':11}

# Distance entre les destinations
tripDistances=pd.DataFrame()

# codes numérique et codes textuelles des aéroports
airport_codes=pd.DataFrame()

# New Year day, Martin Luther King Jr. Day, Presidents' Day, Memorial Day
# Independence Day, Labor Day, Columbus Day, Veterans Day,
# Thanksgiving, Christmas Day
holidays = [datetime.date(2018, 1, 1),datetime.date(2019, 1, 1), datetime.date(2020, 1, 1),
 datetime.date(2018, 1, 15),datetime.date(2019, 1, 21), datetime.date(2020, 1, 20),
 datetime.date(2018, 2, 19), datetime.date(2019, 2, 18), datetime.date(2020, 2, 17),
 datetime.date(2018, 5, 28), datetime.date(2019, 5, 27), datetime.date(2020, 5, 25),
 datetime.date(2018, 7, 4), datetime.date(2019, 7, 4), datetime.date(2020, 7, 4),
 datetime.date(2018, 9, 3), datetime.date(2019, 9, 2), datetime.date(2020, 9, 7),
 datetime.date(2018,10, 8), datetime.date(2019,10, 14), datetime.date(2020,10, 12),
 datetime.date(2018, 11, 11), datetime.date(2019, 11, 11), datetime.date(2020, 11, 11),
 datetime.date(2018, 11, 22), datetime.date(2019, 11, 28), datetime.date(2020, 11, 26),
 datetime.date(2018, 12, 25), datetime.date(2019, 12, 25), datetime.date(2020, 12, 25)]

# Notre modèle de prédiction sauvegardé dans un fichier
predictionModel = SGDRegressor()

encoder = OneHotEncoder()
scaler = StandardScaler()

error_info = ''

def init(model_file='data/flights_delays_model.pkl', trip_distance_file='data/tripDistance.pkl', airport_code_file='data/airportCodesDF.pkl', encoder_file='data/categ_featuresEncoder.pkl', scaler_file='data/numfeaturesScaler.pkl') :
    global predictionModel, tripDistances, airport_codes,encoder, scaler
    predictionModel = joblib.load(model_file)
    pkl_file = open(trip_distance_file, 'rb')
    tripDistances = pickle.load(pkl_file)
    pkl_file = open(airport_code_file, 'rb')
    airport_codes =  pickle.load(pkl_file)
    encoder = joblib.load(encoder_file)
    scaler = joblib.load(scaler_file)

# Retourne le numéro de semaine correspondant à la date
def getWeekNum(day, month,year) :
    global error_info
    try :
        fl_date =  datetime.date(year, month, day)
        return fl_date.isocalendar()[1]
    except Exception as err:
        error_info += 'Invalid date entered (' + str(day) + '/' + str(month) + '/' + str(year) + ') :' + str(err) + '. '
        raise(err)

# Retourne le jour de la semaine (1 = lundi, ...)
def getWeekDay(day, month,year) :
    global error_info
    try :
        return datetime.date(year, month, day).weekday() + 1
    except Exception as err:
        error_info += 'Invalid date entered (' + str(day) + '/' + str(month) + '/' + str(year) + ') :' + str(err) + '. '
        raise(err)

# retourne le code numérique correspondant au code de la compagnies
def getCarrierCodeNum(unique_carrier_code):
    global error_info
    if  unique_carrier_code in carrier_dict :
        return carrier_dict[unique_carrier_code]
    else :
        error_info += 'Cannot find carrier code (' + unique_carrier_code + '). '
        raise ValueError('Bad carrier code')

# retourne la distance de vols entre 2 aéroports
def getTripDistance(origin_code, destination_code):
    global error_info
    try:
        distance = np.array(float(tripDistances[(tripDistances.ORIGIN == origin_code) &
            (tripDistances.DEST == destination_code)].DISTANCE.drop_duplicates()))
        return distance
    except Exception as err:
        error_info += 'Route was not found in the data. Please try a different nearby city or a new route.'
        raise(err)

# Retourne le code numérique de l'aéoport d'origine (si true) ou destination si false.
def getAirportCodeNum(airport_code, origin=True):
    global error_info
    try :
        if origin :
            return int(airport_codes[airport_codes.AIRPORT_CODE == airport_code].ORIGIN_CODE)
        else :
            return int(airport_codes[airport_codes.AIRPORT_CODE == airport_code].DEST_CODE)
    except Exception as err:
        error_info += 'No airport found with code ' + str(airport_code) + '. '
        raise(err)

# Retourne le nombre de jour à proximité d'un jour férié
def getNumDaysToHoliday(day, month, year):
    if year not in [2018, 2019, 2020] :
        error_info += 'No data found for the year ' + str(year) + '. '
        raise ValueError('Bad year')
    c_date = datetime.date(year, month, day)
    return np.min(np.abs(np.array(c_date) - np.array(holidays))).days

# Utilisation de notre modèle pour prédire le retard éventuel.
def delay_prediction(originCode, destCode,  carrier, day, month, year, dep_hour) :
    global error_info
    error_info=''
    try :
        origin_code_num = getAirportCodeNum(originCode, True)
        dest_code_num = getAirportCodeNum(destCode, False)
        carrier_code_num = carrier_dict[carrier]
        weekday = getWeekDay(day, month, year)
        week_num = getWeekNum(day, month, year)
        hdays = getNumDaysToHoliday(day, month, year)
        distance = getTripDistance(originCode, destCode)

        numerical_values = np.c_[distance, hdays]
        # Scale the features
        numerical_values_scaled = scaler.transform(numerical_values)

        categorical_values = np.zeros(8)
        categorical_values[0] = int(month)
        categorical_values[1] = int(day)
        categorical_values[2] = int(weekday)
        categorical_values[3] = int(week_num)
        categorical_values[4] = int(dep_hour)
        categorical_values[5] = int(carrier_code_num)
        categorical_values[6] = int(origin_code_num)
        categorical_values[7] = int(dest_code_num)

        categorical_values_encoded = encoder.transform([categorical_values]).toarray()
        travel = np.c_[numerical_values_scaled, categorical_values_encoded]

        pred_delay = predictionModel.predict(travel)
        return int(pred_delay[0]),error_info
    except Exception as err:
        print(error_info)
        print ('Prediction error.', err)
        return None, error_info

def test() :
    tcarrier = ['AA', 'AS', 'DL', 'HA', 'UA']
    tday = [1,10, 6, 9, 23, 30, 26, 12, 6, 9]
    tmonth = [1,2, 3, 4, 5, 6, 7, 8, 9, 10,11,12]
    tcode = ['BOS', 'JFK', 'SEA', 'SAN', 'DCA']
    tdep_hour = [1, 2, 4, 7, 9, 12, 10, 15, 14, 17, 19, 20, 21, 22, 23]
    for i in range(1000) :
        origcode = random.choice(tcode)
        destcode = random.choice(tcode)
        carrier = random.choice(tcarrier)
        day = random.choice(tday)
        month = random.choice(tmonth)
        dep_hour = random.choice(tdep_hour)
        d = delay_prediction(origcode, destcode, carrier, day, month, 2018, dep_hour)
        if d is not None :
            if d > 5 :
                print(origcode, destcode,carrier,day, month, dep_hour)
                print("delay", d)
                print("----------")


# coding: utf-8
from __future__ import unicode_literals
from django.db import models
from .utils import get_models_from_file

class DynamicModelManager(models.Manager):
    def __init__(self, model, instance=None):
        super(DynamicModelManager, self).__init__()
        self.model = model
        self.instance = instance

    def get_queryset(self):
        if self.instance is None:
            return super(DynamicModelManager, self).get_queryset()

        _filter = {self.instance._meta.pk.name: self.instance.pk}
        return super(DynamicModelManager, self).get_queryset().filter(**_filter)

class DynamicModelDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance):
        if instance is None:
            return DynamicModelManager(self.model)
        return DynamicModelManager(self.model, instance)

class DynamicModel(object):
    registry = {}

    def contribute_to_class(self, cls, name):
        self.manager_name = name
        models.signals.class_prepared.connect(self.finalize, sender=cls)

    def finalize(self, sender, **kwargs):
        models_dict = get_models_from_file()

        for model in models_dict:
            dynamic_model = self.create_dynamic_model(model)
            descriptor = DynamicModelDescriptor(dynamic_model)
            setattr(sender, self.manager_name, descriptor)

    def create_dynamic_model(self, model=None):
        """
        Create a dynamic model from dict data.
        """
        if not model:
            return None

        attrs = self.get_dynamic_model_fields(model)
        # byte string looks sad
        attrs.update(Meta=type(b'Meta', (), self.get_meta_fields(model)))
        name = b'{}DynamicModel'.format(model['name'].title())
        dynamic_model = type(name, (models.Model,), attrs)
        self.__class__.registry[name] = dynamic_model
        return dynamic_model

    def __contains__(self, module_name):
        return module_name in self.__class__.registry

    def get_dynamic_model(self, module_name):
        return self.__class__.registry.get(module_name, None)

    def get_dynamic_model_fields(self, model=None):
        fields = {
            'id': models.AutoField(primary_key=True),
            '__module__': self.__module__,
            '__unicode__': lambda x: u'#{} - {}'.format(x.id, model['name'])
        }
        fields.update(model['fields'])
        return fields

    def get_meta_fields(self, model=None):
        return {
            'ordering': ('-id',),
            'verbose_name': unicode(model['verbose_name'] if model else 'Name'),
            'verbose_name_plural': unicode(model['verbose_name'] if model else 'Names'),
        }

class Model(models.Model):
    models = DynamicModel()


from flask_wtf import FlaskForm
from wtforms import StringField, TextAreaField, FloatField, IntegerField, FileField, validators

class DishForm(FlaskForm):
    name = StringField('Name', [
        validators.DataRequired(),
        validators.Length(min=2, max=100)
    ])
    description = TextAreaField('Description', [
        validators.Optional(),
        validators.Length(max=500)
    ])
    price = FloatField('Price', [
        validators.DataRequired(),
        validators.NumberRange(min=0)
    ])
    image = FileField('Image', [
        validators.Optional()
    ])
    category_id = IntegerField('Category ID', [
        validators.DataRequired()
    ])


import os

from absl import flags

FLAGS = flags.FLAGS

def get_executable_path(py_binary_name):
  """Returns the executable path of a py_binary.

  This returns the executable path of a py_binary that is in another Bazel
  target's data dependencies.

  On Linux/macOS, the path and __file__ has the same root directory.
  On Windows, bazel builds an .exe file and we need to use the MANIFEST file
  the location the actual binary.

  Args:
    py_binary_name: string, the name of a py_binary that is in another Bazel
        target's data dependencies.

  Raises:
    RuntimeError: Raised when it cannot locate the executable path.
  """

  if os.name == 'nt':
    py_binary_name += '.exe'
    manifest_file = os.path.join(FLAGS.test_srcdir, 'MANIFEST')
    workspace_name = os.environ['TEST_WORKSPACE']
    manifest_entry = '{}/{}'.format(workspace_name, py_binary_name)
    with open(manifest_file, 'r') as manifest_fd:
      for line in manifest_fd:
        tokens = line.strip().split(' ')
        if len(tokens) != 2:
          continue
        if manifest_entry == tokens[0]:
          return tokens[1]
    raise RuntimeError(
        'Cannot locate executable path for {}, MANIFEST file: {}.'.format(
            py_binary_name, manifest_file))
  else:
    # NOTE: __file__ may be .py or .pyc, depending on how the module was
    # loaded and executed.
    path = __file__

    # Use the package name to find the root directory: every dot is
    # a directory, plus one for ourselves.
    for _ in range(__name__.count('.') + 1):
      path = os.path.dirname(path)

    root_directory = path
    return os.path.join(root_directory, py_binary_name)


# -*- coding: utf-8 -*-
import scrapy

class AmazonBooksSpiderSpider(scrapy.Spider):
    name = 'amazon_books_spider'
    # allowed_domains = ['amazon.com']
    start_urls = ['https://www.amazon.com/s?i=stripbooks&bbn=283155&rh=n%3A283155%2Cp_n_publication_date%3A1250226011%2Cp_n_feature_browse-bin%3A618073011&s=review-count-rank&dc&fst=as%3Aoff&qid=1588545134&rnid=618072011&ref=sr_pg_2']

    def parse(self, response):
        print(response)
        all_books = response.xpath('//div[@class="sg-col-20-of-24 s-result-item s-asin sg-col-0-of-12 sg-col-28-of-32 sg-col-16-of-20 sg-col sg-col-32-of-36 sg-col-12-of-16 sg-col-24-of-28"]')
        for book in all_books:
            title = book.xpath('.//h2//span/text()').extract_first()
            author = book.xpath('.//a[@class="a-size-base a-link-normal"]/text()').extract_first()
            rating = book.xpath('.//span[@class="a-icon-alt"]/text()').extract_first()
            vote = book.xpath('.//a[@class="a-link-normal"]/span/text()').extract_first()
            kindle_price = book.xpath('.//span[@class="a-offscreen"]/text()').extract_first()

            yield {
                'title': title,
                'author': author,
                'rating': rating,
                'vote': vote,
                'kindle_price': kindle_price
            }


from rest_framework import serializers
from .models import Quizzes, Question, Answer,Score

class QuizSerializer(serializers.ModelSerializer):

    class Meta:
        model = Quizzes
        fields = [
            'title','id'
        ]
class ScoreSerializer(serializers.ModelSerializer):
    user = serializers.ReadOnlyField(source='owner.username')
    class Meta:
        model = Score
        fields = [
            'quiz',
            'score',
            'user',
        ]
class AnswerSerializer(serializers.ModelSerializer):

    class Meta:

        model = Answer
        fields = [
            'id',
            'answer_text',
            'is_right',
        ]

class RandomQuestionSerializer(serializers.ModelSerializer):

    answer = AnswerSerializer(many=True, read_only=True)

    class Meta:

        model = Question
        fields = [
            'title','answers',
        ]

class QuestionSerializer(serializers.ModelSerializer):

    answers = AnswerSerializer(many=True, read_only=True)
    # quiz = QuizSerializer(read_only=True)

    class Meta:

        model = Question
        fields = [
            'quiz','title','answers',
        ]

class QuestionCreateSerializer(serializers.ModelSerializer):
    answers = AnswerSerializer(many=True)

    class Meta:
        model = Question
        fields = [
            'title','answers',
        ]

    def create(self, validated_data):
        answers_data = validated_data.pop('answers')
        question = Question.objects.create(**validated_data)
        # for answer_data in answers_data:
            # Answer.objects.create(question=question, **answer_data)
        answers.set(answers_data)
        return question

class QuizCreateSerializer(serializers.ModelSerializer):
    question = QuestionCreateSerializer(many=True)

    class Meta:
        model = Quizzes
        fields = [
            'title','question',
        ]

    def create(self, validated_data):
        questions_data = validated_data.pop('question')
        print(questions_data)
        quiz = Quizzes.objects.create(**validated_data)
        for question_data in questions_data:
            Question.objects.create(quiz=quiz, **question_data)
        return quiz


from utils.flask.app import app
from utils.db import Book
from flask import request, jsonify

import json

@app.route('/updatespitslot', methods=['GET', 'POST'])
def upload_spitslotinfo():
    data = json.loads(request.get_data(as_text=True))
    if data['key'] != 'updatespitslot' or 'stu_uuid' not in data.keys() or 'info' not in data.keys():
        return jsonify(
            RetCode=1,
            Message='failed because mismatching info'
        )

    stu_uuid, spit_info, book = data['stu_uuid'], data['info'], Book()
    book.insert_spitslot(stu_uuid, spit_info)
    app.logger.info(f"{stu_uuid} upload spit_info: {spit_info}")
    return jsonify(
        RetCode=0,
        Message='上传吐槽信息成功！'
    )

@app.route('/recentspitslot', methods=['GET', 'POST'])
def get_spitslotinfo():
    data = json.loads(request.get_data(as_text=True))
    book = Book()
    data = book.get_recent_spitslot(spit_num=20)
    return jsonify(
        RetCode=0,
        data=data,
        Message='fetch recent spitslot successfully..!'
    )


import os
import test
import shutil
import unittest
from xml.dom import minidom
from xmp import XMP

class XMPTestCase(unittest.TestCase):
    """Tests for `xmp.py`."""

    def test_decode_tag_size(self):
        """decode_tag_size - Read section size from byte pair"""
        self.assertEqual(XMP.decode_tag_size(b'\x00\xff'), 255)
        self.assertEqual(XMP.decode_tag_size(b'\xff\x00'), 65280)
        self.assertEqual(XMP.decode_tag_size(b'\x00\x00'), 0)
        self.assertEqual(XMP.decode_tag_size(b'\xab\xcd'), 43981)

    def test_encode_tag_size(self):
        """encode_tag_size - Convert section size to byte pair"""
        self.assertEqual(XMP.encode_tag_size(255), b'\x00\xff')
        self.assertEqual(XMP.encode_tag_size(65280), b'\xff\x00')
        self.assertEqual(XMP.encode_tag_size(0), b'\x00\x00')
        self.assertEqual(XMP.encode_tag_size(43981), b'\xab\xcd')

    def test_get_xmp(self):
        """get_xmp - Retrieve existing XMP data from file"""
        self.assertEqual(XMP.get_xmp(test.path('img/test-no-XMP.jpg')), '')
        self.assertTrue(len(XMP.get_xmp(test.path('img/test-XMP.jpg'))) > 0)

    def test_set_xmp(self):
        """set_xmp - Write XMP to file"""
        shutil.copy(test.path('img/test-no-XMP.jpg'), test.path('img/test-no-xmp-temp.jpg'))
        xmp_raw = XMP.get_xmp(test.path('img/test-XMP.jpg'))
        XMP.set_xmp(test.path('img/test-no-xmp-temp.jpg'), xmp_raw)
        self.assertTrue(len(XMP.get_xmp(test.path('img/test-no-xmp-temp.jpg'))) > 0)
        os.remove(test.path('img/test-no-xmp-temp.jpg'))

        shutil.copy(test.path('img/test-XMP.jpg'), test.path('img/test-xmp-temp.jpg'))
        self.assertTrue(len(XMP.get_xmp(test.path('img/test-xmp-temp.jpg'))) > 0)
        XMP.set_xmp(test.path('img/test-xmp-temp.jpg'), XMP.XMP_IDENTIFIER)
        self.assertTrue(XMP.get_xmp(test.path('img/test-xmp-temp.jpg')) == XMP.XMP_IDENTIFIER)
        os.remove(test.path('img/test-xmp-temp.jpg'))

    def test_xmp_to_minidom(self):
        """xmp_to_minidom - Convert raw XMP data to minidom object"""
        xmp_raw = XMP.get_xmp(test.path('img/test-XMP.jpg'))
        xmp_minidom = XMP.xmp_to_minidom(xmp_raw)
        self.assertIsInstance(xmp_minidom, minidom.Document)

        xmp_minidom = XMP.xmp_to_minidom(b'')
        self.assertIsInstance(xmp_minidom, minidom.Document)

    def test_minidom_to_xmp(self):
        """minidom_to_xmp - Convert minidom object into raw XMP data"""
        xmp_raw = XMP.get_xmp(test.path('img/test-XMP.jpg'))
        xmp_minidom = XMP.xmp_to_minidom(xmp_raw)
        xmp_raw = XMP.minidom_to_xmp(xmp_minidom)
        self.assertTrue(XMP.XMP_IDENTIFIER in xmp_raw)
        self.assertTrue(XMP.XMP_PACKET_BEGIN in xmp_raw)
        self.assertTrue(XMP.XMP_PACKET_END in xmp_raw)

        xmp_minidom = XMP.xmp_to_minidom(b'')
        xmp_raw = XMP.minidom_to_xmp(xmp_minidom)
        self.assertTrue(XMP.XMP_IDENTIFIER in xmp_raw)
        self.assertTrue(XMP.XMP_PACKET_BEGIN in xmp_raw)
        self.assertTrue(XMP.XMP_PACKET_END in xmp_raw)

    def test_add_panorama_xmp(self):
        """add_panorama_xmp - Add panorama marker to file XMP"""
        shutil.copy(test.path('img/test-no-XMP.jpg'), test.path('img/test-no-xmp-temp.jpg'))
        XMP.add_panorama_xmp(test.path('img/test-no-xmp-temp.jpg'))
        self.assertTrue(b'GPano' in XMP.get_xmp(test.path('img/test-no-xmp-temp.jpg')))
        os.remove(test.path('img/test-no-xmp-temp.jpg'))

if __name__ == '__main__':
    unittest.main()


import random
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import json

URL = "https://www.luogu.com.cn/training/list"
options = Options()
options.add_argument("--headless") # 无头模式
options.set_preference("permissions.default.image", 2) # 无图模式
profile = FirefoxProfile()
profile.set_preference("permissions.default.frame", 3) # 禁用加载 iframe 的功能 (bilibili嵌套)
options.profile = profile
driver = webdriver.Firefox(options=options)
driver.get(URL)
print("[LOG] 加载索引")
TITLE_XPATH_TEMPLATE = '//*[@id="app"]/div[2]/main/div/div[2]/div/div[1]/div[2]/div[TDNUM]/span[2]/a'
TDID_XPATH_TEMPLATE = '//*[@id="app"]/div[2]/main/div/div[2]/div/div[1]/div[2]/div[TDNUM]/span[1]'
title_elements = list()
titles = list()
tdid_elements = list()
tdids = list()
for i in range(1, 41):
    ele1 = driver.find_element(By.XPATH, TITLE_XPATH_TEMPLATE.replace("TDNUM", str(i)));
    ele2 = driver.find_element(By.XPATH, TDID_XPATH_TEMPLATE.replace("TDNUM", str(i)));
    title_elements.append(ele1)
    tdid_elements.append(ele2)
for title_element in title_elements:
    titles.append(title_element.text)
for tdid_element in tdid_elements:
    tdids.append(tdid_element.text)
print("[LOG] 成功加载索引")
# print(titles)
# print(tdids)
TID_TEMPLATE = '//*[@id="app"]/div[2]/main/div/div[2]/div/div[1]/div[2]/div[TNUM]/span[2]'
cnt = 0
plancfg = list()
descriptions = list()
for tdid in tdids:
    print("[LOG] 加载编号: " + tdid)
    cnt += 1
    tids = list()
    driver.get("https://www.luogu.com.cn/training/" + tdid)
    eleone = driver.find_element(By.XPATH, '//*[@id="app"]/div[2]/main/div/div[2]/section[2]/div/div[2]')
    descriptions.append(eleone.text)
    tab2 = driver.find_element(By.XPATH, '//*[@id="app"]/div[2]/main/div/div[1]/div/ul/li[2]/span')
    tab2.click()
    totalnum_ele = driver.find_element(By.XPATH, '//*[@id="app"]/div[2]/div[1]/div[2]/div[2]/div[1]/div/div[1]/span[2]')
    for i in range(1, int(totalnum_ele.text)):
        tidone = driver.find_element(By.XPATH, TID_TEMPLATE.replace("TNUM", str(i)))
        tid = tidone.text
        tids.append("LG" + tid) # 适配 XJYOJ
    totalinf = dict()
    totalinf = {"_id": cnt, "title": titles[cnt - 1], "requireNids": [], "pids": tids}
    plancfg.append(totalinf)
markdown_description = ""
for i,j in zip(descriptions, titles):
    markdown_description += "\n"
    markdown_description += "## "
    markdown_description += j
    markdown_description += "\n"
    markdown_description += i
jsoncfg = json.dumps(plancfg)
with open('cfg.json', 'w') as file1:
    file1.write(jsoncfg)
with open('description.md', 'w') as file2:
    file2.write(markdown_description)
with codecs.open('cfg.json', 'r', encoding='unicode_escape') as f:
    content = f.read()
with codecs.open('cfg.json', 'w', encoding='utf-8') as f:
    f.write(content)
with open('description.md', 'r') as f:
    content = f.read()
modified_content = content.replace('\n', '  \n')
with open('description.md', 'w') as f:
    f.write(modified_content)
driver.quit()


#!/usr/bin/env python
# coding: utf-8

# # Design of a Multi-Zone VAV System (the Shorter Way)
# ---

# In this notebook the example from the previous notebook **Design of a Multi-Zone VAV System (the Long Way)** is repeated, but now the `VAVSystem` class will be used, which automates the design procedure of a multi-zone VAV system. This class resides in the module `hvac.air_conditioning.vav_system.design`. This class can be used for a multi-zone VAV system with cooling and/or heating, having a preheat coil, a cooling coil, and reheat coils at the entrance of the zones. For winter conditions the air is assumed to be totally dry (i.e. only sensible loads are considered).

# In[1]:

from deps import load_packages
load_packages()

# In[2]:

import jupyter_addons as ja
ja.set_css()

# In[3]:

from hvac import Quantity
from hvac.fluids import HumidAir
from hvac.air_conditioning.vav_system.design import Zone, Season, VAVSystem
from hvac.charts import PsychrometricChart, StatePoint

# In[4]:

Q_ = Quantity

# ## Create Zones with Design Data

# The design data of a zone is bundled in a `Zone` data class. First of all, a zone must have a name. The design data concerning the summer peak design day and the design data concerning the winter peak design day are grouped into two separate instances of the `Season` class. The design data are the sensible and latent heat load of the zone and the desired state of the zone air. The `Season` instance with the design data for the summer peak design day is passed through the `summer` parameter of the `Zone` class constructor. The `Season` instance with the design data of the winter peak design day is passed through the `winter` parameter. Should the VAV system only be used for summer cooling, then the `winter` parameter can be simply omitted.

# ### Zone A

# In[5]:

zone_A = Zone(
    name='zone A',
    summer=Season(
        Q_sen=Q_(224_844, 'Btu / hr'),
        Q_lat=Q_(56_000, 'Btu / hr'),
        zone_air=HumidAir(Tdb=Q_(75, 'degF'), RH=Q_(50, 'pct'))
    ),
    winter=Season(
        Q_sen=Q_(-143_000, 'Btu / hr'),
        Q_lat=Q_(0.0, 'Btu / hr'),
        zone_air=HumidAir(Tdb=Q_(75, 'degF'), RH=Q_(0, 'pct'))
    )
)

# ### Zone B

# In[6]:

zone_B = Zone(
    name='zone B',
    summer=Season(
        Q_sen=Q_(103_308, 'Btu / hr'),
        Q_lat=Q_(20_000, 'Btu / hr'),
        zone_air=HumidAir(Tdb=Q_(75, 'degF'), RH=Q_(50, 'pct'))
    ),
    winter=Season(
        Q_sen=Q_(49_092, 'Btu / hr'),
        Q_lat=Q_(0.0, 'Btu / hr'),
        zone_air=HumidAir(Tdb=Q_(75, 'degF'), RH=Q_(0, 'pct'))
    )
)

# ## Create VAV System

# Besides the design data of the zones, the global design data about the outdoor air in summer and winter and the design volume flow rate of outdoor air ventilation must be specified.

# **Outdoor Air Condition on Summer and Winter Design Day**

# In[7]:

outdoor_air_summer = HumidAir(Tdb=Q_(97, 'degF'), Twb=Q_(76, 'degF'))
outdoor_air_winter = HumidAir(Tdb=Q_(7, 'degF'), RH=Q_(0, 'pct'))

# **Design Volume Flow Rate of Outdoor Air Ventilation**

# In[8]:

V_vent = Q_(2400, 'ft ** 3 / min')

# **Instantiate the `VAVSystem` Class with the Design Data**

# In[9]:

vav_system = VAVSystem(
    zones=[zone_A, zone_B],
    outdoor_air_summer=outdoor_air_summer,
    outdoor_air_winter=outdoor_air_winter,
    V_vent=V_vent
)

# ## COOLING DESIGN DAY

# After instantiation of the `VAVSystem` class, call the method `design_summer`. This method can take a number of keyword arguments:
# - the maximum allowed temperature difference between the supply air temperature and the zone air temperature in order to enable proper mixing of the supply air with the zone air: `dT_supply`
# - the pressure of the supply air fan: `supply_fan_pressure`
# - the efficiency of the supply air fan: `supply_fan_efficiency`
# - heat gain of the supply duct: `supply_duct_heat_gain`
# - the pressure of the return air fan: `return_fan_pressure`
# - the efficiency of the return air fan: `return_fan_efficiency`
# - heat gain of the return duct: `return_duct_gain`
#
# These arguments are not mandatory and can be omitted if they are not known. The supply fan and return fan can only be specified after the volume flow rate of supply and return air have first been determined. As such, the first time the notebook would be executed without values for `supply_fan_pressure`, `supply_fan_efficiency`,...

# In[10]:

summer_results = vav_system.design_summer(
    dT_supply=Q_(20, 'delta_degF'),
    supply_fan_pressure=Q_(3, 'inch_H2O_60F'),
    supply_fan_efficiency=Q_(60, 'pct')
)

# The method `design_summer` returns a dictionary with the results as shown below. These results are all `Quantity` objects.
#
# ```
# results = {
#     'cooling coil load': self.summer.cooling_coil.Q,
#     'sensible cooling coil load': self.summer.cooling_coil.Q_sen,
#     'latent cooling coil load': self.summer.cooling_coil.Q_lat,
#     'supply air volume flow rate': self.summer.V_supply,
#     'return air volume flow rate': self.summer.V_return,
#     'system supply air temperature': self.summer.supply_air.Tdb,
#     'system return air temperature': self.summer.return_air.Tdb
# }
# return results
# ```
#
# To quickly show these results in a notebook you may use the (static) method `show_results_markdown` of the `VAVSystem` instance. For this you need to pass the returned results from `design_summer` together with a dictionary `units` containing the units in which you want the results  to be displayed and the number of decimals behind the decimal point, as is demonstrated below.

# In[11]:

ja.display_list(
    vav_system.show_results_markdown(
        summer_results,
        units={
            'Q': ('Btu / hr', 0),
            'V': ('ft ** 3 / min', 0),
            'K': ('degF', 1)
        }
    )
)

# ### Psychrometric Chart
# The data attributes of the `summer` (and `winter`) attribute of the `VAVSystem` class are all accesible. The code below shows the `__init__` method of the `Summer` subclass of the `VAVSystem` class with all its data attributes. The names of the data attributes should speak for themselves.
#
# ```
# def __init__(self, outdoor_air: HumidAir, V_vent: Quantity, system: 'VAVSystem'):
#     self.outdoor_air = outdoor_air
#     self.m_vent = V_vent * outdoor_air.rho
#     self.system = system  # reference to the instance of the `VAVSystem` parent class
#     self.T_supply: Quantity = Q_(float('nan'), 'degC')
#     self.supply_air: Optional[HumidAir] = None
#     self.m_supply: Quantity = Q_(float('nan'), 'kg /s')
#     self.V_supply: Quantity = Q_(float('nan'), 'kg /s')
#     self.T_cold: Quantity = Q_(float('nan'), 'degC')
#     self.cooled_air: Optional[HumidAir] = None
#     self.m_return: Quantity = Q_(float('nan'), 'kg /s')
#     self.V_return: Quantity = Q_(float('nan'), 'kg /s')
#     self.return_air: Optional[HumidAir] = None
#     self.recirculated_air: Optional[HumidAir] = None
#     self.mixed_air: Optional[HumidAir] = None
#     self.cooling_coil: Optional[AirConditioningProcess] = None
#     self.m_supply_part_load: Quantity = Q_(float('nan'), 'kg /s')
#     self.V_supply_part_load: Quantity = Q_(float('nan'), 'kg /s')
# ```

# Taking the appropriate data attributes, it is possible to draw the pyschrometric chart and plot the air conditioning processes in the VAV system.

# In[12]:

chart = PsychrometricChart(fig_size=(8, 6))
chart.plot_process(
    'mixing_chamber',
    StatePoint(vav_system.summer.outdoor_air.Tdb, vav_system.summer.outdoor_air.W),
    StatePoint(vav_system.summer.return_air.Tdb, vav_system.summer.return_air.W),
    StatePoint(vav_system.summer.mixed_air.Tdb, vav_system.summer.mixed_air.W)
)
chart.plot_process(
    'cooling coil',
    StatePoint(vav_system.summer.mixed_air.Tdb, vav_system.summer.mixed_air.W),
    StatePoint(vav_system.summer.cooled_air.Tdb, vav_system.summer.cooled_air.W)
)
# chart.plot_process(
#     'supply fan',
#     StatePoint(vav_system.summer.cooled_air.Tdb, vav_system.summer.cooled_air.W),
#     StatePoint(vav_system.summer.supply_air.Tdb, vav_system.summer.supply_air.W)
# )
chart.plot_process(
    'zones',
    StatePoint(vav_system.summer.supply_air.Tdb, vav_system.summer.supply_air.W),
    StatePoint(vav_system.summer.return_air.Tdb, vav_system.summer.return_air.W)
)
chart.show()

# ### Zone Info

# The zones, instances of the `Zone` class, are kept in a list inside the `VAVSystem` class. A `Zone` object has two members `summer` and `winter` that refer to an instance of the `Season` dataclass that contains the design data for the zone. From the implementation of the `Season` dataclass, it can be seen which data attributes are available. Again the names of the data attributes should speak for themselves.
#
# ```
# @dataclass
# class Season:
#     Q_sen: Quantity
#     Q_lat: Quantity
#     zone_air: HumidAir
#     m_exhaust: Quantity = Q_(0.0, 'kg / s')
#     m_supply: Optional[Quantity] = field(init=False, default=Q_(float('nan'), 'kg / s'))
#     supply_air: Optional[HumidAir] = field(init=False, default=None)
#     return_air: Optional[HumidAir] = field(init=False, default=None)
#
#     @property
#     def m_return(self) -> Quantity:
#         return self.m_supply - self.m_exhaust
#
#     @property
#     def V_supply(self) -> Quantity:
#         return self.m_supply * self.supply_air.v
#
#
# @dataclass
# class Zone:
#     name: str
#     summer: Optional[Season] = None
#     winter: Optional[Season] = None
#     reheat_coil: Optional[AirConditioningProcess] = field(init=False, default=None)
# ```
#
#

# > **Notes**

# >- Attribute `m_exhaust` may refer to local exhaust of air in a zone.

# >- To get at the resulting air state (in particular air humidity) of a zone, the `return_air` attribute should be used, as the `zone_air` attribute is used to specify the desired zone air state when instantiating the zone.

# In[13]:

ja.display_list([
    f"return air at {zone.name}: {zone.summer.return_air.Tdb.to('degF'):~P.1f} TDB, "
    f"{zone.summer.return_air.RH.to('pct'):~P.0f} RH, "
    f"supply air volume flow rate: {zone.summer.V_supply.to('ft ** 3 / min'):~P.0f}"
    for zone in vav_system.zones
])

# ## HEATING DESIGN DAY

# In[14]:

winter_results = vav_system.design_winter(
    T_supply_max=Q_(105, 'degF'),
    supply_fan_pressure=Q_(3.0, 'inch_H2O_60F'),
    supply_fan_efficiency=Q_(60.0, 'pct')
)

# In[15]:

ja.display_list(
    vav_system.show_results_markdown(
        winter_results,
        units={
            'Q': ('Btu / hr', 0),
            'V': ('ft ** 3 / min', 0),
            'K': ('degF', 1)
        }
    )
)

# In[16]:

ja.display_list([
    f"{zone.name}: supply air temperature = {zone.winter.supply_air.Tdb.to('degF'):~P.1f}, "
    f"reheat load = {zone.reheat_coil.Q_sen.to('Btu / hr'):~P.0f}, "
    f"supply air volume flow rate = {zone.winter.V_supply.to('ft ** 3 / min'):~P.0f}"
    for zone in vav_system.zones
])

# In[ ]:


from collections import deque

class Cell:
    def __init__(self, x: int, y: int):
        self.x = x
        self.y = y

class Node:
    def __init__(self, pt: Cell, dist: int):
        self.pt = pt
        self.dist = dist

def is_valid(r, c, tr, tc):
    return (r >= 0) and (r < tr) and (c >= 0) and (c < tc)

def shortest_path(maze, src, dest, r, c):
    if maze[src.x][src.y] != 0 or maze[dest.x][dest.y] != 0:
        return -1

    visited = [[False for i in range(c)] for j in range(r)]
    visited[src.x][src.y] = True
    q = deque()
    s = Node(src, 0)
    q.append(s)

    while q:
        current = q.popleft()
        pt = current.pt
        if pt.x == dest.x and pt.y == dest.y:
            return current.dist

        for i in [[1, 0], [-1, 0], [0, -1], [0, 1]]:
            row, col = pt.x + i[0], pt.y + i[1]

            if is_valid(row, col, r, c) and maze[row][col] == 0 and not visited[row][col]:
                visited[row][col] = True
                neighbor = Node(Cell(row, col), current.dist + 1)
                q.append(neighbor)

    return -1

def main():
    maze = [[0, 1, 1, 0], [0, 0, 0, 1], [1, 1, 0, 0], [1, 1, 1, 0]]
    source = Cell(0, 0)
    dest = Cell(3, 3)

    dist = shortest_path(maze, source, dest, len(maze), len(maze[0]))

    if dist != -1:
        print("Shortest Path:", dist)
    else:
        print("No path exists")

main()