import PIL as pil
from PIL import Image  # Pas encore de sauvegarde implémentée
from PIL import ImageTk  # Idem
import tkinter as tk
from tkinter.filedialog import askopenfilename
import os

C_H, C_W = 250, 250
C_RAY = C_H/25
check = False  # Variable qui vérifie si un QR est bien chargé
dir_frame = os.path.realpath(os.getcwd()+"\DM IN202\\Exemples\\frame.png")  # Si vide : askopenfile request

def nbrCol(matrice):
    return(len(matrice[0]))

def nbrLig(matrice):
    return len(matrice)

###################################INUTILE#####################################

def saving(matPix, filename):
    # sauvegarde l'image contenue dans matpix dans le fichier filename
	# utiliser une extension png pour que la fonction fonctionne sans perte d'information
    toSave = pil.Image.new(mode="1", size=(nbrCol(matPix), nbrLig(matPix)))
    for i in range(nbrLig(matPix)):
        for j in range(nbrCol(matPix)):
            toSave.putpixel((j, i), matPix[i][j])
    toSave.save(filename)

###############################################################################

def loading(filename):
    # charge le fichier image filename et renvoie une matrice de 0 et de 1 qui représente
    # l'image en noir et blanc
    global dimension_qr
    toLoad = pil.Image.open(filename)
    mat = [[0]*toLoad.size[0] for k in range(toLoad.size[1])]
    dimension_qr = toLoad.size[1], toLoad.size[0]
    for i in range(dimension_qr[0]):
        for j in range(dimension_qr[1]):
            mat[i][j] = 0 if toLoad.getpixel((j, i)) == 0 else 1
    return mat

def affiche_mat(matrice):
    """Permet d'afficher la matrice en argument dans le terminal en deux
    dimensions"""
    label = str()
    for i in matrice:
        if len(label) != 0:
            label += '\n'
        for y in i:
            label += str(y)+' '
    return label

def qr_coin():
    '''Cette fonction ne prend rien en argument et retourne un coin de QR code
    sous forme de matrice ! '''
    coin = []
    for i in range(7):
        coin.append([])
        for j in range(7):
            coin[i].append(0)
    for i in range(5):
        i += 1
        for j in range(5):
            j += 1
            coin[i][j] = 1
    for i in range(3):
        i += 2
        for j in range(3):
            j += 2
            coin[i][j] = 0
    return coin

def mat_rotate(matrice, rotation):
    '''Fonction qui prend en entrée une matrice, la rotation désirée
    et qui rend la matrice retournée.'''
    if rotation == 90:
        for i in range(3):
            matrice = list(reversed(list(zip(*matrice))))  # inspiration lib py
    elif rotation == 180:
        for i in range(2):
            matrice = list(reversed(list(zip(*matrice))))
    elif rotation == -90:
        matrice = list(reversed(list(zip(*matrice))))
    return(matrice)

def dir_image():
    file_dir = askopenfilename(initialdir=os.path.realpath(os.getcwd())+"\DM IN202\\Exemples")
    return file_dir

def load_canvas():
    """Cette fonction affiche la matrice présente dans le programme
    sous le nom de 'mat' dans un canevas (gadget mais panache)"""
    canvas.delete("all")
    for line in range(len(mat)):
        for i in range(len(mat[line])):
            if mat[line][i] == 1:
                canvas.create_rectangle(C_RAY*i, C_RAY*line, C_RAY*i+C_RAY,
                                        C_RAY*line+C_RAY, fill="white")
    canvas.update()
    print("qr code affiché")

def load_image():
    """Charge une image de QR Code, vérifie ses dimensions car 25x25 oblige
    dans le cadre de ce projet."""
    global mat, check
    mat = loading(dir_image())
    print(affiche_mat(mat))
    if dimension_qr[0] == 25 and dimension_qr[1] == 25:
        load_canvas()
        check = True

def check_corner():
    """Vérifie si 'mat' est orientée dans le sens de lecture que
    veut la convention et le corrige sinon."""
    global check, mat
    if check:
        coin_ref = qr_coin()
        coins = [[], [], [], []]
        for i in range(7):
            coins[0].append([])
            coins[1].append([])
            for j in range(7):
                coins[0][i].append(mat[i][j])
            for j in range(18, 25):
                coins[1][i].append(mat[i][j])
        for i in range(18, 25):
            coins[2].append([])
            coins[3].append([])
            for j in range(7):
                coins[2][i-18].append(mat[i][j])
            for j in range(18, 25):
                coins[3][i-18].append(mat[i][j])
        print('\n'+affiche_mat(coins[0])+'UP left corner')
        print('\n'+affiche_mat(coins[1])+'UP right corner')
        print('\n'+affiche_mat(coins[2])+'DOWN left corner')
        print('\n'+affiche_mat(coins[3])+'DOWN right corner')
        print('\n'+affiche_mat(coin_ref)+'Référence\n\n')
        if coins[0] != coin_ref:
            print('UP Left corner should be the DOWN Right')
            mat = mat_rotate(mat, 180)
            print('Correction...')
        elif coins[1] != coin_ref:
            print('UP Right corner should be the DOWN Right')
            mat = mat_rotate(mat, 90)
            print('Correction...')
        elif coins[2] != coin_ref:
            print('DOWN Left corner should be the DOWN Right')
            mat = mat_rotate(mat, -90)
            print('Correction...')
        elif coins[3] != coin_ref:
            print('Le QR code est bien placé')
        load_canvas()
    else:
        canvas.delete("all")

def check_line():
    """Fonction qui vérifie que les lignes qui joignent chaques coins
    sont présentes, sinon erreur"""
    global mat, check
    if check:
        erreur = False
        if dir_frame == '':
            print('Il faut localiser le QR frame.png')
            qr_frame = loading(dir_image())
        else:
            qr_frame = loading(dir_frame)
        print('QR Frame chargé')
        for i in range(6, 18):
            if mat[6][i] != qr_frame[6][i] or mat[i][6] != qr_frame[i][6]:
                erreur = True
        if erreur:
            print('Les lignes de coins du QR Code ne sont pas identifiées')
        else:
            print('Les lignes de coins du QR Code sont identifiées')

def parité(bits: list, indices: list):
    """Fonction de calcul de parité basique qui prend en argument une
    quelconque liste composée en integer puis une autre liste qui correspond
    aux indices de comparaison."""
    out = 0
    for bit in indices:
        out += bits[bit]
    return out % 2

def switch_bin(binary: int):
    """return le bit opposé de celui en entrée"""
    if binary == 0:
        binary = 1
    else:
        binary = 0
    return binary

def hamming(bits):
    '''Prend 7 bits en entrée et rend 4 bits de données
    (C'est un code de correction d'erreur de Hamming)'''
    c1, c2, c3 = parité(bits, [0, 1, 3]), parité(bits, [0, 2, 3]), parité(bits, [2, 1, 3])
    if c1 != bits[-3] and c2 != bits[-2] and c3 != bits[-1]:
        bits[3] = switch_bin(bits[3])
    elif c2 != bits[-2] and c3 != bits[-1]:
        bits[2] = switch_bin(bits[2])
    elif c1 != bits[-3] and c3 != bits[-1]:
        bits[1] = switch_bin(bits[1])
    elif c1 != bits[-3] and c2 != bits[-2]:
        bits[0] = switch_bin(bits[0])
    return [bits[0], bits[1], bits[2], bits[3]]

def sorting_qr():
    '''Cette fonction lit une matrice de 25x25 représentant
    un QR code correctement orienté afin de retourner
    les blocs de lecture binaire associés'''
    global mat
    liste_binaire = []
    blocs = []
    size = 24
    for zigzag in range(4):
        zigzag += 1
        liste_binaire.append([])
        for i in range(14):
            x = 24-i
            for j in range(2):
                y = size-j
                liste_binaire[(zigzag*2)-2].append(mat[y][x])
        size -= 2
        liste_binaire.append([])
        for i in range(14):
            x = 11+i
            for j in range(2):
                y = size-j
                liste_binaire[(zigzag*2)-1].append(mat[y][x])
        size -= 2
    for ligne in liste_binaire:
        if ligne[0:len(ligne)//2].count(1) != 14:
            blocs.append(ligne[0:len(ligne)//2])
        if ligne[len(ligne)//2:len(ligne)].count(1) != 14:
            blocs.append(ligne[len(ligne)//2:len(ligne)])
    return blocs

def blocs_to_hamming(blocs):
    """Prend chaque bloc à part puis le décompose en deux sous blocs
    qui sont passés en Hamming, on obtient donc 8 bits de données sous forme
    de liste"""
    output = []
    for bloc in blocs:
        output.append(hamming(bloc[:7]) + hamming(bloc[7:]))
    return output

def bits_to_ascii(bits: list):
    '''Prend du binaire en argument (list) et retourne
    le character ASCII associé'''
    value = str()
    for i in bits:
        value += str(i)
    print(value)
    value = chr(int(value, 2))
    print(value)
    return value

def bits_to_hex(bits: list):
    """Prend une liste binaire en argument pour
    retourner l'hexadécimal associé"""
    value = str()
    for i in bits:
        value += str(i)
    print(value)
    return hex(int(value, 2))[2:]

def check_filter():
    '''Identifie le type de filtre appliqué au QR Code
    pour sa lisibilité puis retire de la matrice le filtre.
    \nLe paramètre du filtre dépend de 2 bits de la matrice:
    (23,8) et (22,8). \n\nSi "00" : entièrement noire (pas de changement).
    \nSi "01" : damier dont la case en haut à gauche est noire.
    \nSi "10" des lignes horizontales alternées noires et blanches,
    la plus haute étant noire.
    \nSi "11" des lignes verticales alternées noires et blanches,
    la plus à gauche étant noire.'''
    global mat
    filtre = str(mat[22][8]) + str(mat[23][8])
    xor_bool = False
    if filtre == '01':
        print("Filtre : Damier")
        for i in range(len(mat)):
            for j in range(len(mat[i])):
                if xor_bool:
                    if mat[i][j] == 0:
                        mat[i][j] = 1
                    else:
                        mat[i][j] = 0
                    xor_bool = False
                else:
                    xor_bool = True
        blocs = sorting_qr()
    elif filtre == '10':
        print("Filtre : Lignes horizontales")
        for i in range(len(mat)):
            if xor_bool:
                for j in range(len(mat[i])):
                    if mat[i][j] == 0:
                        mat[i][j] = 1
                    else:
                        mat[i][j] = 0
            else:
                xor_bool = True
        blocs = sorting_qr()
    elif filtre == '11':
        print("Filtre : Lignes Verticales")
        mat = mat_rotate(mat, 90)
        for i in range(len(mat)):
            if xor_bool:
                for j in range(len(mat[i])):
                    if mat[i][j] == 0:
                        mat[i][j] = 1
                    else:
                        mat[i][j] = 0
            else:
                xor_bool = True
        mat = mat_rotate(mat, -90)
        blocs = sorting_qr()
    else:
        print("Pas de Filtre")
        blocs = sorting_qr()
    return blocs

def check_qr():
    """Fonction maîtresse liée au check & process, c'est elle
    qui exécute la majorité des fonctions du programme,
    d'où l'appellation de son bouton"""
    if check:
        check_corner()
        check_line()
        blocs = check_filter()
        blocs = blocs_to_hamming(blocs)
        if mat[24][8] == 1:  # ASCII
            output = str()
            for bloc in blocs:
                output += bits_to_ascii(bloc)
            print('Contenu du QR Code : ', output)
        else:
            output = str()
            for bloc in blocs:
                output += bits_to_hex(bloc)
            print("Sortie Hexadécimale : ", output)
    else:
        canvas.delete("all")

# Initialisation du GUI

root = tk.Tk()
root.title('QR Code')
root.geometry('420x490')

# Ajout de fonctionnalités

button_charger = tk.Button(root, text='Charger', command=load_image, bg='gray')
button_check_qr = tk.Button(root, text='Check & Process', command=check_qr,
                            bg='gray')
canvas = tk.Canvas(root, width=C_W, height=C_H, bg="black")

button_charger.pack(side=tk.BOTTOM, ipadx=150, ipady=6)
button_check_qr.pack(side=tk.BOTTOM, ipadx=70)
canvas.pack(pady=20)

# Maintien de la fenêtre

root.mainloop()


from setuptools import setup, Extension
from torch.utils import cpp_extension

torch_library_paths = cpp_extension.library_paths(cuda=False)

setup(name='rephine_mt',
      ext_modules=[cpp_extension.CppExtension('rephine_mt', ['ph/rephine_mt_cpu.cpp'], extra_link_args=[
                    '-Wl,-rpath,' + library_path
                    for library_path in torch_library_paths])],
      cmdclass={'build_ext': cpp_extension.BuildExtension})


from skfuzzy.cluster import cmeans
import matplotlib.pyplot as plt
from sklearn import datasets
import numpy as np
import pandas as pd
import seaborn as sns

def process_data(data, centers):
    cntr, u, u0, d, jm, p, fpc = cmeans(data, centers, 2, error=0.005, maxiter=10000, init=None)
    cluster_membership = np.argmax(u, axis=0)

    data = pd.DataFrame(data=data)
    data['target'] = cluster_membership

    g = sns.FacetGrid(data, hue='target', palette='tab20', size=5)
    g.map(plt.scatter, 0, 1, s=100, linewidth=.5, edgecolor='white')
    g.add_legend()
    # for i in range(centers):
    #     plt.scatter(data[cluster_membership == i])
    # for pt in cntr:
    #     plt.plot(pt[0], pt[1], 'rs')
    plt.show()

def main():
    n_samples = 10000
    random_state = 170
    transformation = [[0.6, -0.6], [-0.4, 0.8]]

    models = [
        {
            'name': 'Far Blobs',
            'X':
                datasets.make_blobs(n_samples=n_samples, centers=25, random_state=0, center_box=(-10000, 10000),
                                    cluster_std=50)[0],
            'k': 25
        },
        {
            'name': 'Noisy Circles',
            'X':
                datasets.make_circles(n_samples=n_samples, factor=.5, noise=.05)[0],
            'k': 2
        },
        {
            'name': 'Noisy Moons',
            'X': datasets.make_moons(n_samples=n_samples, noise=.05)[0],
            'k': 2
        },
        {
            'name': 'Blobs',
            'X': datasets.make_blobs(n_samples=n_samples, random_state=8)[0],
            'k': 3
        },
        {
            'name': 'No structure',
            'X': np.random.rand(n_samples, 2),
            'k': 2
        },
        {
            'name': 'Anisotropicly distributed data',
            'X': np.dot(datasets.make_blobs(n_samples=n_samples, random_state=random_state)[0], transformation),
            'k': 2
        },
        {
            'name': 'Blobs with varied variances',
            'X': datasets.make_blobs(n_samples=n_samples,
                                     cluster_std=[1.0, 2.5, 0.5],
                                     random_state=random_state)[0],
            'k': 2
        },
        {
            'name': '2 features, 1 informative, 1 cluster',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=1,
                                              n_clusters_per_class=1)[0],
            'k': 2
        },
        {
            'name': '2 features, 2 informative, 1 cluster',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2,
                                              n_clusters_per_class=1)[0],
            'k': 3
        },
        {
            'name': '2 features, 2 informative',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2)[0],
            'k': 2
        },
        {
            'name': '2 features, 2 informative, 2 cluster, 3 classes',
            'X': datasets.make_classification(n_samples=500, n_features=2, n_redundant=0, n_informative=2,
                                              n_clusters_per_class=1, n_classes=3)[0],
            'k': 3
        },
        {
            'name': '2 features, 5 centers',
            'X': datasets.make_blobs(n_samples=500, n_features=2, centers=5)[0],
            'k': 4,
            'dist_metric': 'manhattan'
        },
        {
            'name': '2 features, 6 classes',
            'X': datasets.make_gaussian_quantiles(n_samples=500, n_features=2, n_classes=6)[0],
            'k': 6
        },
        {
            'name': 'Circles',
            'X': datasets.make_circles(n_samples=500, factor=0.5)[0],
            'k': 4,
            'dist_metric': 'manhattan'
        },
    ]

    for model in models:
        process_data(model['X'], model['k'])

if __name__ == '__main__':
    main()


from celery.task import task
from numpy import mean, std, sqrt, fabs
from pandas import DataFrame

@task
def task_cohen(dataOne, dataTwo):

    labelOne = 'column1'
    labelTwo = 'column2'

    # Create input
    df = DataFrame.from_items([(labelOne, dataOne), (labelTwo, dataTwo)])

    meanx = mean(df[labelOne])
    meany = mean(df[labelTwo])
    sdx = std(df[labelOne])
    sdy = std(df[labelTwo])
    s = sqrt((sdx**2 + sdy**2)/2)
    d = round(fabs(meanx-meany)/s, 4)

    return d


import torch
import os
import numpy as np
from torch.utils.data import DataLoader
from post_process.post_data import Post_2d
from Demo.Rotor37_2d.utilizes_rotor37 import get_grid, get_origin
from Utilizes.process_data import DataNormer
import yaml
from utilizes_rotor37 import get_gemodata_from_mat,get_origin_gemo

def get_noise(shape, scale):
    random_array = np.random.randn(np.prod(shape)) #  randn生成的是标准正态分布
    # random_array = (random_array-1)*2
    random_array = random_array.reshape(shape)

    return random_array * scale

def loaddata_Sql(name,
             ntrain=2500,
             nvalid=400,
             shuffled=False,
             noise_scale=None,
             batch_size=32):

    design, fields = get_origin_gemo(realpath=os.path.join("..", "data"),shuffled=shuffled) # 获取原始数据
    if name in ("FNO", "FNM", "UNet", "Transformer"):
        input = np.tile(design[:, None, None, :], (1, 64, 64, 1))
        r1 = 10
        input = design[:, :, ::r1, :]
        input = input.reshape([input.shape[0], -1])  # 二维数据
        input = np.tile(input[:, None, None, :], (1, 64, 64, 1))
    else:
        input = design
    output = fields

    # input = torch.tensor(input, dtype=torch.float)
    # output = torch.tensor(output, dtype=torch.float)
    print(input.shape, output.shape)

    train_x = input[:ntrain, :]
    train_y = output[:ntrain, :]
    valid_x = input[-nvalid:, :]
    valid_y = output[-nvalid:, :]

    x_normalizer = DataNormer(train_x, method='mean-std')
    train_x = x_normalizer.norm(train_x)
    valid_x = x_normalizer.norm(valid_x)

    y_normalizer = DataNormer(train_y, method='mean-std')
    train_y = y_normalizer.norm(train_y)
    valid_y = y_normalizer.norm(valid_y)

    if name in ("MLP"):
        train_y = train_y.reshape([train_y.shape[0], -1])
        valid_y = valid_y.reshape([valid_y.shape[0], -1])

    if noise_scale is not None and noise_scale > 0: # 向数据中增加噪声
        noise_train = get_noise(train_y.shape, noise_scale)
        train_y = train_y + noise_train

    # 完成了归一化后再转换数据
    train_x = torch.tensor(train_x, dtype=torch.float)
    train_y = torch.tensor(train_y, dtype=torch.float)
    valid_x = torch.tensor(valid_x, dtype=torch.float)
    valid_y = torch.tensor(valid_y, dtype=torch.float)

    if name in ("deepONet"):
        grid = get_grid(real_path=os.path.join("..", "data"))
        grid_trans = torch.tensor(grid[np.newaxis, :, :, :], dtype=torch.float)
        train_grid = torch.tile(grid_trans, [train_x.shape[0], 1, 1, 1])  # 所有样本的坐标是一致的。
        valid_grid = torch.tile(grid_trans, [valid_x.shape[0], 1, 1, 1])

        grid_normalizer = DataNormer(train_grid.numpy(), method='mean-std')  # 这里的axis不一样了
        train_grid = grid_normalizer.norm(train_grid)
        valid_grid = grid_normalizer.norm(valid_grid)

        # grid_trans = grid_trans.reshape([1, -1, 2])
        train_grid = train_grid.reshape([train_x.shape[0], -1, 2])
        valid_grid = valid_grid.reshape([valid_x.shape[0], -1, 2])
        train_y = train_y.reshape([train_y.shape[0], -1, 5])
        valid_y = valid_y.reshape([valid_y.shape[0], -1, 5])

        train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(train_x, train_grid, train_y),
                                                   batch_size=batch_size, shuffle=True, drop_last=True)
        valid_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(valid_x, valid_grid, valid_y),
                                                   batch_size=batch_size, shuffle=False, drop_last=True)
    else:
        train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(train_x, train_y),
                                                   batch_size=batch_size, shuffle=True, drop_last=True)
        valid_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(valid_x, valid_y),
                                                   batch_size=batch_size, shuffle=False, drop_last=True)

    return train_loader, valid_loader, x_normalizer, y_normalizer

def loaddata(name,
             ntrain=2500,
             nvalid=400,
             shuffled=False,
             noise_scale=None,
             x_norm=None,
             y_norm=None,
             batch_size=32):

    design, fields = get_origin(realpath=os.path.join("..", "data"), shuffled=shuffled)  # 获取原始数据

    if name in ("FNO", "FNM", "UNet"):
        input = np.tile(design[:, None, None, :], (1, 64, 64, 1))
    else:
        input = design
    output = fields

    # input = torch.tensor(input, dtype=torch.float)
    # output = torch.tensor(output, dtype=torch.float)
    print(input.shape, output.shape)

    train_x = input[:ntrain, :]
    train_y = output[:ntrain, :]
    valid_x = input[-nvalid:, :]
    valid_y = output[-nvalid:, :]

    if x_norm is None:
        x_normalizer = DataNormer(train_x, method='mean-std')
    else:
        x_normalizer = x_norm

    if y_norm is None:
        y_normalizer = DataNormer(train_y, method='mean-std')
    else:
        y_normalizer = y_norm

    train_x = x_normalizer.norm(train_x)
    valid_x = x_normalizer.norm(valid_x)

    train_y = y_normalizer.norm(train_y)
    valid_y = y_normalizer.norm(valid_y)

    if name in ("MLP"):
        train_y = train_y.reshape([train_y.shape[0], -1])
        valid_y = valid_y.reshape([valid_y.shape[0], -1])

    if noise_scale is not None and noise_scale > 0: # 向数据中增加噪声
        noise_train = get_noise(train_y.shape, noise_scale)
        train_y = train_y + noise_train

    # 完成了归一化后再转换数据
    train_x = torch.tensor(train_x, dtype=torch.float)
    train_y = torch.tensor(train_y, dtype=torch.float)
    valid_x = torch.tensor(valid_x, dtype=torch.float)
    valid_y = torch.tensor(valid_y, dtype=torch.float)

    if name in ("deepONet"):
        grid = get_grid(real_path=os.path.join("..", "data"))
        grid_trans = torch.tensor(grid[np.newaxis, :, :, :], dtype=torch.float)
        train_grid = torch.tile(grid_trans, [train_x.shape[0], 1, 1, 1])  # 所有样本的坐标是一致的。
        valid_grid = torch.tile(grid_trans, [valid_x.shape[0], 1, 1, 1])

        grid_normalizer = DataNormer(train_grid.numpy(), method='mean-std')  # 这里的axis不一样了
        train_grid = grid_normalizer.norm(train_grid)
        valid_grid = grid_normalizer.norm(valid_grid)

        # grid_trans = grid_trans.reshape([1, -1, 2])
        train_grid = train_grid.reshape([train_x.shape[0], -1, 2])
        valid_grid = valid_grid.reshape([valid_x.shape[0], -1, 2])
        train_y = train_y.reshape([train_y.shape[0], -1, 5])
        valid_y = valid_y.reshape([valid_y.shape[0], -1, 5])

        train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(train_x, train_grid, train_y),
                                                   batch_size=batch_size, shuffle=True, drop_last=True)
        valid_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(valid_x, valid_grid, valid_y),
                                                   batch_size=batch_size, shuffle=False, drop_last=True)
    else:
        train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(train_x, train_y),
                                                   batch_size=batch_size, shuffle=True, drop_last=True)
        valid_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(valid_x, valid_y),
                                                   batch_size=batch_size, shuffle=False, drop_last=True)

    return train_loader, valid_loader, x_normalizer, y_normalizer

def rebuild_model(work_path, Device, in_dim=28, out_dim=5, name=None, mode=10):
    """
    rebuild the model with pth files
    """
    # rebuild the model
    if 'MLP' in name:
        from run_MLP import MLP
        from run_MLP import inference
        layer_mat = [in_dim, 256, 256, 256, 256, 256, 256, 256, 256, out_dim * 64 * 64]
        Net_model = MLP(layer_mat, is_BatchNorm=False).to(Device)
    elif 'deepONet' in name:
        from don.DeepONets import DeepONetMulti
        from run_deepONet import inference
        Net_model = DeepONetMulti(input_dim=2, operator_dims=[28, ], output_dim=5,
                                  planes_branch=[64] * 3, planes_trunk=[64] * 3).to(Device)
    elif 'FNO' in name:
        from fno.FNOs import FNO2d
        from run_FNO import inference
        Net_model = FNO2d(in_dim=in_dim, out_dim=out_dim, modes=mode, width=64, depth=4, steps=1,
                          padding=8, activation='gelu').to(Device)
    elif 'UNet' in name:
        from cnn.ConvNets import UNet2d
        from run_UNet import inference
        Net_model = UNet2d(in_sizes=(64, 64, 28), out_sizes=(64, 64, 5), width=64,
                           depth=4, steps=1, activation='gelu', dropout=0).to(Device)
    elif 'Transformer' in name:
        from basic.basic_layers import FcnSingle
        from fno.FNOs import FNO2d
        from transformer.Transformers import SimpleTransformer, FourierTransformer
        from run_Trans import inference, predictor

        # with open(os.path.join('transformer_config.yml')) as f:
        with open(r"D:\WQN\CODE\DENO4pytorch-main\Demo\Rotor37_2d\transformer_config.yml") as f:
            config = yaml.full_load(f)
            config = config['Rotor37_2d']

            # 建立网络
        Tra_model = FourierTransformer(**config).to(Device)
        FNO_model = FNO2d(in_dim=2, out_dim=config['n_targets'], modes=(16, 16), width=64, depth=4,
                          padding=9, activation='gelu').to(Device)
        MLP_model = FcnSingle(planes=(in_dim, 64, 64, config['n_targets']), last_activation=True).to(Device)
        Net_model = predictor(trunc=Tra_model, branch=MLP_model, field_dim=out_dim).to(Device)

        # from transformer.Transformers import FourierTransformer2D
        # from run_Trans import inference
        # with open(os.path.join('transformer_config_sql.yml')) as f:
        #     config = yaml.full_load(f)
        #     config = config['Rotor37_2d']
        #     config['fourier_modes'] = mode
        # # 建立网络
        # Net_model = FourierTransformer2D(**config).to(Device)

    isExist = os.path.exists(os.path.join(work_path, 'latest_model.pth'))
    if isExist:
        checkpoint = torch.load(os.path.join(work_path, 'latest_model.pth'), map_location=Device)
        Net_model.load_state_dict(checkpoint['net_model'])
        return Net_model, inference
    else:
        print("The pth file is not exist, CHECK PLEASE!")
        return None, None

def import_model_by_name(name):
    model_func = None
    inference = None
    train = None
    valid = None

    if 'MLP' in name:
        from run_MLP import MLP
        from run_MLP import inference, train, valid
        model_func = MLP
    elif 'deepONet' in name:
        from don.DeepONets import DeepONetMulti
        from run_deepONet import inference, train, valid
        model_func = DeepONetMulti
    elif 'FNO' in name:
        from fno.FNOs import FNO2d
        from run_FNO import inference, train, valid
        model_func = FNO2d
    elif 'FNM' in name:
        from fno.FNOs import FNO2dMultChannel
        from run_FNO_multi import inference, train, valid
        model_func = FNO2dMultChannel
    elif 'UNet' in name:
        from cnn.ConvNets import UNet2d
        from run_UNet import inference, train, valid
        model_func = UNet2d
    elif 'Transformer' in name:
        from transformer.Transformers import FourierTransformer2D
        from run_Trans import inference, train, valid
        model_func = FourierTransformer2D

    return model_func, inference, train, valid

def build_model_yml(yml_path, device, name=None):
    """
    build a new model based on yml file
    """
    # load the yml file
    with open(yml_path) as f:
        config = yaml.full_load(f)
        config = config[name + '_config'] #字典里面有字典

    # build the model
    model_func, inference, train, valid = import_model_by_name(name)
    net_model = model_func(**config).to(device)

    return net_model, inference, train, valid

def get_true_pred(loader, Net_model, inference, Device,
                  name=None, out_dim=5, iters=0, alldata=False):
    true_list = []
    pred_list = []
    set_size_sub  = 32
    if alldata:
        num = len(loader.dataset)
        iters = (num + set_size_sub -1)//set_size_sub

        new_loader = torch.utils.data.DataLoader(loader.dataset,
                                                 batch_size=set_size_sub,
                                                 shuffle=False,
                                                 drop_last=False)
        loader = new_loader

    for ii, data_box in enumerate(loader):
        if ii > iters:
            break
        if name in ("deepONet"):
            (data_x, data_f, data_y) = data_box
            sub_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(data_x, data_f, data_y),
                                                     batch_size=set_size_sub,
                                                     shuffle=False,
                                                     drop_last=False)
        else:
            (data_x, data_y) = data_box
            sub_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(data_x, data_y),
                                                     batch_size=set_size_sub,
                                                     shuffle=False,
                                                     drop_last=False)
    # for ii in range(iters):
        if name in ('MLP','Transformer'):
            _, true, pred = inference(sub_loader, Net_model, Device)
        else:
            _, _, true, pred = inference(sub_loader, Net_model, Device)
        true = true.reshape([true.shape[0], 64, 64, out_dim])
        pred = pred.reshape([pred.shape[0], 64, 64, out_dim])

        true_list.append(true)
        pred_list.append(pred)

    true = np.concatenate(true_list, axis=0)
    pred = np.concatenate(pred_list, axis=0)

    return true, pred

if __name__ == "__main__":
    noi = get_noise([3,3], 0.1)
    print(noi)

    #输出预测结果


from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import ssl
import os
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import codecs

def checkChar(token):
    for char in token:
        if (0 <= ord(char) and ord(char) <= 64) or (91 <= ord(char) and ord(char) <= 96) or (123 <= ord(char)):
           return False
        else:
            continue
    return True

def cleanMe(html):
    soup = BeautifulSoup(html, "html.parser")                                                       # create a new bs4 object from the html data loaded
    for script in soup(["script", "style"]):                                                        # remove all javascript and stylesheet code
        script.extract()
    # get text
    text = soup.get_text()
    # break into lines and remove leading and trailing space on each
    lines = (line.strip() for line in text.splitlines())
    # break multi-headlines into a line each
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    # drop blank lines
    text = '\n'.join(chunk for chunk in chunks if chunk)
    return text

path = 'crawled_html_pages/'
index = {}
docNum = 0
stop_words = set(stopwords.words('english'))                                                            #collection of stopwords to avoid
for filename in os.listdir(path):                                                                       #for every doc in folder...
    collection = {}                                                                                #all words collected from this html file
    docNum += 1                                                                                #Start with doc 1, then 2..3..4..500
    file = codecs.open('crawled_html_pages/'+ filename,'r', 'utf-8')                                  #open the html file
    page_text = cleanMe(file)                                                                         #collect page text from html file
    tokens = nltk.word_tokenize(page_text)                                                              #tokenize page text
    filtered_sentence = [w for w in tokens if not w in stop_words]                                      #remove stopwords from page text
    filtered_sentence = []

    breakWord = ''

    for w in tokens:                                                                                #place non-stopwords in token list
        if w not in stop_words:
            filtered_sentence.append(w.lower())

    for token in filtered_sentence:
        if len(token) == 1 or token == 'by':
            continue
        if checkChar(token) == False:
            continue
        if token == 'australia':
            breakWord = token
            continue
        if token == 'bangladesh' and breakWord == 'australia':
            break
        if token not in collection:
            collection[token] = 0
        collection[token] += 1

    for token in collection:
        if token not in index:
            index[token] = ''
        index[token] = index[token] + '['+ str(docNum) + ', ' + str(collection[token]) + ']'

f = open('index.txt', 'w')
vocab = open('uniqueWords.txt', 'w')
for term in index:
    f.write(term + ' => ' + index[term])
    vocab.write(term + '\n')
    f.write('\n')
f.close()
vocab.close()

print('Finished...')


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
import matplotlib.animation as animation

#Reading the input instructions and names of the objects.
obj_names=[]
f = open("../run/input.dat","r")
obj_count=int(f.readline())
for obj in range(obj_count):
    obj_names.append(f.readline().split()[0])
step=float(f.readline())
sim_length=float(f.readline())
step_count=float(f.readline())
if step==0:
    step=sim_length/step_count
elif sim_length==0:
    sim_length=step*step_count
else:
    step_count=sim_length/step
f.close()
obj_names=sorted(obj_names)

#Reading the results and sorting them first by name and then by index (time)
sim_results=pd.read_csv("../run/output.dat")
sim_results["indCol"]=sim_results.index
sim_results=sim_results.sort_values(["Object","indCol"])

#Number of iterations
iters=int((sim_results.count()/obj_count)["indCol"])

#Visualization initialization
fig=plt.figure()
ax=p3.Axes3D(fig)
ax.set_xlim3d([sim_results["X"].min(),sim_results["X"].max()])
ax.set_xlabel("X")
ax.set_ylim3d([sim_results["Y"].min(),sim_results["Y"].max()])
ax.set_ylabel("Y")
ax.set_zlim3d([sim_results["Z"].min(),sim_results["Z"].max()])
ax.set_zlabel("Z")
ax.set_title("SC2 Planetary Motion Simulation")
ax.view_init(40,10)

#Initial locations
scatters= [ax.scatter(sim_results["X"].iloc[i*iters],sim_results["Y"].iloc[i*iters],sim_results["Z"].iloc[i*iters],label=str(obj_names[i])) for i in range(obj_count)]

#Change of location with each iteration
def animate_scatters(iteration, data, scatters):
    for i in range(obj_count):
        scatters[i]._offsets3d = ([data["X"].iloc[i*iters+iteration]],
                                    [data["Y"].iloc[i*iters+iteration]],
                                    [data["Z"].iloc[i*iters+iteration]])
    return scatters

#Animation
ani=animation.FuncAnimation(fig,animate_scatters,iters,fargs=(sim_results,scatters),interval=10,blit=False,repeat=False)

#Option for saving the animation.
#ani.save('../run/animation.mp4', fps=30, extra_args=['-vcodec', 'libx264'])

#Drawing the animation
plt.legend()
plt.show()


from api.views import (APIFavorite, APIShoppingList, APIShoppingListDownload,
                       APIUserFollow, GetFollowViewSet, IngredientViewSet,
                       RecipeViewSet, TagViewSet)
from django.urls import include, path
from rest_framework.routers import DefaultRouter

router = DefaultRouter()

router.register('tags', TagViewSet, basename='tags')
router.register('ingredients', IngredientViewSet, basename='ingredients')
router.register('recipes', RecipeViewSet, basename='recipes')

urlpatterns = [
    path('users/subscriptions/', GetFollowViewSet.as_view({'get': 'list'})),
    path('users/<int:user_id>/subscribe/', APIUserFollow.as_view()),
    path('recipes/<int:pk>/favorite/', APIFavorite.as_view()),
    path('recipes/<int:pk>/shopping_cart/', APIShoppingList.as_view()),
    path('recipes/download_shopping_cart/', APIShoppingListDownload.as_view()),
    path('', include('djoser.urls')),
    path('auth/', include('djoser.urls.authtoken')),
    path('', include(router.urls)),
]


import pytest
from main import *

@pytest.fixture
def client():
    """Configures the app for testing

    Sets app config variable ``TESTING`` to ``True``

    :return: App for testing
    """

    # app.config['TESTING'] = True
    client = app.test_client()

    yield client

def test_landing(client):
    landing = client.get("/")
    html = landing.data.decode()

    assert "Select The State You Live In" in html
    assert landing.status_code == 200

def test_file_display(client):
    aws_files = client.get('/files')
    html = aws_files.data.decode()

    assert "Click on the filename to download it" in html
    assert aws_files.status_code == 200


import ast
import os.path as osp
import platform
import shutil
import sys
import warnings
import tempfile
from argparse import ArgumentParser
from collections import abc
from importlib import import_module
import re
from addict import Dict
from mmcv.utils.misc import import_modules_from_strings
from mmcv.utils.path import check_file_exist
from yapf.yapflib.yapf_api import FormatCode

if platform.system() == "Windows":
    import regex as re
else:
    import re
from src.utils.web_utils import load_text_from_web, check_url_exist
from src.utils.web_utils import shortname as web_shortcut

BASE_KEY = "_base_"
DELETE_KEY = "_delete_"
RESERVED_KEYS = ["filename", "text", "pretty_text"]

def check_path_type(filepath):
    if ":" in filepath:
        if filepath.startswith("file:"):
            return "abs_path"
        elif filepath.startswith("https:") or filepath.startswith("http:"):
            return "web_url"
        elif filepath.split(":")[0] in web_shortcut:
            return "web_shortcut"
        else:
            raise ValueError(f"Unrecognized file type for file {filepath}")

    else:
        return "rel_path"

def find_correct_path(cur_dir, filepath):
    path_type = check_path_type(filepath)
    if path_type == "rel_path":
        return osp.join(cur_dir, filepath)
    elif path_type == "abs_path":
        # remove indicator
        return filepath.split(":")[1]
    elif path_type == "web_shortcut":
        # replace indicator
        shortcut = filepath.split(":")[0]
        return osp.join(web_shortcut[shortcut], filepath.split(":")[1])
    else:
        return filepath

def load_file(filepath):
    path_type = check_path_type(filepath)
    if path_type in ["abs_path", "rel_path"]:
        with open(filepath, "r") as f:
            config_file = f.read()
    elif path_type in ["web_url", "web_shortcut"]:
        return load_text_from_web(filepath)
    else:
        raise NotImplementedError(f"load {filepath} failed")
    return config_file

def copy_file(src_path, target_path):
    path_type = check_path_type(src_path)
    if path_type in ["abs_path", "rel_path"]:
        shutil.copyfile(src_path, target_path)
    elif path_type == "web_url":
        content = load_file(src_path)
        with tempfile.TemporaryDirectory() as temp_config_dir:
            temp_config_file = tempfile.NamedTemporaryFile(
                dir=temp_config_dir, suffix=osp.splitext(src_path)[1]
            )
            if platform.system() == "Windows":
                temp_config_file.close()
            with open(temp_config_file.name, "w") as f:
                f.write(content)
            shutil.copyfile(temp_config_file.name, target_path)
    else:
        raise NotImplementedError(f"{src_path} to {target_path} failed")

class ConfigDict(Dict):
    def __missing__(self, name):
        raise KeyError(name)

    def __getattr__(self, name):
        try:
            value = super(ConfigDict, self).__getattr__(name)
        except KeyError:
            ex = AttributeError(
                f"'{self.__class__.__name__}' object has no " f"attribute '{name}'"
            )
        except Exception as e:
            ex = e
        else:
            return value
        raise ex

class VariableParser(object):
    def __init__(self, symbol=["\{", "\}"]):
        self.symbol = symbol
        self.pattern = re.compile(self.general_pattern)

    def find_all(self, string):
        vars = self.pattern.findall(string)
        return vars

    def replace(self, string, values):
        for k, v in values.items():
            string = re.sub(self.specfic_pattern(k), v, string)
        return string

    @property
    def general_pattern(self):
        return self.symbol[0] + "(.+?)" + self.symbol[1]

    def specfic_pattern(self, word):
        return self.symbol[0] + word + self.symbol[1]

    def fetch_value(self, cfg_dict, key):
        if key not in cfg_dict:
            warnings.warn(
                "Attr {} not found in {}. Set it to None by default.".format(
                    key, type(cfg_dict)
                )
            )
            return "none"
        return cfg_dict[key]

    def parse(self, cfg, total_cfg_dict: ConfigDict):
        if isinstance(cfg, dict):
            return ConfigDict(
                {k: self.parse(v, total_cfg_dict) for k, v in cfg.items()}
            )
        elif isinstance(cfg, list):
            return [self.parse(v, total_cfg_dict) for v in cfg]
        elif isinstance(cfg, str):
            expr = self.find_all(cfg)
            if expr is not None:
                return self.replace(
                    cfg, {e: self.fetch_value(total_cfg_dict, e) for e in expr}
                )
        else:
            return cfg
        return cfg

def add_args(parser, cfg, prefix=""):
    for k, v in cfg.items():
        if isinstance(v, str):
            parser.add_argument("--" + prefix + k)
        elif isinstance(v, int):
            parser.add_argument("--" + prefix + k, type=int)
        elif isinstance(v, float):
            parser.add_argument("--" + prefix + k, type=float)
        elif isinstance(v, bool):
            parser.add_argument("--" + prefix + k, action="store_true")
        elif isinstance(v, dict):
            add_args(parser, v, prefix + k + ".")
        elif isinstance(v, abc.Iterable):
            parser.add_argument("--" + prefix + k, type=type(v[0]), nargs="+")
        else:
            print(f"cannot parse key {prefix + k} of type {type(v)}")
    return parser

class Config:
    """A facility for config and config files.

    It supports common file formats as configs: python/json/yaml. The interface
    is the same as a dict object and also allows access config values as
    attributes.

    Example:
        >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1])))
        >>> cfg.a
        1
        >>> cfg.b
        {'b1': [0, 1]}
        >>> cfg.b.b1
        [0, 1]
        >>> cfg = Config.fromfile('tests/data/config/a.py')
        >>> cfg.filename
        "/home/kchen/projects/mmcv/tests/data/config/a.py"
        >>> cfg.item4
        'test'
        >>> cfg
        "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: "
        "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}"
    """

    @staticmethod
    def _substitute_predefined_vars(filename, temp_config_name):
        file_dirname = osp.dirname(filename)
        file_basename = osp.basename(filename)
        file_basename_no_extension = osp.splitext(file_basename)[0]
        file_extname = osp.splitext(filename)[1]
        support_templates = dict(
            fileDirname=file_dirname,
            fileBasename=file_basename,
            fileBasenameNoExtension=file_basename_no_extension,
            fileExtname=file_extname,
        )
        config_file = load_file(filename)
        for key, value in support_templates.items():
            regexp = r"\{\{\s*" + str(key) + r"\s*\}\}"
            value = value.replace("\\", "/")
            config_file = re.sub(regexp, value, config_file)
        with open(temp_config_name, "w") as tmp_config_file:
            tmp_config_file.write(config_file)

    @staticmethod
    def _validate_py_syntax(filename):
        content = load_file(filename)
        try:
            ast.parse(content)
        except SyntaxError as e:
            raise SyntaxError(
                "There are syntax errors in config " f"file {filename}: {e}"
            )

    @staticmethod
    def _file2dict(filename, use_predefined_variables=True):
        file_type = check_path_type(filename)
        if file_type in ["rel_path", "abs_path"]:
            filename = osp.abspath(osp.expanduser(filename))
            check_file_exist(filename)
        else:
            check_url_exist(filename)

        fileExtname = osp.splitext(filename)[1]
        if fileExtname not in [".py", ".json", ".yaml", ".yml"]:
            raise IOError("Only py/yml/yaml/json type are supported now!")

        with tempfile.TemporaryDirectory() as temp_config_dir:
            temp_config_file = tempfile.NamedTemporaryFile(
                dir=temp_config_dir, suffix=fileExtname
            )
            if platform.system() == "Windows":
                temp_config_file.close()
            temp_config_name = osp.basename(temp_config_file.name)
            # Substitute predefined variables
            if use_predefined_variables:
                Config._substitute_predefined_vars(filename, temp_config_file.name)
            else:
                copy_file(filename, temp_config_file.name)

            if filename.endswith(".py"):
                temp_module_name = osp.splitext(temp_config_name)[0]
                sys.path.insert(0, temp_config_dir)
                Config._validate_py_syntax(filename)
                mod = import_module(temp_module_name)
                sys.path.pop(0)
                cfg_dict = {
                    name: value
                    for name, value in mod.__dict__.items()
                    if not name.startswith("__")
                }
                # delete imported module
                del sys.modules[temp_module_name]
            elif filename.endswith((".yml", ".yaml", ".json")):
                import mmcv

                cfg_dict = mmcv.load(temp_config_file.name)
            # close temp file
            temp_config_file.close()

        cfg_text = filename + "\n"
        cfg_text += load_file(filename)

        if BASE_KEY in cfg_dict:
            cfg_dir = osp.dirname(filename)
            base_filename = cfg_dict.pop(BASE_KEY)
            base_filename = (
                base_filename if isinstance(base_filename, list) else [base_filename]
            )

            cfg_dict_list = list()
            cfg_text_list = list()
            for f in base_filename:
                ref_file_path = find_correct_path(cfg_dir, f)
                _cfg_dict, _cfg_text = Config._file2dict(ref_file_path)
                cfg_dict_list.append(_cfg_dict)
                cfg_text_list.append(_cfg_text)

            base_cfg_dict = dict()
            for c in cfg_dict_list:
                if len(base_cfg_dict.keys() & c.keys()) > 0:
                    raise KeyError("Duplicate key is not allowed among bases")
                base_cfg_dict.update(c)

            base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict)
            cfg_dict = base_cfg_dict

            # merge cfg_text
            cfg_text_list.append(cfg_text)
            cfg_text = "\n".join(cfg_text_list)

        return cfg_dict, cfg_text

    @staticmethod
    def _merge_a_into_b(a, b, allow_list_keys=False):
        """merge dict ``a`` into dict ``b`` (non-inplace).

        Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid
        in-place modifications.

        Args:
            a (dict): The source dict to be merged into ``b``.
            b (dict): The origin dict to be fetch keys from ``a``.
            allow_list_keys (bool): If True, int string keys (e.g. '0', '1')
              are allowed in source ``a`` and will replace the element of the
              corresponding index in b if b is a list. Default: False.

        Returns:
            dict: The modified dict of ``b`` using ``a``.

        Examples:
            # Normally merge a into b.
            >>> Config._merge_a_into_b(
            ...     dict(obj=dict(a=2)), dict(obj=dict(a=1)))
            {'obj': {'a': 2}}

            # Delete b first and merge a into b.
            >>> Config._merge_a_into_b(
            ...     dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1)))
            {'obj': {'a': 2}}

            # b is a list
            >>> Config._merge_a_into_b(
            ...     {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True)
            [{'a': 2}, {'b': 2}]
        """
        b = b.copy()
        for k, v in a.items():
            if allow_list_keys and k.isdigit() and isinstance(b, list):
                k = int(k)
                if len(b) <= k:
                    raise KeyError(f"Index {k} exceeds the length of list {b}")
                b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys)
            elif isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False):
                allowed_types = (dict, list) if allow_list_keys else dict
                if not isinstance(b[k], allowed_types):
                    raise TypeError(
                        f"{k}={v} in child config cannot inherit from base "
                        f"because {k} is a dict in the child config but is of "
                        f"type {type(b[k])} in base config. You may set "
                        f"`{DELETE_KEY}=True` to ignore the base config"
                    )
                b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys)
            else:
                b[k] = v
        return b

    @staticmethod
    def fromfile(filename, use_predefined_variables=True, import_custom_modules=True):
        cfg_dict, cfg_text = Config._file2dict(filename, use_predefined_variables)
        if import_custom_modules and cfg_dict.get("custom_imports", None):
            import_modules_from_strings(**cfg_dict["custom_imports"])
        return Config(cfg_dict, cfg_text=cfg_text, filename=filename)

    @staticmethod
    def auto_argparser(description=None):
        """Generate argparser from config file automatically (experimental)"""
        partial_parser = ArgumentParser(description=description)
        partial_parser.add_argument("config", help="config file path")
        cfg_file = partial_parser.parse_known_args()[0].config
        cfg = Config.fromfile(cfg_file)
        parser = ArgumentParser(description=description)
        parser.add_argument("config", help="config file path")
        add_args(parser, cfg)
        return parser, cfg

    def __init__(self, cfg_dict=None, cfg_text=None, filename=None):
        if cfg_dict is None:
            cfg_dict = dict()
        elif not isinstance(cfg_dict, dict):
            raise TypeError("cfg_dict must be a dict, but " f"got {type(cfg_dict)}")
        for key in cfg_dict:
            if key in RESERVED_KEYS:
                raise KeyError(f"{key} is reserved for config file")

        super(Config, self).__setattr__("_cfg_dict", ConfigDict(cfg_dict))
        super(Config, self).__setattr__("_filename", filename)
        if cfg_text:
            text = cfg_text
        elif filename:
            with open(filename, "r") as f:
                text = f.read()
        else:
            text = ""
        super(Config, self).__setattr__("_text", text)
        self._cfg_dict["exp_name"] = osp.basename(filename).split(".")[0]

    @property
    def filename(self):
        return self._filename

    @property
    def text(self):
        return self._text

    @property
    def pretty_text(self):

        indent = 4

        def _indent(s_, num_spaces):
            s = s_.split("\n")
            if len(s) == 1:
                return s_
            first = s.pop(0)
            s = [(num_spaces * " ") + line for line in s]
            s = "\n".join(s)
            s = first + "\n" + s
            return s

        def _format_basic_types(k, v, use_mapping=False):
            if isinstance(v, str):
                v_str = f"'{v}'"
            else:
                v_str = str(v)

            if use_mapping:
                k_str = f"'{k}'" if isinstance(k, str) else str(k)
                attr_str = f"{k_str}: {v_str}"
            else:
                attr_str = f"{str(k)}={v_str}"
            attr_str = _indent(attr_str, indent)

            return attr_str

        def _format_list(k, v, use_mapping=False):
            # check if all items in the list are dict
            if all(isinstance(_, dict) for _ in v):
                v_str = "[\n"
                v_str += "\n".join(
                    f"dict({_indent(_format_dict(v_), indent)})," for v_ in v
                ).rstrip(",")
                if use_mapping:
                    k_str = f"'{k}'" if isinstance(k, str) else str(k)
                    attr_str = f"{k_str}: {v_str}"
                else:
                    attr_str = f"{str(k)}={v_str}"
                attr_str = _indent(attr_str, indent) + "]"
            else:
                attr_str = _format_basic_types(k, v, use_mapping)
            return attr_str

        def _contain_invalid_identifier(dict_str):
            contain_invalid_identifier = False
            for key_name in dict_str:
                contain_invalid_identifier |= not str(key_name).isidentifier()
            return contain_invalid_identifier

        def _format_dict(input_dict, outest_level=False):
            r = ""
            s = []

            use_mapping = _contain_invalid_identifier(input_dict)
            if use_mapping:
                r += "{"
            for idx, (k, v) in enumerate(input_dict.items()):
                is_last = idx >= len(input_dict) - 1
                end = "" if outest_level or is_last else ","
                if isinstance(v, dict):
                    v_str = "\n" + _format_dict(v)
                    if use_mapping:
                        k_str = f"'{k}'" if isinstance(k, str) else str(k)
                        attr_str = f"{k_str}: dict({v_str}"
                    else:
                        attr_str = f"{str(k)}=dict({v_str}"
                    attr_str = _indent(attr_str, indent) + ")" + end
                elif isinstance(v, list):
                    attr_str = _format_list(k, v, use_mapping) + end
                else:
                    attr_str = _format_basic_types(k, v, use_mapping) + end

                s.append(attr_str)
            r += "\n".join(s)
            if use_mapping:
                r += "}"
            return r

        cfg_dict = self._cfg_dict.to_dict()
        text = _format_dict(cfg_dict, outest_level=True)
        # copied from setup.cfg
        yapf_style = dict(
            based_on_style="pep8",
            blank_line_before_nested_class_or_def=True,
            split_before_expression_after_opening_paren=True,
        )
        text, _ = FormatCode(text, style_config=yapf_style, verify=True)

        return text

    def __repr__(self):
        return f"Config (path: {self.filename}): {self._cfg_dict.__repr__()}"

    def __len__(self):
        return len(self._cfg_dict)

    def __getattr__(self, name):
        return getattr(self._cfg_dict, name)

    def __getitem__(self, name):
        return self._cfg_dict.__getitem__(name)

    def __setattr__(self, name, value):
        if isinstance(value, dict):
            value = ConfigDict(value)
        self._cfg_dict.__setattr__(name, value)

    def __setitem__(self, name, value):
        if isinstance(value, dict):
            value = ConfigDict(value)
        self._cfg_dict.__setitem__(name, value)

    def __iter__(self):
        return iter(self._cfg_dict)

    def __getstate__(self):
        return (self._cfg_dict, self._filename, self._text)

    def __setstate__(self, state):
        _cfg_dict, _filename, _text = state
        super(Config, self).__setattr__("_cfg_dict", _cfg_dict)
        super(Config, self).__setattr__("_filename", _filename)
        super(Config, self).__setattr__("_text", _text)

    def dump(self, file=None):
        cfg_dict = super(Config, self).__getattribute__("_cfg_dict").to_dict()
        if self.filename.endswith(".py"):
            if file is None:
                return self.pretty_text
            else:
                with open(file, "w") as f:
                    f.write(self.pretty_text)
        else:
            import mmcv

            if file is None:
                file_format = self.filename.split(".")[-1]
                return mmcv.dump(cfg_dict, file_format=file_format)
            else:
                mmcv.dump(cfg_dict, file)

    def merge_from_dict(self, options, allow_list_keys=True):
        """Merge list into cfg_dict.

        Merge the dict parsed by MultipleKVAction into this cfg.

        Examples:
            >>> options = {'model.backbone.depth': 50,
            ...            'model.backbone.with_cp':True}
            >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet'))))
            >>> cfg.merge_from_dict(options)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(
            ...     model=dict(backbone=dict(depth=50, with_cp=True)))

            # Merge list element
            >>> cfg = Config(dict(pipeline=[
            ...     dict(type='LoadImage'), dict(type='LoadAnnotations')]))
            >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')})
            >>> cfg.merge_from_dict(options, allow_list_keys=True)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(pipeline=[
            ...     dict(type='SelfLoadImage'), dict(type='LoadAnnotations')])

        Args:
            options (dict): dict of configs to merge from.
            allow_list_keys (bool): If True, int string keys (e.g. '0', '1')
              are allowed in ``options`` and will replace the element of the
              corresponding index in the config if the config is a list.
              Default: True.
        """
        option_cfg_dict = {}
        for full_key, v in options.items():
            d = option_cfg_dict
            key_list = full_key.split(".")
            for subkey in key_list[:-1]:
                d.setdefault(subkey, ConfigDict())
                d = d[subkey]
            subkey = key_list[-1]
            d[subkey] = v

        cfg_dict = super(Config, self).__getattribute__("_cfg_dict")
        super(Config, self).__setattr__(
            "_cfg_dict",
            Config._merge_a_into_b(
                option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys
            ),
        )

    def build(self):
        var_parser = VariableParser()
        cfg_dict = var_parser.parse(self._cfg_dict, self._cfg_dict)
        super(Config, self).__setattr__("_cfg_dict", cfg_dict)


import telebot
import flask
import conf
from telebot import types
import re
from work_with_db import add_to_data, collect_vacancies_data, check_experience, \
    get_calendar, add_to_calendar, db
from work_with_answers import process_values
import logging
logging.basicConfig(level=logging.ERROR,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

WEBHOOK_URL_BASE = "https://{}:{}".format(conf.WEBHOOK_HOST, conf.WEBHOOK_PORT)
WEBHOOK_URL_PATH = "/{}/".format(conf.TOKEN)
bot = telebot.TeleBot(conf.TOKEN, threaded=False)
bot.remove_webhook()
bot.set_webhook(url=WEBHOOK_URL_BASE+WEBHOOK_URL_PATH)
app = flask.Flask(__name__)

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db.app = app
db.init_app(app)

applicant_dict = {}  # Словарь с пользователями
vacancies_data = collect_vacancies_data()  # информация о вакансиях из базы данных

# класс пользователя
class Applicant:
    def __init__(self, name):
        self.name = name
        self.city = None
        self.age = None
        self.sex = None
        self.number = None
        self.email = None
        self.social = None
        self.vacancies = []  # сюда будем складывать вакансии, которыми заинтересовался пользователь

@bot.message_handler(commands=['start'])
def send_welcome(message):
    """
    Функция, которой бот приветствует пользователя
    """
    bot.send_message(message.chat.id, 'Привет!\n'
                                      'Меня зовут Бушенка. Я чат-бот компании "Буше". '
                                      'Буду рад рассказать о нашей компании, наших вакансиях, '
                                      'а также помочь тебе попасть именно к нам! '
                                      'У нас обалденные вензели с вишней! :)\n'
                                      'Чтобы узнать, что я умею нажми /help')

@bot.message_handler(commands=['help'])
def help_func(message):
    """
    Функция, которая выводит все полезные кнопки бота
    """
    bot.send_message(message.chat.id, "Я супер-умный и всегда готов тебе помочь. Надеюсь, мы с тобой подружимся)\n"
                                      "У меня ты можешь:\n"
                                      "-узнать все о компании /about\n"
                                      "-найти контакты HR /contacts (Ну вдруг я тебе не понравлюсь:()\n"
                                      "-получить ответ на интересующий вопрос /Q_A\n"
                                      "-посмотреть актуальные вакансии и выбрать то, что тебе подходит /vacancies\n"
                                      "-вспомнить, что я умею /help\n"
                                      "-познакомиться со мной поближе /intro")

@bot.message_handler(commands=['about'])
def about_func(message):
    """
    Функция, которая выводит информацию о компании
    """
    bot.send_message(message.chat.id, "Наш сайт - https://bushe.ru/\n"
                                      "Наши соц. сети:\n"
                                      "Вконтакте - https://vk.com/bushe.bakery\n"
                                      "Инстаграм - https://www.instagram.com/bushe.bakery/\n"
                                      "Одноклассники - https://ok.ru/bushe.bakery\n"
                                      "Facebook - https://www.facebook.com/bushe.bakery/\n"
                                      "YouTube - https://www.youtube.com/c/BusheBakery\n\n"
                                      "Мы - 'Буше'. Меняем вкус повседневной жизни, "
                                      "создавая новые традиции для еды, общения и вдохновения.\n"
                                      "Работать с нами, значит быть живым в любой момент времени!")

@bot.message_handler(commands=['contacts'])
def contacts_func(message):
    """
    Функция, которая выводит контакты HR
    """
    bot.send_message(message.chat.id, "Позвони в HR: +7 (812) 640-01-10\n"
                                      "Напиши: personal@bushe.ru\n"
                                      "Посети наш сайт с вакансиями - https://careers.bushe.ru/\n"
                                      "Или на hh.ru: "
                                      "https://hh.ru/search/"
                                      "vacancy?st=searchVacancy&from=employerPage&employer_id=114223")

@bot.message_handler(commands=['Q_A'])
def qa_func(message):
    """
    Функция для ответов на вопросы пользователя
    """
    msg = bot.send_message(message.chat.id, 'Можешь задать мне любой вопрос!')
    bot.register_next_step_handler(msg, no_answer)

def no_answer(message):
    """
    Функция, которая генерирует ответ на вопрос. Сейчас бот отвечает только на один вопрос :)
    """
    if message.text == 'Какие этапы отбора?':
        bot.send_message(message.chat.id, 'Сначала ты пообщаешься со мной. А дальше если на твою вакансию '
                                          'нужны hard-skills, то мы тебя позовем на прохождение конкурса. '
                                          'И последний этап - это краткое интервью с hr')
    else:
        bot.send_message(message.chat.id, 'Упс! Такое мне не рассказывали. '
                                          'Можешь связаться со знающими людьми /contacts')

@bot.message_handler(commands=['intro'])
def intro_func(message):
    """
    Функция, с которой начинается заполнение информации о пользователе
    """
    if message.chat.id in applicant_dict:  # проверяем, нет ли пользователя уже в БД,
        # если нет, то спрашиваем имя
        bot.send_message(message.chat.id, "Мы с тобой уже знакомы! :)\n"
                                          "Если хочешь посмотреть вакансии, нажми /vacancies")
    else:
        msg = bot.send_message(message.chat.id, 'Как тебя зовут? (Лучше полностью ФИО)')
        bot.register_next_step_handler(msg, process_name_step)  # переходим к следующему вопросу

def process_name_step(message):
    """
    Функция, которая добавляет пользователя в словарь и записывает его имя, затем спрашиваем возраст
    """
    applicant = Applicant(message.text)  # создаем объект класса Applicant и передаем name
    applicant_dict[message.chat.id] = applicant
    msg = bot.send_message(message.chat.id, 'Сколько тебе лет?')
    bot.register_next_step_handler(msg, process_age_step)

def process_age_step(message):
    """
    Функция, которая записывает возраст пользователя. Если возраст указан не цифрой, то просим ответить корректно.
    Потом спрашиваем пол
    """
    age = message.text
    if not age.isdigit():  # проверяем цифрой ли ответил пользователь
        msg = bot.send_message(message.chat.id, 'Пж, укажи возраст одной цифрой')
        bot.register_next_step_handler(msg, process_age_step)
        return
    applicant_dict[message.chat.id].age = int(age)
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('М', 'Ж')
    msg = bot.send_message(message.chat.id, 'Укажи свой пол', reply_markup=markup)
    bot.register_next_step_handler(msg, process_sex_step)

def process_sex_step(message):
    """
    Функция, которая записывает пол пользователя. Затем спрашиваем город
    """
    applicant_dict[message.chat.id].sex = message.text
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Москва', 'Санкт-Петербург')
    msg = bot.send_message(message.chat.id, 'Из какого ты города?', reply_markup=markup)
    bot.register_next_step_handler(msg, process_city_step)

def process_city_step(message):
    """
    Функция, которая записывает город пользователя. Затем спрашиваем номер телефона
    """
    applicant_dict[message.chat.id].city = message.text
    msg = bot.send_message(message.chat.id, 'Дашь номерочек? Вдруг мне захочется тебе позвонить в 5 утра в вс :)')
    bot.register_next_step_handler(msg, take_number)

def take_number(message):
    """
    Функция, которая записывает номер пользователя. Затем спрашиваем email
    """
    applicant_dict[message.chat.id].number = message.text
    msg = bot.send_message(message.chat.id, 'И email тоже (постараюсь не спамить)')
    bot.register_next_step_handler(msg, take_email)

def take_email(message):
    """
    Функция, которая записывает email пользователя. Затем спрашиваем соц. сети
    """
    applicant_dict[message.chat.id].email = message.text
    msg = bot.send_message(message.chat.id, 'Можешь еще оставить ссылку на какую-нибудь соц. сеть')
    bot.register_next_step_handler(msg, take_social)

def take_social(message):
    """
    Функция, которая записывает соц.сети пользователя, выводит все введенные пользователем данные
    и спрашивает, верны ли они
    """
    applicant_dict[message.chat.id].social = message.text
    bot.send_message(message.chat.id, f'Проверь, все ли я правильно записал:\n'
                                      f'Имя: {applicant_dict[message.chat.id].name}\n'
                                      f'Возраст: {applicant_dict[message.chat.id].age}\n'
                                      f'Пол: {applicant_dict[message.chat.id].sex}\n'
                                      f'Город: {applicant_dict[message.chat.id].city}\n'
                                      f'Телефон: {applicant_dict[message.chat.id].number}\n'
                                      f'Email: {applicant_dict[message.chat.id].email}\n'
                                      f'Соц. сети: {applicant_dict[message.chat.id].social}')
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Да', 'Нет')
    msg = bot.send_message(message.chat.id, 'Все верно?', reply_markup=markup)
    bot.register_next_step_handler(msg, check_correctness_step)

def check_correctness_step(message):
    """
    Функция, которая записывает все данные в базу данных, если они верны, а если нет, то идем по второму кругу
    """
    if message.text == 'Да':
        user = applicant_dict[message.chat.id]  # получаем информацию о пользователе из словаря
        data_app = [(message.chat.id, user.name, user.city, user.age, user.sex, user.number, user.email, user.social)]
        # подготавливаем данные
        add_to_data(data_app)  # записываем данные в БД
        bot.send_message(message.chat.id, 'Приятно познакомиться! Теперь можешь посмотреть свежие вакансии /vacancies')
    else:
        applicant_dict.pop(message.chat.id, None)
        bot.send_message(message.chat.id, 'Ой-ой, ну ты чего, теперь придется проходить все заново... Нажми /intro')

@bot.message_handler(commands=['vacancies'])
def vacancies_func(message):
    """
    Функция, которая проверяет, есть ли данные о пользователе в БД. Если нет, то спрашиваем,
    если есть, то показываем вакансии
    """
    if message.chat.id not in applicant_dict:  # проверяем, есть ли id пользователя в словаре
        bot.send_message(message.chat.id, "Кажется, мы с тобой еще не знакомы, давай знакомиться!")
        intro_func(message)  # отправляем к функции, которая собирает информацию о пользователе
    else:
        bot.send_message(message.chat.id, "Привет! Сейчас посмотрю, что появилось из новеньких вакансий")
        choose_city(message)

def choose_city(message):
    """
    Функция для выбора города, по которому будут фильтрованы вакансии
    """
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Москва', 'Санкт-Петербург')
    msg = bot.send_message(message.chat.id, 'В каком городе тебе бы хотелось работать?', reply_markup=markup)
    bot.register_next_step_handler(msg, buttons_vac)

def buttons_vac(message):
    """
    Функция, которая фильтрует вакансии по городу и выводит все актуальные вакансии в виде кнопок с названиями
    """
    city = message.text
    vacancies_city = []
    for vacancy in vacancies_data:
        if vacancy.city == city:
            vacancies_city.append((vacancy.id, vacancy.name))
    keyboard = types.InlineKeyboardMarkup(row_width=1)
    keyboard.add(*[types.InlineKeyboardButton(text=vac[1], callback_data=f"button_{int(vac[0])}")
                   for vac in vacancies_city])
    bot.send_message(message.chat.id, 'Можешь выбрать из этих вакансий:', reply_markup=keyboard)

@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    """
    Функция, которая выводит условия и обязанности по выбранной должности. Дальше спрашиваем про опыт работы
    """
    if call.message:
        _, i = call.data.split('_')  # получаем id вакансии
        vacancy_id = int(i)
        applicant_dict[call.message.chat.id].vacancies.append(vacancy_id)  # добавляем вакансию в список всех вакансий,
        # которыми интересовался пользователь
        for vacancy in vacancies_data:
            if vacancy.id == vacancy_id:
                conditions = vacancy.conditions
                responsibilities = vacancy.responsibilities
        bot.send_message(call.message.chat.id, 'Вот, что мы предлагаем:\n'
                                               f'{conditions}')
        bot.send_message(call.message.chat.id, 'А вот, что тебе предстоит делать:\n '
                                               f'{responsibilities}')
        msg = bot.send_message(call.message.chat.id, 'Если тебя все устраивает, то укажи свой опыт работы на схожей '
                                                     'должности (просто цифрой сколько лет)')
        bot.register_next_step_handler(msg, process_exp_step)

def process_exp_step(message):
    """
    Функция, которая записывает id пользователя, id вакансии и опыт работы в БД
    """
    if re.search('\d+', message.text):  # если в ответе пользователя есть цифра, то записываем ее
        experience = int(re.findall('\d+', message.text)[0])
    else:
        experience = 0  # иначе записываем 0
    vac_id = applicant_dict[message.chat.id].vacancies[-1]  # получаем id вакансии
    if check_experience(vac_id, experience):  # если кандидат подходит по опыту работы, то спрашиваем про ценности
        msg = bot.send_message(message.chat.id, 'Отлично! Теперь мы хотели бы узнать тебя поближе. '
                                                'Пожалуйста, опиши кратко, какой философии по жизни ты придерживаешься,'
                                                'почему хочешь попасть в нашу команду')
        bot.register_next_step_handler(msg, values_step)
    else:
        if applicant_dict[message.chat.id].sex == 'M':  # подберем слово по полу пользователя
            word = 'готов'
        else:
            word = 'готова'
        bot.send_message(message.chat.id, 'Друг, к сожалению, ты не подходишь для данной вакансии из-за нехватки опыта.'
                                          f' Мы с удовольствием примем тебя в нашу команду, когда ты будешь {word}.'
                                          'Или можешь посмотреть другие вакансии /vacancies')

def values_step(message):
    """
    Функция, которая проверяет кандидата на совпадение ценностей.
    Если все ценности совпадают, то выводит требования к кандидату
    """
    answer = message.text
    if process_values(answer):  # проверяем на совпадение ценностей
        vac_id = applicant_dict[message.chat.id].vacancies[-1]
        for vacancy in vacancies_data:
            if vacancy.id == vac_id:
                requirements = vacancy.requirements.split('\n')
        bot.send_message(message.chat.id, 'Теперь тебе надо будет отметить те пункты, по которым ты подходишь')
        ask_requirements(requirements, message)
    else:
        bot.send_message(message.chat.id, 'Прости, друг, но я считаю, что наши мировоззрения не сходятся. '
                                          'Удачи тебе в поисках. Кто ищет, тот найдет!')

def ask_requirements(requirements, message):
    """
    Функция, которая выводит по очереди пункты с требованиями и спрашивает, относятся ли они к кандидату
    Если кандидат подошел по всем требованиям, то предлагает выбрать дату и время для собеседования
    :param requirements: список пунктов с требованиями
    """
    if not requirements:
        dates = get_calendar()  # получаем расписание
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add(*[x.day + ', ' + x.time for x in dates if not x.applicant_id])  # выводим свободные слоты в виде кнопок
        msg = bot.send_message(message.chat.id, 'Мы друг другу подходим! Выбери дату и время, когда ты можешь прийти '
                                                'на собес', reply_markup=markup)
        bot.register_next_step_handler(msg, choose_date)
    else:
        requirement = requirements[0]  # получаем пункт с требованием
        requirements.remove(requirement)  # убираем этот пункт из общего списка требований
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('Да', 'Нет')
        msg = bot.send_message(message.chat.id, f'{requirement}', reply_markup=markup)
        bot.register_next_step_handler(msg, lambda m: check(m, requirements))

def check(message, requirements):
    """
    Функция, которая обрабатывает ответ пользователя по каждому пункту
    Если ответ отрицательный, то кандидат не прошел отбор
    :param requirements: список пунктов с требованиями
    """
    if message.text == 'Да':
        ask_requirements(requirements, message)
    else:
        bot.send_message(message.chat.id, 'Это печально, но по результатам тестового задания мы вынуждены '
                                          'расстаться... Но твои контакты будут добавлены в нашу базу данных, '
                                          'к которым мы обязательно обратимся при необходимости в найме.\n'
                                          'Успехов тебе и побольше вензелей с вишней!')

def choose_date(message):
    """
    Функция, которая записывает пользователя в БД с расписанием
    """
    date = message.text
    day, time = date.split(', ')
    add_to_calendar(day, time, message.chat.id)  # добавляем в БД
    bot.send_message(message.chat.id, f'Супер! Ждем тебя {day} в {time}')

@bot.message_handler(func=lambda m: True)
def send_common_msg(message):
    """
    Функция, которая выводит сообщение, если пользователь ввел что-то не то
    """
    bot.send_message(message.chat.id, 'Дружок, ты что-то потерялся. Нажми на /help, чтобы найти то, что тебе нужно')

@app.route('/', methods=['GET', 'HEAD'])
def index():
    return 'ok'

@app.route(WEBHOOK_URL_PATH, methods=['POST'])
def webhook():
    if flask.request.headers.get('content-type') == 'application/json':
        json_string = flask.request.get_data().decode('utf-8')
        update = telebot.types.Update.de_json(json_string)
        bot.process_new_updates([update])
        return ''
    else:
        flask.abort(403)


import tensorflow as tf
print(tf.__version__)

import gym
import time
import numpy as np
import tensorflow as tf
import sys
from collections import namedtuple

np.random.seed(1)
tf.random.set_seed(1)

def eprint(*args):
    print(*args, file=sys.stderr)

class Model(tf.keras.Model):
    def __init__(self, num_actions):
        super().__init__()
        self.num_actions = num_actions
        self.fc1 = tf.keras.layers.Dense(32, activation='relu', kernel_initializer='he_uniform')
        self.fc2 = tf.keras.layers.Dense(32, activation='relu', kernel_initializer='he_uniform')
        self.logits = tf.keras.layers.Dense(num_actions)

    def call(self, inputs):
        # inputs: [batch, 4]
        # print("inputs: {}".format(inputs))
        x = self.fc1(inputs)
        x = self.fc2(x)
        x = self.logits(x)
        return x

    def get_action_value(self, observation):
        q_values = self.predict(observation)
        best_action = np.argmax(q_values, axis=-1)
        return best_action, np.max(q_values, axis=-1)

Transition = namedtuple("Transition", field_names=["state", "state_next", "action", "reward", "done"])

class DQNAgent:
    def __init__(self, model, target_model, env, buffer_size):
        self.model = model
        self.target_model = target_model
        self.env = env
        self.buffer_size = buffer_size
        self.state_buffer = np.empty([buffer_size, self.env.reset().shape[0]], dtype=np.float32)
        self.state_next_buffer = np.empty([buffer_size, self.env.reset().shape[0]], dtype=np.float32)
        self.action_buffer = np.empty(buffer_size, dtype=np.int8)
        self.reward_buffer = np.empty(buffer_size, dtype=np.float32)
        self.done_buffer = np.empty(buffer_size, dtype=np.int8)
        self.next_buffer_index = 0
        self.num_transition = 0

    def evaluation(self, rounds):
        # play #rounds, each round take #max_steps, calc avg_steps that model can take before done
        steps = 0
        for i in range(rounds):
            state, done, step = self.env.reset(), False, 0
            while not done:
                action, _ = self.model.get_action_value(state[None])
                state, _, done, _ = self.env.step(action[0])
                step += 1
            steps += step
        return steps / rounds

    def evalation1(self, render=True):
        obs, done, ep_reward = self.env.reset(), False, 0
        # one episode until done
        while not done:
            action, q_values = self.model.get_action_value(obs[None])  # Using [None] to extend its dimension (4,) -> (1, 4)
            obs, reward, done, info = self.env.step(action[0])
            ep_reward += reward
            if render:  # visually show
                self.env.render()
            time.sleep(0.05)
        env.close()
        return ep_reward

    def train(self, steps, batch_size, gamma, target_update_iter,
              learning_rate, epsilon, epsilon_decay, min_epsilon, start_learning_steps):
        # using "compile" cause slow prediction
        opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, clipvalue=10.0)  # do gradient clip
        state = self.env.reset()
        for step in range(1, steps+1):
            # play one time
            action, value = self.model.get_action_value(state[None])
            epsilon_ = max(min_epsilon, epsilon * np.power(epsilon_decay, step))
            action = self._get_action(action[0], epsilon_)
            next_state, reward, done, _ = self.env.step(action)
            # store to buffer
            self._store_to_replay_buffer(Transition(state, next_state, action, reward, done))
            if step > start_learning_steps:
                # make one train step
                loss = self._train_step(opt, batch_size, gamma)
                if step % 1000 == 0:
                    eprint("step={}, loss={}".format(step, loss))

            if step % target_update_iter == 0:
                self._update_target_model()

            if done:
                state = self.env.reset()
            else:
                state = next_state

    def _get_action(self, action, epsilon):
        if np.random.rand() < epsilon:
            return self.env.action_space.sample()
        return action

    def _update_target_model(self):
        self.target_model.set_weights(self.model.get_weights())

    def _train_step(self, opt, batch_size, gamma):
        # take a batch from buffer
        if batch_size > self.num_transition:
            sample_indices = np.arange(self.num_transition)
            np.random.shuffle(sample_indices)
        else:
            sample_indices = np.random.randint(0, self.num_transition, batch_size)

        batch_state = self.state_buffer[sample_indices]
        batch_state_next = self.state_next_buffer[sample_indices]
        batch_action = self.action_buffer[sample_indices]
        reward_batch = self.reward_buffer[sample_indices]
        done_batch = self.done_buffer[sample_indices]

        # next line should not be in tape：ValueError: Could not pack sequence.
        _, q_state_next = self.target_model.get_action_value(batch_state_next)
        with tf.GradientTape() as tape:
            q_state_indices = np.c_[np.arange(batch_action.shape[-1]), batch_action]
            q_state = tf.gather_nd(self.model(batch_state), q_state_indices)
            # if is not done, y = 1 + gamma * q_state_next
            # if is done, y = -99
            y = reward_batch - 100 * done_batch + gamma * q_state_next * (1 - done_batch)
            loss = tf.keras.losses.mean_squared_error(y, q_state)
            # calculate the gradients using our tape and then update the
            # model weights
            grads = tape.gradient(loss, self.model.trainable_variables)
            opt.apply_gradients(zip(grads, self.model.trainable_variables))
            return loss

    def _store_to_replay_buffer(self, transition):
        idx = self.next_buffer_index
        self.state_buffer[idx] = transition.state
        self.state_next_buffer[idx] = transition.state_next
        self.action_buffer[idx] = transition.action
        self.reward_buffer[idx] = transition.reward
        self.done_buffer[idx] = transition.done
        self.next_buffer_index = (idx + 1) % self.buffer_size
        self.num_transition = min(self.num_transition + 1, self.buffer_size)

def train_and_evaluate(env, buffer_size, steps, batch_size, gamma, target_update_iter,
                       learning_rate, epsilon, epsilon_decay, min_epsilon, start_learning_steps,
                       rounds=10, play_after_learn=False):
    num_actions = env.action_space.n
    reward_list = []
    agent = None
    for i in range(rounds):
        tf.keras.backend.clear_session()
        model = Model(num_actions)
        target_model = Model(num_actions)
        agent = DQNAgent(model, target_model, env, buffer_size=buffer_size)
        agent.train(steps, batch_size, gamma, target_update_iter, learning_rate, epsilon,
                    epsilon_decay, min_epsilon, start_learning_steps)
        reward_list.append(agent.evaluation(10))
    print(buffer_size, steps, batch_size, gamma, target_update_iter,
          learning_rate, epsilon, epsilon_decay, start_learning_steps)
    reward = sum(reward_list) / rounds
    print("reward: {}".format(reward))
    if play_after_learn:
        agent.evalation1(render=True)
    return reward

if __name__ == '__main__':
    env = gym.make("CartPole-v0")
    # train_and_evaluate(env, buffer_size=100, steps=5000, batch_size=5, gamma=0.9835, target_update_iter=400,
    #                    learning_rate=0.002811, epsilon=0.1316, epsilon_decay=0.9962, min_epsilon=0.007501,
    #                    start_learning_steps=100, rounds=1, play_after_learn=True)
    tf.keras.backend.clear_session()
    model = Model(env.action_space.n)
    target_model = Model(env.action_space.n)
    agent = DQNAgent(model, target_model, env, buffer_size=100)
    print(agent.evaluation(1))


import datetime
import logging
import pandas as pd

from covid19_scrapers.utils.http import get_json
from covid19_scrapers.scraper import ScraperBase

_logger = logging.getLogger(__name__)

class Indiana(ScraperBase):
    """Indiana lists their COVID-19 case demographics dataset on their
    state health department's open data portal:
    https://hub.mph.in.gov/dataset/62ddcb15-bbe8-477b-bb2e-175ee5af8629

    This includes both the data as an Excel file and a data dictionary.
    """

    DATA_URL = 'https://hub.mph.in.gov/dataset/62ddcb15-bbe8-477b-bb2e-175ee5af8629/resource/2538d7f1-391b-4733-90b3-9e95cd5f3ea6/download/covid_report_demographics.xlsx'
    METADATA_URL = 'https://hub.mph.in.gov/api/3/action/resource_show?id=2538d7f1-391b-4733-90b3-9e95cd5f3ea6'

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _scrape(self, **kwargs):
        _logger.debug('Find the update date')
        metadata = get_json(self.METADATA_URL)
        date = datetime.datetime.fromisoformat(
            metadata['result']['last_modified']).date()
        _logger.info(f'Processing data for {date}')

        _logger.debug('Read in the file')
        df_in = pd.read_excel(
            self.DATA_URL,
            sheet_name='Race'
        ).set_index(
            'RACE'
        )

        total_cases = df_in['COVID_COUNT'].sum()
        total_deaths = df_in['COVID_DEATHS'].sum()
        aa_cases = df_in.loc['Black or African American', 'COVID_COUNT']
        aa_deaths = df_in.loc['Black or African American', 'COVID_DEATHS']
        aa_cases_pct = df_in.loc['Black or African American',
                                 'COVID_COUNT_PCT']
        aa_deaths_pct = df_in.loc['Black or African American',
                                  'COVID_DEATHS_PCT']

        return [self._make_series(
            date=date,
            cases=total_cases,
            deaths=total_deaths,
            aa_cases=aa_cases,
            aa_deaths=aa_deaths,
            pct_aa_cases=aa_cases_pct,
            pct_aa_deaths=aa_deaths_pct,
            pct_includes_unknown_race=True,
            pct_includes_hispanic_black=True,
        )]


import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer

class DataConverter(object):

    def __init__(self, target_data_list):
        self.target_data_list = target_data_list

    def convert_text_dictionary(self, text_feature_dict):

        count_vectorizer = CountVectorizer()
        feature_vectors = count_vectorizer.fit_transform(
            [' '.join(text_feature_dict.values())]
        )
        vocabulary = count_vectorizer.get_feature_names()

        keyword_dic = dict()
        for key, value in zip(vocabulary, feature_vectors.toarray()[0]):
            keyword_dic.update({key: value})

        return keyword_dic

    def convert_to_panda(self):

        data_df = None
        cost_df = None

        for target_data in self.target_data_list[:20]:

            # unpack target data
            id = target_data['id']
            cost = target_data['cost']

            # get feature dictionary
            feature_dictionary = dict()
            text_feature_dict = target_data['feature_dict']['text_feature']
            feature_dictionary = self.convert_text_dictionary(text_feature_dict)
            value_feature_dict = target_data['feature_dict']['value_feature']
            keyword_feature_dict = target_data['feature_dict']['keyword_feature']

            if data_df is None:
                data_df = pd.DataFrame(
                    [feature_dictionary.values()],
                    index=[id],
                    columns=feature_dictionary.keys()
                )
                cost_df = pd.DataFrame(
                    [cost],
                    index=[id],
                    columns=['cost']
                )
            else:
                additional_columns = list(set(feature_dictionary.keys()) - set(data_df.columns))
                additional_values = []
                for i in additional_columns:
                    additional_values.append(feature_dictionary[i])
                empty_df = pd.DataFrame(
                    [[0] * len(additional_columns)],
                    index=data_df.index,
                    columns=additional_columns
                )
                data_df = pd.concat([data_df, empty_df], axis=1)
                additional_data_values = []
                for column in data_df.columns:
                    if column in feature_dictionary.keys():
                        additional_data_values.append(feature_dictionary[column])
                    else:
                        additional_data_values.append(0)
                additional_data_df = pd.DataFrame(
                    [additional_data_values],
                    index=[id],
                    columns=data_df.columns
                )
                data_df = pd.concat([data_df, additional_data_df])
                additional_cost_df = pd.DataFrame(
                    [cost],
                    index=[id],
                    columns=['cost']
                )
                cost_df = pd.concat([cost_df, additional_cost_df])

        return {'data_df': data_df, 'cost_df': cost_df}


from django.http import HttpResponseRedirect
from django.shortcuts import render
from .forms import CodeForm
import clues.models as cModels

def index(request):
    if request.method == 'POST':
        form = CodeForm(request.POST)
        if form.is_valid():
            code = form.cleaned_data['code'].lower()
            if cModels.Clue.objects.filter(codeword_1=code).exists():
                print('Code {} found.'.format(code))
                c = cModels.Clue.objects.get(codeword_1=code)
                redirect = c.view_1
                c.viewed_1 = True
                c.save()
                return HttpResponseRedirect('/clues/{}'.format(redirect))
            elif cModels.Clue.objects.filter(codeword_2=code).exists():
                print('Code {} found.'.format(code))
                c = cModels.Clue.objects.get(codeword_2=code)
                redirect = c.view_2
                c.viewed_2 = True
                c.save()
                return HttpResponseRedirect('/clues/{}'.format(redirect))
            else:
                print('Code {} not found.'.format(code))
                return HttpResponseRedirect('/clues/thanks')
    else:
        form = CodeForm()

    return render(request, 'index.html', {'form': form})

def landing(request, page):
    return render(request, '{}.html'.format(page))


import re

from wagtail.embeds.finders.base import EmbedFinder
from wagtail.embeds.exceptions import EmbedNotFoundException

from urllib import request as urllib_request
from urllib.error import URLError
from urllib.parse import urlencode
from urllib.request import Request

class RtvEmbedFinder(EmbedFinder):
    def __init__(self, providers=None, options=None):
        self._endpoints = {}

        for provider in providers:
            patterns = []

            endpoint = provider["endpoint"]

            for url in provider["urls"]:
                patterns.append(re.compile(url))

            self._endpoints[endpoint] = patterns

    def _get_endpoint(self, url):
        for endpoint, patterns in self._endpoints.items():
            for pattern in patterns:
                if re.match(pattern, url):
                    return endpoint

    def accept(self, url):
        """
        Returns True if this finder knows how to fetch an embed for the URL.

        This should not have any side effects (no requests to external servers)
        """

        # check if url matches any of the providers
        return self._get_endpoint(url) is not None

    def find_embed(self, url, max_width=None):
        """
        Takes a URL and max width and returns a dictionary of information about the
        content to be used for embedding it on the site.

        This is the part that may make requests to external APIs.
        """

        return {
            'title': "",
            'author_name': "Multimedijski center RTV Slovenija",
            'provider_name': "RTVSLO.si",
            'type': "video",
            'thumbnail_url': "",
            'width': "",
            'height': "",
            'html': f'<iframe class="custom-video-embed" src="{url}"></iframe>',
        }