import torch
import math
import torch.nn as nn
import torch.nn.functional as F
from typing import List

class Convolution(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(Convolution, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 3, 1, 1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_ch, out_ch, 3, 1, 1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(inplace=True)
        )

    def forward(self, input):
        return self.conv(input)

class Curvature(torch.nn.Module):
    def __init__(self, ratio):
        super(Curvature, self).__init__()
        weights = torch.tensor([[[[-1/16, 5/16, -1/16], [5/16, -1, 5/16], [-1/16, 5/16, -1/16]]]])
        self.weight = torch.nn.Parameter(weights).cuda()
        self.ratio = ratio

    def forward(self, x):
        B, C, H, W = x.size()
        x_origin = x
        x = x.reshape(B*C,1,H,W)
        out = F.conv2d(x, self.weight)
        out = torch.abs(out)
        p = torch.sum(out, dim=-1)
        p = torch.sum(p, dim=-1)
        p=p.reshape(B, C)

        _, index = torch.topk(p, int(self.ratio*C), dim=1)
        selected = []
        for i in range(x_origin.shape[0]):
            selected.append(torch.index_select(x_origin[i], dim=0, index=index[i]).unsqueeze(0))
        selected = torch.cat(selected, dim=0)

        return selected

class Entropy_Hist(nn.Module):
    def __init__(self, ratio, win_w=3, win_h=3):
        super(Entropy_Hist, self).__init__()
        self.win_w = win_w
        self.win_h = win_h
        self.ratio = ratio

    def calcIJ_new(self, img_patch):
        total_p = img_patch.shape[-1] * img_patch.shape[-2]
        if total_p % 2 != 0:
            tem = torch.flatten(img_patch, start_dim=-2, end_dim=-1)
            center_p = tem[:, :, :, int(total_p / 2)]
            mean_p = (torch.sum(tem, dim=-1) - center_p) / (total_p - 1)
            if torch.is_tensor(img_patch):
                return center_p * 100 + mean_p
            else:
                return (center_p, mean_p)
        else:
            print("modify patch size")

    def histc_fork(ij):
        BINS = 256
        B, C = ij.shape
        N = 16
        BB = B // N
        min_elem = ij.min()
        max_elem = ij.max()
        ij = ij.view(N, BB, C)

        def f(x):
            with torch.no_grad():
                res = []
                for e in x:
                    res.append(torch.histc(e, bins=BINS, min=min_elem, max=max_elem))
                return res
        futures : List[torch.jit.Future[torch.Tensor]] = []

        for i in range(N):
            futures.append(torch.jit.fork(f, ij[i]))

        results = []
        for future in futures:
            results += torch.jit.wait(future)
        with torch.no_grad():
            out = torch.stack(results)
        return out

    def forward(self, img):
        with torch.no_grad():
            B, C, H, W = img.shape
            ext_x = int(self.win_w / 2) # 考虑滑动窗口大小，对原图进行扩边，扩展部分长度
            ext_y = int(self.win_h / 2)

            new_width = ext_x + W + ext_x # 新的图像尺寸
            new_height = ext_y + H + ext_y

            # 使用nn.Unfold依次获取每个滑动窗口的内容
            nn_Unfold=nn.Unfold(kernel_size=(self.win_w,self.win_h),dilation=1,padding=ext_x,stride=1)
            # 能够获取到patch_img，shape=(B,C*K*K,L),L代表的是将每张图片由滑动窗口分割成多少块---->28*28的图像，3*3的滑动窗口，分成了28*28=784块
            x = nn_Unfold(img) # (B,C*K*K,L)
            x= x.view(B,C,3,3,-1).permute(0,1,4,2,3) # (B,C*K*K,L) ---> (B,C,L,K,K)
            ij = self.calcIJ_new(x).reshape(B*C, -1) # 计算滑动窗口内中心的灰度值和窗口内除了中心像素的灰度均值,(B,C,L,K,K)---> (B,C,L) ---> (B*C,L)

            fij_packed = self.histc_fork(ij)
            p = fij_packed / (new_width * new_height)
            h_tem = -p * torch.log(torch.clamp(p, min=1e-40)) / math.log(2)

            a = torch.sum(h_tem, dim=1) # 对所有二维熵求和，得到这张图的二维熵
            H = a.reshape(B,C)

            _, index = torch.topk(H, int(self.ratio*C), dim=1) # Nx3
        selected = []
        for i in range(img.shape[0]):
            selected.append(torch.index_select(img[i], dim=0, index=index[i]).unsqueeze(0))
        selected = torch.cat(selected, dim=0)

        return selected

class Network(nn.Module):
    def __init__(self, in_ch=3, mode='ori', ratio=None):
        super(Network, self).__init__()
        self.mode = mode
        if self.mode == 'ori':
            self.ratio = [0,0]
        if self.mode == 'curvature':
            self.ratio = ratio
            self.ife1 = Curvature(self.ratio[0])
            self.ife2 = Curvature(self.ratio[1])
        if self.mode == 'entropy':
            self.ratio = ratio
            self.ife1 = Entropy_Hist(self.ratio[0])
            self.ife2 = Entropy_Hist(self.ratio[1])

        # ---- U-Net ----
        self.conv1 = Convolution(in_ch, 64)
        self.pool1 = nn.MaxPool2d(2)  # feature map = shape(m/2,n/2,64)
        self.conv2 = Convolution(64, 128)
        self.pool2 = nn.MaxPool2d(2)  # feature map = shapem/4,n/4,128)
        self.conv3 = Convolution(128, 256)
        self.pool3 = nn.MaxPool2d(2)  # feature map = shape(m/8,n/8,256)
        self.conv4 = Convolution(256, 512)
        self.pool4 = nn.MaxPool2d(2)  # feature map = shape(m/16,n/16,512)

        self.conv5 = Convolution(512, 1024)  # feature map = shape(m/16,n/16,1024)

        self.up_conv1 = nn.ConvTranspose2d(in_channels=1024, out_channels=512, kernel_size=2, stride=2, padding=0, output_padding=0)
        self.conv6 = Convolution(1024, 512)  # feature map = shape(m/8,n/8,512)
        self.up_conv2 = nn.ConvTranspose2d(512, 256, 2, 2, 0, 0)
        self.conv7 = Convolution(int(256*(2+self.ratio[1])), 256)  # feature map = shape(m/4,n/4,256）
        self.up_conv3 = nn.ConvTranspose2d(256, 128, 2, 2, 0, 0)
        self.conv8 = Convolution(int(128*(2+self.ratio[0])), 128)  # feature map = shape(m/2,n/2,128）
        self.up_conv4 = nn.ConvTranspose2d(128, 64, 2, 2, 0, 0)
        self.conv9 = Convolution(128, 64)  # feature map = shape(m,n,64)

        self.out_conv1 = nn.Conv2d(64, 1, 1, 1, 0)

    def forward(self, x):
        c1 = self.conv1(x)
        p1 = self.pool1(c1)

        c2 = self.conv2(p1)
        p2 = self.pool2(c2)

        c3 = self.conv3(p2)
        p3 = self.pool3(c3)

        c4 = self.conv4(p3)
        p4 = self.pool4(c4)

        c5 = self.conv5(p4)

        if self.mode != 'ori':
            c2 = torch.cat([c2, self.ife1(c2)])
            c3 = torch.cat([c3, self.ife2(c3)])

        up1 = self.up_conv1(c5)
        merge1 = torch.cat([up1, c4], dim=1)
        c6 = self.conv6(merge1)
        up2 = self.up_conv2(c6)
        merge2 = torch.cat([up2, c3], dim=1)
        c7 = self.conv7(merge2)
        up3 = self.up_conv3(c7)
        merge3 = torch.cat([up3, c2], dim=1)
        c8 = self.conv8(merge3)
        up4 = self.up_conv4(c8)
        merge4 = torch.cat([up4, c1], dim=1)
        c9 = self.conv9(merge4)

        S_g_pred = self.out_conv1(c9)

        return S_g_pred


from os import system
import matplotlib
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.collections import PolyCollection

from mpl_toolkits.axes_grid import make_axes_locatable

##############################################################################
# matplotlib configuration

linewidth = 2.0
fontsize = 12

params = {  # 'backend': 'ps',
    'axes.labelsize': fontsize,
    'text.fontsize': fontsize,
    'legend.fontsize': 0.9*fontsize,
    'xtick.labelsize': 0.9*fontsize,
    'ytick.labelsize': 0.9*fontsize,
    'text.usetex': False,
    # 'figure.figsize': fig_size
}

matplotlib.rcParams.update(params)

markers = ['o', 's', '^', 'd', 'v', '*', 'h', '<', '>']
markersize = 8
nodesize = 1000
##############################################################################

def init_plot(is_tight_layout=False, ind_fig=0, **kwargs):
    plt.close("all")
    fig = plt.figure(ind_fig, **kwargs)
    ax = fig.add_subplot(111)
    if is_tight_layout:
        fig.tight_layout()
    return ax

def new_plot(is_tight_layout=False, ind_fig=0):

    fig = plt.figure(ind_fig)
    ax = fig.add_subplot(111)
    if is_tight_layout:
        fig.tight_layout()
    ind_fig += 1
    return ax, ind_fig

def save_fig(figname, is_adjust_border=False):
    #ffigname = figname+".png"
    # plt.savefig(ffigname,format='PNG')

    ffigname = figname+".pdf"
    if is_adjust_border:
        plt.subplots_adjust(left=0.12, bottom=0.1, right=0.86, top=0.9, wspace=0.2, hspace=0.2)

    plt.savefig(figname+".pdf", format='PDF')
    # plt.savefig(figname+".eps",format='eps',transparent=True)
    #system("ps2pdf -dEPSCrop "+figname+".eps "+figname+".pdf")
    #system("rm "+figname+".eps")
    return ffigname


from pyspark.ml.classification import NaiveBayes
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()
spark.sparkContext.setLogLevel("ERROR")

data = spark.read.format("libsvm").load("file:///usr/lib/spark/data/mllib/sample_libsvm_data.txt")

splits = data.randomSplit([0.6, 0.4], 1234)
train  = splits[0]
test   = splits[1]

nb          = NaiveBayes(smoothing=1.0, modelType="multinomial")
model       = nb.fit(train)
predictions = model.transform(test)
predictions.show()

evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction",metricName="accuracy")
accuracy  = evaluator.evaluate(predictions)
print("Test set accuracy = " + str(accuracy))

spark.stop()


import pickle
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, accuracy_score
from tensorflow import keras
import matplotlib.pyplot as plt
import tensorflow_addons as tfa
import health_doc
import matplotlib.pyplot as plt
import gc
from imp import reload
from doc_preprocessing import get_data_from_kfold

import BERT
reload(BERT)
from BERT import make_model, model_fit, model_save, model_load
from BERT import get_tokenizer, get_tokenized_data, get_model_result, calc_score

# model
#   0: Normal multi-label classification
#   1: Knowledge Distillation
mode = 0

if (mode):
    # ### Get Teacher model prediction
    with open('id_teacher_predict','rb') as f:
        id_teacher_predict = pickle.load(f)

if __name__ == '__main__':
    # ### Loading HealthDoc dataset
    dataset_path = "../dataset/HealthDoc/"
    dataset_id, dataset_label, dataset_content, dataset_label_name = health_doc.loading(dataset_path)

    # ### Loading K-fold list
    with open('k_id', 'rb') as f:
        k_id = pickle.load(f)
    with open('k_label', 'rb') as f:
        k_label = pickle.load(f)

    K = len(k_id)

    tokenizer = get_tokenizer() # get BERT tokenizer

    for cv_times in range(10):
        cv_micro_f1 = []
        cv_macro_f1 = []
        cv_accuray = []
        cv_weighted_f1 = []
        cv_label_f1 = []

        for testing_time in range(K):
            # ### Split data for train and test
            subset_test = [testing_time]
            subset_train = np.delete(np.arange(K), subset_test)
            x_train, y_train = get_data_from_kfold(k_id, k_label, subset_train)
            x_test, y_test = get_data_from_kfold(k_id, k_label, subset_test)

            model_path = f'/content/model/{subset_test[0]}/'

            # ### Training  Model
            #x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.15)

            # get tokenized data with BERT input format
            x_train_vec = get_tokenized_data(x_train, dataset_content, tokenizer)
            x_test = get_tokenized_data(x_test, dataset_content, tokenizer)
            #x_val = getTokenized(x_val, dataset_content, tokenizer)

            tf.keras.backend.clear_session()
            model = make_model(9)

            if (mode):
                y_train_teacher = np.empty(x_train.shape+(9,))
                for i, x in enumerate(x_train):
                    y_train_teacher[i,:] = id_teacher_predict[x]

                print('Training Multi-label model with KD')
                history = model_fit(model, x_train_vec, y_train_teacher)
            else:
                print('Training Multi-label model without KD')
                history = model_fit(model, x_train_vec, y_train)

            gc.collect()

            # ### Predict Result
            y_pred = get_model_result(model, x_test)

            # ### Calculate Predict Reslut
            micro_f1, macro_f1, weighted_f1, subset_acc = calc_score(y_test, y_pred)

            cv_micro_f1.append(micro_f1)
            cv_macro_f1.append(macro_f1)
            cv_weighted_f1.append(weighted_f1)
            cv_accuray.append(subset_acc)

            label_f1=[]
            for i, label_name in enumerate(dataset_label_name):
                label_f1.append(f1_score(y_test[:,i], y_pred[:,i]))
                print(f'{label_name:<15}:{label_f1[-1]: .4f}')
            cv_label_f1.append(label_f1)

        with open('multi-times cv result.csv', 'a') as f:
            f.write(f'{sum(cv_micro_f1)/K: .4f},')
            f.write(f'{sum(cv_macro_f1)/K: .4f},')
            f.write(f'{sum(cv_weighted_f1)/K: .4f},')
            f.write(f'{sum(cv_accuray)/K: .4f},')

            label_f1_mean = np.mean(cv_label_f1, axis=0)
            for f1_mean in label_f1_mean:
                f.write(f'{f1_mean: .4f},')
            f.write('\n')


# -*- coding: utf-8 -*-
"""
Created on Wed Dec  5 16:42:07 2018

@author: lud
"""
import matplotlib
#import matplotlib.pyplot as plt
matplotlib.use('TkAgg')

from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
# implement the default mpl key bindings
from matplotlib.backend_bases import key_press_handler
from matplotlib.figure import Figure

import tkinter as Tk

from mpl_toolkits.mplot3d.art3d import Poly3DCollection
import numpy as np
import pandas as pd
from argparse import ArgumentParser
import os

def cuboid_data2(o, size=(1,1,1)):
    X = [[[0, 1, 0], [0, 0, 0], [1, 0, 0], [1, 1, 0]],
         [[0, 0, 0], [0, 0, 1], [1, 0, 1], [1, 0, 0]],
         [[1, 0, 1], [1, 0, 0], [1, 1, 0], [1, 1, 1]],
         [[0, 0, 1], [0, 0, 0], [0, 1, 0], [0, 1, 1]],
         [[0, 1, 0], [0, 1, 1], [1, 1, 1], [1, 1, 0]],
         [[0, 1, 1], [0, 0, 1], [1, 0, 1], [1, 1, 1]]]
    X = np.array(X).astype(float)
    for i in range(3):
        X[:,:,i] *= size[i]
    X += np.array(o)
    return X

def plotCubeAt2(positions,sizes=None,colors=None, **kwargs):
    if not isinstance(colors,(list,np.ndarray)): colors=["C0"]*len(positions)
    if not isinstance(sizes,(list,np.ndarray)): sizes=[(1,1,1)]*len(positions)
    g = []
    for p,s,c in zip(positions,sizes,colors):
        g.append( cuboid_data2(p, size=s) )
    return Poly3DCollection(np.concatenate(g),
                            facecolors=np.repeat(colors,6), **kwargs)

def main(path, width, depth, height):
    #get all data files
    source_files = []
    for file in os.listdir(path):
        if file.endswith(".csv"):
            source_files.append(os.path.join(path, file))

    #get data
    def getData(df):
        if len(df.columns < 7):
            df['6'] = 0
        sizes = [tuple(x) for x in df.iloc[:,[1,2,3]].values]
        positions = [tuple(x) for x in df.iloc[:,[4,5,6]].values]
        colors = ["limegreen"]*df.shape[0]
        pc = plotCubeAt2(positions,sizes,colors=colors, edgecolor="k", linewidth = 0.4)
        return pc

    #create figure

    fig = Figure()
    root = Tk.Tk()
    root.wm_title("Plot boxes")
    canvas = FigureCanvasTkAgg(fig, master=root)
    ax = fig.add_subplot(111,projection='3d')
    ax.set_aspect('equal')
    ax.set_xlim([0,width])
    ax.set_ylim([0,depth])
    ax.set_zlim([0,height])

    if len(source_files) > 0:
        box_data = pd.read_csv(source_files[0], header = None)
    else:
        box_data = pd.DataFrame(np.full((1,6),0,dtype = int))
    ax.add_collection3d(getData(box_data))

    canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)

    toolbar = NavigationToolbar2TkAgg(canvas, root)
    toolbar.update()
    canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)

    def refresh(df):
        ax.collections.clear()
        ax.add_collection(getData(df))
        canvas.draw()

    def ok():
        newfile = tkvar.get()
        box_data = pd.read_csv(newfile, header = None)
        refresh(box_data)

    def option_changed(*args):
        newfile = tkvar.get()
        box_data = pd.read_csv(newfile, header = None)
        refresh(box_data)

    # Create a Tkinter variable
    tkvar = Tk.StringVar(root)
    if len(source_files) > 0:
        tkvar.set(source_files[0])
    else:
        tkvar.set('No file')
    tkvar.trace("w", option_changed)

    popupMenu = Tk.OptionMenu(root, tkvar, '', *source_files)
    popupMenu.pack(side=Tk.TOP)

    def on_key_event(event):
        print('you pressed %s' % event.key)
        key_press_handler(event, canvas, toolbar)

    canvas.mpl_connect('key_press_event', on_key_event)

    def _quit():
        root.quit()     # stops mainloop
        root.destroy()  # this is necessary on Windows to prevent
                        # Fatal Python Error: PyEval_RestoreThread: NULL tstate

    button = Tk.Button(master=root, text='Quit', command=_quit)
    button.pack(side=Tk.BOTTOM)

    root.mainloop()

# main('E:\\Projects\\BinPacking\\test',800,1200,2055)
if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument("-p", "--path", dest="layer_data_path",
                        help="find data from path", metavar="PATH")
    parser.add_argument("-w", "--width", dest="width", type = int, default=800,
                        help="plane width, default 800")
    parser.add_argument("-d", "--depth", dest="depth", type = int, default=1200,
                        help="plane depth, default 1200")
    parser.add_argument("-hei", "--height", dest="height", type = int, default=2055,
                        help="bin height, default 2055")
    args = parser.parse_args()
    main(args.layer_data_path, args.width, args.depth, args.height)


import scrapy

class KistaSpider(scrapy.Spider):
    name = "kista"

    def start_requests(self):
        urls = ['https://www.hemnet.se/bostader?location_ids%5B%5D=473377&item_types%5B%5D=bostadsratt',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        yield {
            'sold': response.css("span.result-type-toggle__sold-count::text").re(r'\d+'),
            'for_sell': response.css("span.result-type-toggle__for-sale-count::text").re(r'\d+')
        }


import os
import string
import json
from collections import namedtuple
from sys import stdout

from lex.oed.languagetaxonomy import LanguageTaxonomy
from apps.tm.models import Lemma, Wordform, Definition, Language, ProperName
from apps.tm.build import buildconfig

LEMMA_FIELDS = buildconfig.LEMMA_FIELDS
BlockData = namedtuple('BlockData', LEMMA_FIELDS)

def populate_db():
    """
    Populate the database table for Language, Lemma, Wordform, and Definition
    """
    stdout.write('Emptying the tables...\n')
    empty_tables()
    stdout.write('Populating Language records...\n')
    populate_language()
    stdout.write('Populating Lemma, Wordform, and Definition records...\n')
    populate_lexical()
    stdout.write('Populating ProperName records...\n')
    populate_proper_names()

def empty_tables():
    """
    Empty the database tables of any existing content
    """
    Wordform.objects.all().delete()
    Lemma.objects.all().delete()
    Definition.objects.all().delete()
    Language.objects.all().delete()
    ProperName.objects.all().delete()

def populate_language():
    """
    Populate the Language table
    """
    taxonomy = LanguageTaxonomy()
    taxonomy.families = set(buildconfig.LANGUAGE_FAMILIES)

    max_length = Language._meta.get_field('name').max_length

    language_objects = []
    for language in taxonomy.languages():
        name = language.name[:max_length]
        language_objects.append(Language(id=language.id, name=name, family=None))
    Language.objects.bulk_create(language_objects)

    for language in taxonomy.languages():
        family = taxonomy.family_of(language.name)
        if family is not None:
            src = Language.objects.get(id=language.id)
            target = Language.objects.get(id=family.id)
            src.family = target
            src.save()

def populate_lexical():
    """
    Populate the Lemma, Wordform, and Definition tables
    """
    in_dir = os.path.join(buildconfig.FORM_INDEX_DIR, 'refined')
    frequency_cutoff = buildconfig.FREQUENCY_CUTOFF

    taxonomy = LanguageTaxonomy()
    lemma_counter = 0
    definition_counter = 0

    for letter in string.ascii_lowercase:
        stdout.write('Inserting data for %s...\n' % letter)
        blocks = []
        in_file = os.path.join(in_dir, letter + '.json')
        with open(in_file, 'r') as filehandle:
            for line in filehandle:
                data = json.loads(line.strip())
                blocks.append(BlockData(*data))

        lemmas = []
        wordforms = []
        definitions = []
        for i, block in enumerate(blocks):
            lang_node = taxonomy.node(language=block.language)
            if lang_node is None:
                language_id = None
            else:
                language_id = lang_node.id

            if block.definition and block.f2000 < frequency_cutoff:
                definition_counter += 1
                definitions.append(Definition(id=definition_counter,
                                              text=block.definition[:100]))
                definition_id = definition_counter
            else:
                definition_id = None

            lemma_counter += 1
            lemmas.append(Lemma(id=lemma_counter,
                                lemma=block.lemma,
                                sort=block.sort,
                                wordclass=block.wordclass,
                                firstyear=block.start,
                                lastyear=block.end,
                                refentry=block.refentry,
                                refid=block.refid,
                                thesaurus_id=block.htlink,
                                language_id=language_id,
                                definition_id=definition_id,
                                f2000=_rounder(block.f2000),
                                f1950=_rounder(block.f1950),
                                f1900=_rounder(block.f1900),
                                f1850=_rounder(block.f1850),
                                f1800=_rounder(block.f1800),
                                f1750=_rounder(block.f1750),))

            for typelist in (block.standard_types,
                             block.variant_types,
                             block.alien_types):
                for typeunit in typelist:
                    wordforms.append(Wordform(sort=typeunit[0],
                                              wordform=typeunit[1],
                                              wordclass=typeunit[2],
                                              lemma_id=lemma_counter,
                                              f2000=_rounder(typeunit[4]),
                                              f1900=_rounder(typeunit[5]),
                                              f1800=_rounder(typeunit[6]),))

            if i % 1000 == 0:
                Definition.objects.bulk_create(definitions)
                Lemma.objects.bulk_create(lemmas)
                Wordform.objects.bulk_create(wordforms)
                definitions = []
                lemmas = []
                wordforms = []

        Definition.objects.bulk_create(definitions)
        Lemma.objects.bulk_create(lemmas)
        Wordform.objects.bulk_create(wordforms)

def populate_proper_names():
    """
    Populate the ProperName table
    """
    in_dir = os.path.join(buildconfig.FORM_INDEX_DIR, 'proper_names')
    in_file = os.path.join(in_dir, 'all.txt')
    names = []
    counter = 0
    with open(in_file) as filehandle:
        for line in filehandle:
            data = line.strip().split('\t')
            if len(data) == 3:
                counter += 1
                sortable, name, common = data
                if common.lower() == 'true':
                    common = True
                else:
                    common = False

                names.append(ProperName(lemma=name,
                                        sort=sortable,
                                        common=common))
                if counter % 1000 == 0:
                    ProperName.objects.bulk_create(names)
                    names = []

    ProperName.objects.bulk_create(names)

def _rounder(n):
    n = float('%.2g' % n)
    if n == 0 or n > 1:
        return int(n)
    else:
        return n


import datetime
import hashlib
import json
from urllib.parse import urlparse

import requests
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding

import config
import crypto

class Blockchain:
    def __init__(self, key_path=None):
        # Initialize a chain which will contain blocks
        self.chain = []  # a simple list containing blovks
        # Create a list which contains a list of transactions before they
        # are added to the block. Think of it as a cache of transactions which
        # happened, but are not yet written to a block in a blockchain.
        self.transactions = []
        # Create a genesis block - the first block
        # Previous hash is 0 because this is a genesis block!
        self.create_block(proof=1, previous_hash='0')
        # Create a set of nodes
        self.nodes = set()
        if key_path:
            self.private_key = crypto.load_private_key(key_path)
            self.address = self.generate_address(self.private_key.public_key())

    def create_block(self, proof, previous_hash):
        # Define block as a dictionary
        block = {'index': len(self.chain) + 1,
                 'timestamp': str(datetime.datetime.now()),
                 'proof': proof,
                 'previous_hash': previous_hash,
                 # Here we can add any additional data related to the currency
                 'transactions': self.transactions
                 }
        # Now we need to empty the transactions list, since all those transactions
        # are now contained in the block.
        self.transactions = []
        # Append block to the blockchain
        self.chain.append(block)
        return block

    def get_previous_block(self):
        return self.chain[-1]

    def get_address(self):
        return self.address

    def proof_of_work(self, previous_proof):
        new_proof = 1  # nonce value
        check_proof = False
        while check_proof is False:
            # Problem to be solved (this makes the minig hard)
            # operation has to be non-symetrical!!!
            hash_operation = hashlib.sha256(str(config.BLOCKCHAIN_PROBLEM_OPERATION_LAMBDA(
                previous_proof, new_proof)).encode()).hexdigest()
            # Check if first 4 characters are zeros
            if hash_operation[:len(config.LEADING_ZEROS)] == config.LEADING_ZEROS:
                check_proof = True
            else:
                new_proof += 1
        # Check proof is now true
        return new_proof

    def hash_of_block(self, block):
        # Convert a dictionary to string (JSON)
        encoded_block = json.dumps(block, sort_keys=True).encode()
        return hashlib.sha256(encoded_block).hexdigest()

    def is_chain_valid(self, chain):
        previous_block = chain[0]
        block_index = 1
        while block_index < len(chain):
            # 1 Check the previous hash
            block = chain[block_index]
            if block['previous_hash'] != self.hash_of_block(previous_block):
                return False
            # 2 Check all proofs of work
            previous_proof = previous_block['proof']
            proof = block['proof']
            hash_operation = hashlib.sha256(str(config.BLOCKCHAIN_PROBLEM_OPERATION_LAMBDA(
                previous_proof, proof)).encode()).hexdigest()
            if hash_operation[:len(config.LEADING_ZEROS)] != config.LEADING_ZEROS:
                return False
            # Update variables
            previous_block = block
            block_index += 1
        return True

    def add_transaction(self, sender, receiver, amount, private_key):
        # Create a transaction dictionary
        transaction = {
            'sender': sender,
            'receiver': receiver,
            'amount': amount
        }

        # Sign the transaction
        signature = private_key.sign(
            json.dumps(transaction, sort_keys=True).encode(),
            padding.PSS(
                mgf=padding.MGF1(hashes.SHA256()),
                salt_length=padding.PSS.MAX_LENGTH
            ),
            hashes.SHA256()
        )

        # Add the signature and public key to the transaction
        transaction['signature'] = signature
        transaction['public_key'] = private_key.public_key().public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )

        # Add the transaction to the list of transactions
        self.transactions.append(transaction)

        # Return the index of the next block in the blockchain
        previous_block = self.get_previous_block()

        return previous_block['index'] + 1

    def add_node(self, address):
        parsed_url = urlparse(address)
        # Add to the list of nodes
        # parsed_url() method returns ParseResult object which has an attribute netloc
        # which is in a format adress:port eg. 127.0.0.1:5000
        self.nodes.add(parsed_url.netloc)

    def replace_chain(self):
        network = self.nodes
        longest_chain = None
        max_length = len(self.chain)
        for node in network:
            # Find the largest chain (send a request)
            response = requests.get(f'http://{node}/get-chain')
            if response.status_code == 200:
                length = response.json()['length']
                chain = response.json()['chain']
                # Check chain if it is the longest one and also a valid one
                if length > max_length and self.is_chain_valid(chain):
                    max_length = length
                    longest_chain = chain
        if longest_chain:
            # Replace the chain
            self.chain = longest_chain
            return True
        # Otherwise, the chain is not replaced
        return False

    def save_blockchain(self, filename):
        with open(filename, 'w') as file:
            json.dump(self.chain, file, indent=4)

    def load_blockchain(self, filename):
        with open(filename, 'r') as file:
            self.chain = json.load(file)

    def generate_address(self, public_key):
        public_key_bytes = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return hashlib.sha256(public_key_bytes).hexdigest()

EP -- zadatak 03/bez master key/blockchain.py


from flask import Flask, render_template, request, redirect, url_for
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Movie

app = Flask(__name__)

engine = create_engine('sqlite:///books-collection.db?check_same_thread=False')
Base.metadata.bind = engine

DBSession = sessionmaker(bind=engine)
session = DBSession()

@app.route('/')
@app.route('/movies')
def showMovies():
    movies = session.query(Movie).all()
    return render_template("movies.html", movies=movies)

@app.route('/movies/new/', methods=['GET', 'POST'])
def newMovie():
    if request.method == 'POST':
        newMovie = Movie(title=request.form['name'], author=request.form['author'], cast=request.form['cast'], price=request.form['price'])
        session.add(newMovie)
        session.commit()
        return redirect(url_for('showMovies'))
    else:
        return render_template('newMovie.html')

    # Эта функция позволит нам обновить книги и сохранить их в базе данных.
@app.route("/movies/<int:movie_id>/edit/", methods=['GET', 'POST'])
def editMovie(movie_id):
    editedMovie = session.query(Movie).filter_by(id=movie_id).one()
    if request.method == 'POST':
        if request.form['name'] or request.form['author'] or request.form['cast'] or request.form['price']:
            editedMovie.title = request.form['name']
            editedMovie.title = request.form['author']
            editedMovie.title = request.form['cast']
            editedMovie.title = request.form['price']
            return redirect(url_for('showMovies'))
    else:
        return render_template('editMovie.html', movie=editedMovie)

    # Эта функция для удаления книг

@app.route('/movies/<int:movie_id>/delete/', methods=['GET', 'POST'])
def deleteMovie(movie_id):
    movieToDelete = session.query(Movie).filter_by(id=movie_id).one()
    if request.method == 'POST':
        session.delete(movieToDelete)
        session.commit()
        return redirect(url_for('showMovies', movie_id=movie_id))
    else:
        return render_template('deleteMovie.html', movie=movieToDelete)

if __name__ == '__main__':
    app.debug = True
    app.run(port=4996)


import ROOT
#from root_numpy import root2array, root2rec, tree2rec
import pylab,numpy,pickle
import matplotlib

pylab.rcParams['font.size'] = 14.0
pylab.rcParams['axes.labelsize']=18.0
pylab.rcParams['axes.titlesize']=20.0
pylab.rcParams['ytick.labelsize']='large'
pylab.rcParams['xtick.labelsize']='large'
pylab.rcParams['lines.markeredgewidth']=1.0
pylab.rc ('text', usetex=True)
pylab.rc ('font', family='serif')
pylab.rc ('font', serif='Computer Modern Roman')

log_sigma_days = numpy.array([-5,-4,-3,-2,-1,-0.52287874528033762,0,1])

### NEW GENIE 1460 Included ###
dec0_e3_foldedspectrum = (1072.916206382002,0)
dec16_e3_foldedspectrum = (1545.0315486757047,0)
dec30_e3_foldedspectrum = (1803.4879220886971,0)
dec45_e3_foldedspectrum = (1955.9670994116407,0)
dec60_e3_foldedspectrum = (2117.1599069802728,0)
dec75_e3_foldedspectrum = (2228.3197855702933,0)
sa_avg_foldedspectrum = (1654.0807981564465,0)

sys_adjustment = 0.89559693491089454

### Int(EffaE-3)  (JF,RH)###
#samp2_e3_foldedspectrum_sum = (1759.219287256351,0) ## 100 GeV flux equal to 1.0 GeV^-1 cm^-2 s^-1
#samp2_e35_foldedspectrum_sum = (2925.5560058208703,0) ##
#samp2_e25_foldedspectrum_sum = (1320.5883336274608,0) ##

sens_e3_dec0_meansrc_events = numpy.array([6.4656,6.70643,6.7344,7.38432,10.4106,13.2816,16.2928,28.1549])
sens_e3_dec16_meansrc_events = numpy.array([6.4384,6.62176,6.79315,7.4096,10.5558,13.0896,16.5709,30.3184])
sens_e3_dec30_meansrc_events = numpy.array([7.632,7.32,7.54048,8.00864,10.68,12.6272,16.0406,27.1056])
sens_e3_dec45_meansrc_events = numpy.array([6.86976,6.87104,7.09792,8.60768,11.3456,12.983,16.1408,27.0288])
sens_e3_dec60_meansrc_events = numpy.array([6.77216,6.54144,7.29088,8.584,11.0262,13.2019,15.5658,24.368])
sens_e3_dec75_meansrc_events = numpy.array([5.6608,5.64512,5.95296,7.37824,10.8947,12.7984,15.9766,28.8221])

ul_e3_dec0_meansrc_events = numpy.array([7.5456,8.09952,9.06432,11.376,17.5674,22.2304,29.9581,60.232])
ul_e3_dec16_meansrc_events = numpy.array([7.77754,8.51104,9.67872,11.8336,18.1984,23.208,30.528,64.568])
ul_e3_dec30_meansrc_events = numpy.array([8.95392,9.34349,10.2138,12.5501,18.1462,22.568,29.6342,59.744])
ul_e3_dec45_meansrc_events = numpy.array([8.45888,8.73325,9.74496,12.8112,19.0477,22.5107,29.5024,59.3357])
ul_e3_dec60_meansrc_events = numpy.array([8.17261,8.74912,10.1846,13.3968,19.3747,23.0784,30.0032,57.7504])
ul_e3_dec75_meansrc_events = numpy.array([7.30272,7.66144,8.52512,11.688,19.0272,24.0032,31.9216,64.608])

ilow_en_bins = pickle.load(open("./pickles/effarea_low_energy_bins.pkl",'r'))
high_en_bins = pickle.load(open("./pickles/effarea_high_energy_bins.pkl",'r'))

genie_avg_area = pickle.load(open("./pickles/g1460_numu_effarea_avg.pkl",'r'))
genie_dec0_area = pickle.load(open("./pickles/g1460_numu_effarea_dec0.pkl",'r'))
genie_dec16_area = pickle.load(open("./pickles/g1460_numu_effarea_dec16.pkl",'r'))
genie_dec30_area = pickle.load(open("./pickles/g1460_numu_effarea_dec30.pkl",'r'))
genie_dec45_area = pickle.load(open("./pickles/g1460_numu_effarea_dec45.pkl",'r'))
genie_dec60_area = pickle.load(open("./pickles/g1460_numu_effarea_dec60.pkl",'r'))
genie_dec75_area = pickle.load(open("./pickles/g1460_numu_effarea_dec75.pkl",'r'))

nugen_avg_area = pickle.load(open("./pickles/g1460_nugmu_effarea_avg.pkl",'r'))
nugen_dec0_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec0.pkl",'r'))
nugen_dec16_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec16.pkl",'r'))
nugen_dec30_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec30.pkl",'r'))
nugen_dec45_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec45.pkl",'r'))
nugen_dec60_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec60.pkl",'r'))
nugen_dec75_area = pickle.load(open("./pickles/g1460_nugmu_effarea_dec75.pkl",'r'))

sa0 = 2*numpy.pi*((1-numpy.cos(numpy.deg2rad(95.))) - (1-numpy.cos(numpy.deg2rad(80.))))
sa16 = 2*numpy.pi*((1-numpy.cos(numpy.deg2rad(80.))) - (1-numpy.cos(numpy.deg2rad(65.))))
sa30 = 2*numpy.pi*((1-numpy.cos(numpy.deg2rad(65.))) - (1-numpy.cos(numpy.deg2rad(50.))))
sa45 = 2*numpy.pi*((1-numpy.cos(numpy.deg2rad(50.))) - (1-numpy.cos(numpy.deg2rad(35.))))
sa60 = 2*numpy.pi*((1-numpy.cos(numpy.deg2rad(35.))) - (1-numpy.cos(numpy.deg2rad(20.))))
sa75 = 2*numpy.pi*(1-numpy.cos(numpy.deg2rad(20.)))
saTotal = 2*numpy.pi*(1-numpy.cos(numpy.deg2rad(95.)))

sky_frac = [0.23989563791056959, 0.22901050354066707, 0.20251868181221927, 0.16222554659621455, 0.11087700847006936, 0.055472621670260208]

fluxnorm_dec16_e3 = ul_e3_dec16_meansrc_events/dec16_e3_foldedspectrum[0]
fluxnorm_dec0_e3 = ul_e3_dec0_meansrc_events/dec0_e3_foldedspectrum[0]
fluxnorm_dec30_e3 = ul_e3_dec30_meansrc_events/dec30_e3_foldedspectrum[0]
fluxnorm_dec45_e3 = ul_e3_dec45_meansrc_events/dec45_e3_foldedspectrum[0]
fluxnorm_dec60_e3 = ul_e3_dec60_meansrc_events/dec60_e3_foldedspectrum[0]
fluxnorm_dec75_e3 = ul_e3_dec75_meansrc_events/dec75_e3_foldedspectrum[0]

uls = [ul_e3_dec0_meansrc_events,ul_e3_dec16_meansrc_events,ul_e3_dec30_meansrc_events,ul_e3_dec45_meansrc_events,ul_e3_dec60_meansrc_events,ul_e3_dec75_meansrc_events]
event_ul_avg_list = [uls[i]*sky_frac[i] for i in range(len(sky_frac))]

event_ul_avg = numpy.array([0.,0.,0.,0.,0.,0.,0.,0.])
for listy in event_ul_avg_list:
        event_ul_avg+=listy

fluxnorm_sa_avg_e3 = event_ul_avg / sa_avg_foldedspectrum[0]

#fluxnorm_0 = sens_bdt0_e3_meansrc_events/samp2_e3_foldedspectrum_sum[0]
#fluxnorm_0_disco = disco_bdt0_e3_meansrc_events/samp2_e3_foldedspectrum_sum[0]

#fluxnorm_0_25 = sens_bdt0_e25_meansrc_events/samp2_e25_foldedspectrum_sum[0]
#fluxnorm_0_35 = sens_bdt0_e35_meansrc_events/samp2_e35_foldedspectrum_sum[0]

pylab.figure()
pylab.plot(log_sigma_days,event_ul_avg,'k-',lw=2,label="Averaged")
pylab.plot(log_sigma_days,ul_e3_dec0_meansrc_events,'k--',lw=2,label=r"$\delta=0^{\circ}$")
pylab.plot(log_sigma_days,ul_e3_dec30_meansrc_events,'k-.',lw=2,label=r"$\delta=30^{\circ}$")
pylab.plot(log_sigma_days,ul_e3_dec60_meansrc_events,'k:',lw=2,label=r"$\delta=60^{\circ}$")
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad (Days)$')
pylab.ylabel("NSrc Events")
pylab.axis([-5,1,3,60])
pylab.grid()
pylab.legend(loc="upper left")
matplotlib.pyplot.gcf().subplots_adjust(right=.85)
pylab.title(r"Upper Limit $E^{-3}$ 90% C.L.")
pylab.savefig("LowEnTransient_NEventUpperLimit_E3_G1460_MultiDec")

fig1=pylab.figure()
pylab.plot(log_sigma_days,event_ul_avg,'k-',lw=2)
#pylab.plot(0.77011529478710161,13.5279,"w*",ms=20.0,label="Most Significant Flare")
#pylab.plot(log_sigma_days,disco_bdt0_e3_meansrc_events,'k-',lw=2,label="Discovery Potential")
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad$ (Days)')
pylab.ylabel("NSrc Events")
pylab.axis([-5,1,0,62])
pylab.grid()
pylab.legend(loc="upper left")
matplotlib.pyplot.gcf().subplots_adjust(right=.85)
pylab.title(r"Upper Limit $E^{-3}$ 90$\%$ C.L.")
pylab.savefig("LowEnTransient_NEventUpperLimit_E3_G1460_Avg.pdf")

figgy=pylab.figure()
ax = figgy.add_subplot(111)
pylab.plot(log_sigma_days,fluxnorm_sa_avg_e3,'k-',lw=2,label=r"$E^{-3.0}$")
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad$ (Days)')
pylab.ylabel(r"$\frac{dN}{dE}$ @ 100 GeV ($10^{-2}$GeV$^{-1}$ cm$^{-2}$)")
pylab.axis([-5,1,0.00,0.037483054073961818])
pylab.yticks([0.0060456538828970677,0.012091307765794135,0.018136961648691202,0.024182615531588271,0.030228269414485337,0.036273923297382403],["0.6","1.21","1.81","2.42","3.02","3.63"])
ax.yaxis.tick_right()
ax.yaxis.set_label_position("right")
matplotlib.pyplot.gcf().subplots_adjust(right=.85)
pylab.grid()
#pylab.legend(loc="upper left")
pylab.title(r"Time-Integrated Flux Upper Limit $E^{-3}$")
pylab.savefig("LowEnTransient_FluxUpperLimit_E3_G1460_Avg.pdf")

figgy=pylab.figure()
ax = figgy.add_subplot(111)
pylab.plot(log_sigma_days,event_ul_avg,'k-',lw=2,label=r"$E^{-3.0}$")
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad$ (Days)')
pylab.ylabel("NSrc Events")
pylab.axis([-5,1,0.00,62])
pylab.yticks([  0.,  10.,  20.,  30.,  40.,  50.,  60.])
pylab.grid()
ax2 = ax.twinx()
ax2.set_ylim(0,0.037483054073961818)
ax2.set_xlim(-5,1)
ax2.set_yticks([0.0060456538828970677,0.012091307765794135,0.018136961648691202,0.024182615531588271,0.030228269414485337,0.036273923297382403])
ax2.set_yticklabels(["0.6","1.21","1.81","2.42","3.02","3.63"])
ax2.set_ylabel(r"$\frac{dN}{dE}$ @ 100 GeV ($10^{-2}$GeV$^{-1}$ cm$^{-2}$)")
matplotlib.pyplot.gcf().subplots_adjust(right=.85)
#pylab.legend(loc="upper left")
pylab.title(r"Time-Integrated Flux Upper Limit $E^{-3}$")
pylab.savefig("LowEnTransient_FluxUpperLimit_E3_G1460_Avg_DoubleY.pdf")

figgy=pylab.figure()
ax = figgy.add_subplot(111)
pylab.plot(log_sigma_days,fluxnorm_dec0_e3,'k--',lw=2,label=r"$\delta = 0^{\circ}$")
pylab.plot(log_sigma_days,fluxnorm_dec16_e3,'k-',lw=2,label=r"$\delta = 16^{\circ}$")
pylab.plot(log_sigma_days,fluxnorm_dec30_e3,'k-.',lw=2,label=r"$\delta = 30^{\circ}$")
pylab.plot(log_sigma_days,fluxnorm_dec60_e3,'k:',lw=2,label=r"$\delta = 60^{\circ}$")
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad$ (Days)')
pylab.ylabel(r"$\frac{dN}{dE}$ @ 100 GeV ($10^{-2}$GeV$^{-1}$ cm$^{-2}$)")
pylab.axis([-5,1,0.00,0.058])
pylab.yticks([0.00 ,  0.00828571,  0.01657143,  0.02485714,  0.03314286, 0.04142857, 0.04971429, 0.058],["0.0","0.83","1.7","2.5","3.3","4.1","5.0","5.8"])
ax.yaxis.tick_right()
ax.yaxis.set_label_position("right")
matplotlib.pyplot.gcf().subplots_adjust(right=.85)
pylab.grid()
pylab.legend(loc="upper left")
pylab.title(r"Time-Integrated Flux Upper Limit $E^{-3}$")
pylab.savefig("LowEnTransient_FluxUpperLimit_E3_G1460_MultiDec.pdf")

'''
pylab.figure(figsize=(10,8))
pylab.plot(log_sigma_days,fluxnorm_0,'b-',lw=2,label='Sensitivity (90% C.L.)')
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad (Days)$')
pylab.ylabel(r"$\frac{dN}{dE}$ [$GeV^{-1} cm^{-2} s^{-1}$] @ 100 GeV Pivot Energy")
#pylab.axis([-5,1,5e3,5e4])
pylab.yticks([0.001,0.005,0.01,0.015,0.02,0.025],["$1e-3$","$5.0e-3$","$1.0e-2$","1.5e-2","2.0e-2","2.5e-2"])
pylab.grid()
pylab.legend(loc="upper left")
pylab.title(r"Flux Sensitivity (MergedSim) $E^{-3}$")
pylab.savefig("LowEnTransient_FluenceSensitivity_E3_MergedSim_FinalCut")

pylab.figure(figsize=(10,8))
pylab.plot(log_sigma_days,fluxnorm_0,'b-',lw=2,label='Sensitivity (90% C.L.)')
pylab.plot(log_sigma_days,fluxnorm_0_disco,'k-',lw=2,label='Discovery Potential')
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad (Days)$')
pylab.ylabel(r"$\frac{dN}{dE}$ [$GeV^{-1} cm^{-2} s^{-1}$] @ 100 GeV Pivot Energy")
#pylab.axis([-5,1,5e3,5e4])
pylab.yticks([0.001,0.005,0.01,0.015,0.02,0.025],["$1e-3$","$5.0e-3$","$1.0e-2$","1.5e-2","2.0e-2","2.5e-2"])
pylab.grid()
pylab.legend(loc="upper left")
pylab.title(r"Flux Sensitivity (MergedSim) $E^{-3}$")
pylab.savefig("LowEnTransient_FluenceSensitivityAndDisco_E3_MergedSim_FinalCut")

pylab.figure(figsize=(10,8))
pylab.plot(log_sigma_days,merged_samp1_e2_meansrc_events,'g-',lw=2,label='Sample 1')
pylab.plot(log_sigma_days,merged_samp2_e2_meansrc_events,'b-',lw=2,label='Sample 2')
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad (Days)$')
pylab.ylabel("NSrc Events")
pylab.axis([-6,1,3,15])
pylab.grid()
pylab.title("Sensitivity (MergedSim)")
pylab.legend(loc='upper left')
pylab.savefig("LowEnTransient_DiscoPotential_E2_MergedSim_SampleComparison")

pylab.figure(figsize=(10,8))
pylab.plot(log_sigma_days,nugen_samp1_e2_meansrc_events,'g--',lw=2,label='Sample 1 (Nugen)')
pylab.plot(log_sigma_days,nugen_samp2_e2_meansrc_events,'b--',lw=2,label='Sample 2 (Nugen)')
pylab.plot(log_sigma_days,merged_samp1_e2_meansrc_events,'g-',lw=2,label='Sample 1 (MergedSim)')
pylab.plot(log_sigma_days,merged_samp2_e2_meansrc_events,'b-',lw=2,label='Sample 2 (MergedSim)')
pylab.xlabel(r'$Log_{10}(\sigma_{\omega})\quad (Days)$')
pylab.ylabel("NSrc Events")
pylab.axis([-6,1,3,15])
pylab.grid()
pylab.title("Sensitivity")
pylab.legend(loc='upper left')
pylab.savefig("LowEnTransient_DiscoPotential_E2_NugenANDMerged_SampleComparison")
'''


# 从爬虫生成的Excel表格中读取数据并生成词云图
import os
import sys
import PIL
import jieba
import openpyxl
import wordcloud
import configparser
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from collections import Counter
from multiprocessing import Pool

# 定义一些参数，参数的详细介绍见GitHub上的readme.md
config_file = 'config/config.ini'
config_Section_Name = 'GC_DEFAULT'  # 要读取的配置页名
stop_Word = ['!', '！', ':', '*', '，', ',', '？','《','》',
             '。', ' ', '的', '了', '是', '啊', '吗', '吧','这','你','我','他','就']  # 停用词表

def read_Danmu(workbook_Name, sheet_Name):  # 从Excel表中读取数据
    try:
        workbook = openpyxl.load_workbook(workbook_Name)
        worksheet = workbook[sheet_Name]  # 当然也可以通过索引读sheet,为了可读性选择用名称
        data = worksheet.iter_rows(values_only=1)
        return data
    #若报错，则返回空迭代器
    except openpyxl.utils.exceptions.InvalidFileException:
        print(f"输入文件的路径或格式错误，请打开{config_file}文件重新配置路径\n")
        return iter(())
    except KeyError:
        print(f"工作表页名错误，请检查Sheet的名字和{config_file}中是否一致\n")
        return iter(())
    except:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print(f"发生错误: {exc_type} - {exc_value}")
        return iter(())

def cut_words(row):
    try:
        # 每行第一列是弹幕，第二列是出现次数
        sentence = row[0]
        count = row[1]
        # 运用jieba 进行分词，将结果储存在Counter中，再将其中词语的出现次数翻count倍
        words = jieba.lcut(sentence)
        # 去除停用词表中的词
        cut_Words = pd.Series(words)
        cut_Words = cut_Words[~cut_Words.isin(stop_Word)]
        # 将分词存入计数器中
        new_Counter = Counter(cut_Words.tolist())
        for item in new_Counter:
            new_Counter[item] *= count              # 弹幕中词语出现数 = 弹幕出现次数*弹幕中词语出现次数
        return new_Counter
    except TypeError:
        return Counter()                            #遇见异常输入的情况，返回空计数器。

def generate_Word_Cloud(counter):  # 生成词云图
    try:
        if not counter:  # 如果计数器对象为空，则给出提示并退出函数
            return "输入的词频为空！"
        img = PIL.Image.open(pic_Path).convert('RGBA') # 解决灰度图像ERROR
        pic = np.array(img)
        image_colors = wordcloud.ImageColorGenerator(pic)
        word_Cloud = wordcloud.WordCloud(
            font_path=font_Path, mask=pic, width=WC_Width, height=WC_Height, mode="RGBA", background_color='white')
        word_Cloud.generate_from_frequencies(counter)
        plt.imshow(word_Cloud.recolor(color_func=image_colors),
                interpolation='bilinear')
        word_Cloud.to_file(output_Path)
        plt.axis('off')
        plt.show()
        return f"词云图生成完成，请前往{output_Path}查看"
    except FileNotFoundError : #pic_Path 或 font_Path错误的情况
        return f"图片或字体路径错误，请前往{config_file}核查。"
    except TypeError or ValueError : #WC_Width 或WC_Height类型或数组错误的情况
        return f"图片的Height与Width设置有误，请前往{config_file}核查。"
    except PIL.UnidentifiedImageError :
        return f"不支持该类型的图片，请修改图片路径。"
    except Exception as e:
        return f"生成词云图时发生错误：{e}"

def main():
    rows = read_Danmu(workbook_Name, sheet_Name)
    word_counts = Counter()
    # 利用线程池优化分词速度，在生成所有弹幕的词云图是能节省时间
    with Pool() as pool:
        cut_words_results = pool.map(cut_words, rows)
        for result in cut_words_results:
            word_counts.update(result)

    print(generate_Word_Cloud(word_counts))

if __name__ == "__main__":
    # 读取参数的配置
    config = configparser.ConfigParser()
    if not os.path.exists(config_file):
        print(f"配置文件 {config_file} 不存在！")
        exit(1)
    config.read(config_file)
    workbook_Name = config.get(config_Section_Name, 'workbook_name',
                               fallback='output/Top_20_danmu.xlsx')  # 要读取的Excel表的名称，默认为crawler.py生成的文件
    # 要读取的Excel表的页的名称，可从['Top 20', '所有弹幕']中选择
    sheet_Name = config.get(config_Section_Name, 'sheet_Name', fallback='所有弹幕')
    WC_Width = config.getint(
        config_Section_Name, 'WC_Width', fallback=1200)  # 词云图的宽度
    WC_Height = config.getint(
        config_Section_Name, 'WC_Height', fallback=1200)  # 词云图的高度
    font_Path = config.get(config_Section_Name, 'font_Path',
                           fallback="config/msyh.ttc")  # 字体存储路径
    pic_Path = config.get(config_Section_Name, 'pic_Path',
                          fallback="config/m.png")  # 词云背景图路径
    output_Path = config.get(
        config_Section_Name, 'output_Path', fallback="output/word_could.png")
    main()


import math
from datetime import datetime, timedelta
from decimal import Decimal

from financial.input import (
    FinancialDataInput,
    FinancialStatisticsInput,
    NullFinancialDataInput,
    NullFinancialStatisticsInput,
)
from financial.model import FinancialData, db

class FinancialDataInputValidationService:
    def __init__(self, request_args):
        self.validation_errors = []
        self.financial_data = self.validate_and_parse_financial_data_input(request_args)

    def validate_and_parse_financial_data_input(
        self, request_args
    ) -> FinancialDataInput | NullFinancialDataInput:
        # default start_date is 14 days ago
        start_date = request_args.get(
            "start_date", (datetime.now() + timedelta(days=-14)).strftime("%Y-%m-%d")
        )
        # default end_date is today
        end_date = request_args.get("end_date", datetime.now().strftime("%Y-%m-%d"))
        for field_name, date in (("start_date", start_date), ("end_date", end_date)):
            try:
                datetime.strptime(date, "%Y-%m-%d")
            except ValueError:
                self.validation_errors.append(f"{field_name} is not a valid date")
                return NullFinancialDataInput()

        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()

        if start_date > end_date:
            self.validation_errors.append("start_date is after end_date")
            return NullFinancialDataInput()

        # use "IBM" as default symbol
        symbol = request_args.get("symbol", "IBM")
        if symbol not in ["IBM", "AAPL"]:
            self.validation_errors.append("symbol is not valid")
            return NullFinancialDataInput()

        limit = request_args.get("limit", "5")

        # Use 1 as default page number. Page 1 is the first page.
        page = request_args.get("page", "1")
        for field_name, value in [("limit", limit), ("page", page)]:
            try:
                int(value)
            except ValueError:
                self.validation_errors.append(f"{field_name} is not a valid integer")
                return NullFinancialDataInput()

        return FinancialDataInput(
            start_date=start_date,
            end_date=end_date,
            symbol=symbol,
            limit=int(limit),
            page=int(page),
        )

class FinancialStatisticsInputValidationService:
    def __init__(self, request_args):
        self.validation_errors = []
        self.financial_statistics = self.validate_and_parse_financial_statistics_input(
            request_args
        )

    def validate_and_parse_financial_statistics_input(
        self, request_args
    ) -> FinancialStatisticsInput | NullFinancialStatisticsInput:
        # check if all required fields are present
        for required_field in ("start_date", "end_date", "symbol"):
            if required_field not in request_args:
                self.validation_errors.append(f"{required_field} is required")
                return NullFinancialStatisticsInput()

        start_date = request_args.get("start_date")
        end_date = request_args.get("end_date")
        for field_name, date in (("start_date", start_date), ("end_date", end_date)):
            try:
                datetime.strptime(date, "%Y-%m-%d")
            except ValueError:
                self.validation_errors.append(f"{field_name} is not a valid date")
                return NullFinancialStatisticsInput()

        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()

        if start_date > end_date:
            self.validation_errors.append("start_date is after end_date")
            return NullFinancialStatisticsInput()

        symbol = request_args.get("symbol")
        # symbol only allows IBM and AAPL
        if symbol not in ("IBM", "AAPL"):
            self.validation_errors.append("symbol is not valid")
            return NullFinancialStatisticsInput()

        return FinancialStatisticsInput(
            start_date=start_date, end_date=end_date, symbol=symbol
        )

class GetFinancialDataService:
    """Service to get financial data from database"""

    def __init__(self, financial_data_input: FinancialDataInput):
        self.financial_data_input = financial_data_input
        self.financial_data_output = []
        self.pagination = {}

    def get_financial_data(self) -> None:
        financial_data = db.session.scalars(
            db.select(FinancialData)
            .where(
                FinancialData.symbol == self.financial_data_input.symbol,
                FinancialData.date >= self.financial_data_input.start_date,
                FinancialData.date <= self.financial_data_input.end_date,
            )
            .order_by(FinancialData.date)
        ).all()

        self.format_pagination(len(financial_data))
        self.format_financial_data(financial_data)

    def format_financial_data(self, financial_data: list[FinancialData]) -> None:
        start_index = (
            self.financial_data_input.page - 1
        ) * self.financial_data_input.limit
        end_index = start_index + self.financial_data_input.limit
        self.financial_data_output = [
            {
                "symbol": row.symbol,
                "date": row.date.strftime("%Y-%m-%d"),
                "open_price": row.open_price,
                "close_price": row.close_price,
                "volume": row.volume,
            }
            for row in financial_data[start_index:end_index]
        ]

    def format_pagination(self, total_length: int) -> None:
        # page starts at 1
        self.pagination = {
            "total": total_length,
            "limit": self.financial_data_input.limit,
            "page": self.financial_data_input.page,
            "pages": math.ceil(total_length / self.financial_data_input.limit),
        }

class CalculateFinancialStatisticsService:
    """Service to get financial data from database and calculate financial statistics"""

    def __init__(self, financial_statistics_input: FinancialStatisticsInput):
        self.financial_statistics_input = financial_statistics_input
        self.financial_statistics_output = {}

    def calculate_financial_statistics(self) -> None:
        financial_data = db.session.scalars(
            db.select(FinancialData).where(
                FinancialData.symbol == self.financial_statistics_input.symbol,
                FinancialData.date >= self.financial_statistics_input.start_date,
                FinancialData.date <= self.financial_statistics_input.end_date,
            )
        ).all()

        self.format_financial_statistics(financial_data)

    def format_financial_statistics(self, financial_data: list[FinancialData]) -> None:
        self.financial_statistics_output = {
            "symbol": self.financial_statistics_input.symbol,
            "start_date": self.financial_statistics_input.start_date.strftime(
                "%Y-%m-%d"
            ),
            "end_date": self.financial_statistics_input.end_date.strftime("%Y-%m-%d"),
            "average_daily_open_price": str(
                self.calculate_average_daily_open_price(financial_data)
            ),
            "average_daily_close_price": str(
                self.calculate_average_daily_close_price(financial_data)
            ),
            "average_daily_volume": str(
                self.calculate_average_daily_volume(financial_data)
            ),
        }

    def calculate_average_daily_volume(
        self, financial_data: list[FinancialData]
    ) -> Decimal:
        """Calculate average daily volume. Round to nearest integer"""
        return round(sum(row.volume for row in financial_data) / len(financial_data))

    def calculate_average_daily_open_price(
        self, financial_data: list[FinancialData]
    ) -> Decimal:
        """Calculate average daily open price. Round to 2 decimal places"""
        return round(
            (sum(row.open_price for row in financial_data) / len(financial_data)), 2
        )

    def calculate_average_daily_close_price(
        self, financial_data: list[FinancialData]
    ) -> Decimal:
        """Calculate average daily close price. Round to 2 decimal places"""
        return round(
            (sum(row.close_price for row in financial_data) / len(financial_data)), 2
        )


from __future__ import division

from copy import deepcopy
import torch
from torch.autograd import Variable
import torch.nn.functional as F

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

import numpy as np
import torch

def average_rule(keys, Temp_state_dict, neighbors):
    aggr_state_dict = {}
    # aggr_state_dict= torch.sum(Temp_state_dict, 0)
    for key in keys:
        temp_state_dict = [deepcopy(Temp_state_dict[key][i]) for i in neighbors]
        aggr_state_dict[key] = torch.mean(torch.stack(temp_state_dict), 0)
    return aggr_state_dict

def median_rule(keys, Temp_state_dict, neighbors):
    aggr_state_dict = {}
    for key in keys:
        temp_state_dict = [Temp_state_dict[key][i] for i in neighbors]
        aggr_state_dict[key], _ = torch.median(torch.stack(temp_state_dict), 0)
    return aggr_state_dict

def actor_rule(agent_id, policy, Model_actor, Model_critic, Model_critic_2, ram, keys, ActorDict, neighbors, alpha, Accumu_Q_actor, filter, normalize=False, softmax=False):
    random_batch_size = 256
    # gamma = 1
    s1, a1, s2, _, _ = ram.sample(random_batch_size)
    # s1 = Variable(torch.from_numpy(np.float32(s1))).to(device)

    for neigh in neighbors:
        if policy == "TD3":
            pred_a1 = Model_actor[neigh](s1)
            Q_actor = Model_critic[agent_id].Q1(s1, pred_a1).mean()
            # Accumu_loss_actor[agent_id, neigh] = (1 - gamma) * Accumu_loss_actor[agent_id, neigh] + gamma * loss_actor
            Accumu_Q_actor[agent_id, neigh] = Q_actor
        elif policy == "DDPG":
            pred_a1 = Model_actor[neigh](s1)
            Q_actor = Model_critic[agent_id].forward(s1, pred_a1).mean()
            # Accumu_loss_actor[agent_id, neigh] = (1 - gamma) * Accumu_loss_actor[agent_id, neigh] + gamma * loss_actor
            Accumu_Q_actor[agent_id, neigh] = Q_actor
        elif policy == "PPO":
            pass
        elif policy == "SAC":
            # Prediction π(a|s), logπ(a|s), π(a'|s'), logπ(a'|s'), Q1(s,a), Q2(s,a)
            _, pi, log_pi = Model_actor[neigh](s1)
            # Min Double-Q: min(Q1(s,π(a|s)), Q2(s,π(a|s))), min(Q1‾(s',π(a'|s')), Q2‾(s',π(a'|s')))
            min_q_pi = torch.min(Model_critic[agent_id](s1, pi), Model_critic_2[agent_id](s1, pi)).squeeze(1)
            # SAC losses
            para = 0.2
            policy_loss = (para * log_pi - min_q_pi).mean()
            Accumu_Q_actor[agent_id, neigh] = -policy_loss
        else:
            raise NameError("Policy name is not defined!")

    Q = deepcopy(Accumu_Q_actor[agent_id, :])
    min_Q = np.min(Accumu_Q_actor[agent_id, neighbors])
    max_Q = np.max(Accumu_Q_actor[agent_id, neighbors])

    if normalize:
        # Q = np.array([Q[neigh] - min_Q if neigh in neighbors else 0 for neigh in range(len(Q))])
        # Q = Q / (max_Q - min_Q)
        Q = [Q[neigh] - max_Q if neigh in neighbors else 0 for neigh in range(len(Q))]
        Q = [np.exp(Q[neigh]) if neigh in neighbors else 0 for neigh in range(len(Q))]

    if softmax:
        if not normalize:
            Q = [Q[neigh] - max_Q if neigh in neighbors else 0 for neigh in range(len(Q))]
        Q = [np.exp(Q[neigh]) if neigh in neighbors else 0 for neigh in range(len(Q))]

    if filter:
        Q = [Q[neigh] if Q[neigh] >= Q[agent_id] else 0 for neigh in range(len(Q))]

    Q[agent_id] *= alpha[agent_id]
    sum_Q = sum(Q)
    Weight = Q / sum_Q

    # in case sum is not 1
    Weight[agent_id] = 1 - sum(Weight[:agent_id]) - sum(Weight[agent_id + 1:])
    print("agent %d, actor weight, loss" % agent_id, Weight, Accumu_Q_actor[agent_id, :])

    aggr_state_dict = {}
    for key in keys:
        # temp_state_dict = [ActorDict[key][i] * Weight[i] * len(neighbors) for i in neighbors]
        # aggr_state_dict[key] = torch.mean(torch.stack(temp_state_dict), 0)
        temp_state_dict = [ActorDict[key][i] * Weight[i] for i in neighbors]
        aggr_state_dict[key] = torch.sum(torch.stack(temp_state_dict), 0)

    # filtering
    # aggr_actor = deepcopy(Model_actor[agent_id])
    # aggr_actor.load_state_dict(aggr_state_dict)
    # pred_a1 = aggr_actor(s1)
    # Q_actor = Model_critic[agent_id].Q1(s1, pred_a1).mean()
    # if Q_actor > Accumu_Q_actor[agent_id, agent_id]:
    #     print("agent %d, return aggregate model" % agent_id)
    #     return aggr_state_dict
    # else:
    #     return Model_actor[agent_id].state_dict()

    return aggr_state_dict

def critic_rule(agent_id, policy, Model_actor, Model_critic, Model_critic_2, Model_target_critic, Model_target_critic_2, ram, keys, CriticDict, Critic2Dict, neighbors, alpha, Accumu_loss_critic, filter, softmax=False):
    random_batch_size = 256
    GAMMA = 0.99
    gamma = 1
    s1, a1, s2, r1, not_done = ram.sample(random_batch_size)
    if policy == "SAC":
        r1, not_done = r1.squeeze(1), not_done.squeeze(1)

    for neigh in neighbors:
        # Use target actor exploitation policy here for loss evaluation
        if policy == "TD3":
            a2_k = Model_actor[agent_id](s2).detach()
            target_Q1, target_Q2 = Model_target_critic[agent_id].forward(s2, a2_k)
            target_Q = torch.min(target_Q1, target_Q2)
            # y_exp = r + gamma*Q'( s2, pi'(s2))
            y_expected = r1 + not_done * GAMMA * target_Q
            # y_pred = Q( s1, a1)
            y_predicted_1, y_predicted_2 = Model_critic[neigh].forward(s1, a1)
            # compute critic loss, and update the critic
            loss_critic = F.mse_loss(y_predicted_1, y_expected) + F.mse_loss(y_predicted_2, y_expected)
        elif policy == "DDPG":
            a2_k = Model_actor[agent_id](s2).detach()
            target_Q = Model_target_critic[agent_id].forward(s2, a2_k)
            # y_exp = r + gamma*Q'( s2, pi'(s2))
            y_expected = r1 + not_done * GAMMA * target_Q
            # y_pred = Q( s1, a1)
            y_predicted = Model_critic[neigh].forward(s1, a1)
            # compute critic loss, and update the critic
            loss_critic = F.mse_loss(y_predicted, y_expected)
        elif policy == "PPO":
            pass
        elif policy == "SAC":
            para = 0.2
            # Prediction π(a|s), logπ(a|s), π(a'|s'), logπ(a'|s'), Q1(s,a), Q2(s,a)
            _, next_pi, next_log_pi = Model_actor[agent_id](s2)
            q1 = Model_critic[neigh](s1, a1).squeeze(1)
            q2 = Model_critic_2[neigh](s1, a1).squeeze(1)

            min_q_next_pi = torch.min(Model_target_critic[agent_id](s2, next_pi),
                                      Model_target_critic_2[agent_id](s2, next_pi)).squeeze(1)

            v_backup = min_q_next_pi - para * next_log_pi
            q_backup = r1 + GAMMA * not_done * v_backup

            qf1_loss = F.mse_loss(q1, q_backup.detach())
            qf2_loss = F.mse_loss(q2, q_backup.detach())
            loss_critic = qf1_loss + qf2_loss
        else:
            raise NameError("Policy name is not defined!")
        Accumu_loss_critic[agent_id, neigh] = (1 - gamma) * Accumu_loss_critic[agent_id, neigh] + gamma * loss_critic

    loss = deepcopy(Accumu_loss_critic[agent_id, :])

    # if normalize:
    #     min_Q = np.min(loss)
    #     max_Q = np.max(loss)
    #     loss = (loss - min_Q) / (max_Q - min_Q)

    reversed_Loss = np.zeros(len(Model_actor))
    for neigh in neighbors:
        if filter:
            if Accumu_loss_critic[agent_id, neigh] <= Accumu_loss_critic[agent_id, agent_id]:
                reversed_Loss[neigh] = 1 / loss[neigh]
        else:
            # if softmax:
            #     reversed_Loss[neigh] = np.exp(-loss[neigh])  # 1 / np.exp(loss[neigh])
            # else:
            reversed_Loss[neigh] = 1 / loss[neigh]

    reversed_Loss[agent_id] *= alpha[agent_id]

    sum_reversedLoss = sum(reversed_Loss)
    # Weight = np.zeros(numAgent)
    # for neigh in range(0, numAgent):
    Weight = reversed_Loss / sum_reversedLoss
    # in case sum is not 1
    Weight[agent_id] = 1 - sum(Weight[:agent_id]) - sum(Weight[agent_id + 1:])
    print("agent %d, critic weight, loss, reversedloss" % agent_id, Weight, loss, reversed_Loss)
    # weight = torch.from_numpy(weight)

    aggr_state_dict = {}
    for key in keys:
        # temp_state_dict = [ActorDict[key][i] * Weight[i] * len(neighbors) for i in neighbors]
        # aggr_state_dict[key] = torch.mean(torch.stack(temp_state_dict), 0)
        temp_state_dict = [CriticDict[key][i] * Weight[i] for i in neighbors]
        aggr_state_dict[key] = torch.sum(torch.stack(temp_state_dict), 0)

    if policy == "SAC":
        aggr_state_dict_2 = {}
        for key in keys:
            # temp_state_dict = [ActorDict[key][i] * Weight[i] * len(neighbors) for i in neighbors]
            # aggr_state_dict[key] = torch.mean(torch.stack(temp_state_dict), 0)
            temp_state_dict_2 = [Critic2Dict[key][i] * Weight[i] for i in neighbors]
            aggr_state_dict_2[key] = torch.sum(torch.stack(temp_state_dict_2), 0)
        return aggr_state_dict, aggr_state_dict_2

    return aggr_state_dict


# encoding: utf-8
# pylint: disable=redefined-outer-name,missing-docstring
import pytest

from tests import utils

from app import create_app

@pytest.yield_fixture(scope='session')
def flask_app():
    app = create_app(flask_config='testing')
    from app.extensions import db

    with app.app_context():
        db.create_all()
        yield app
        db.drop_all()

@pytest.yield_fixture()
def db(flask_app):
    # pylint: disable=unused-argument,invalid-name
    from app.extensions import db as db_instance
    yield db_instance
    db_instance.session.rollback()

@pytest.fixture(scope='session')
def flask_app_client(flask_app):
    flask_app.test_client_class = utils.AutoAuthFlaskClient
    flask_app.response_class = utils.JSONResponse
    return flask_app.test_client()

@pytest.yield_fixture(scope='session')
def regular_user(flask_app):
    # pylint: disable=invalid-name,unused-argument
    from app.extensions import db

    regular_user_instance = utils.generate_user_instance(
        username='regular_user'
    )

    db.session.add(regular_user_instance)
    db.session.commit()
    yield regular_user_instance
    db.session.delete(regular_user_instance)
    db.session.commit()

@pytest.yield_fixture(scope='session')
def readonly_user(flask_app):
    # pylint: disable=invalid-name,unused-argument
    from app.extensions import db

    readonly_user_instance = utils.generate_user_instance(
        username='readonly_user',
        is_readonly=True
    )

    db.session.add(readonly_user_instance)
    db.session.commit()
    yield readonly_user_instance
    db.session.delete(readonly_user_instance)
    db.session.commit()

@pytest.yield_fixture(scope='session')
def admin_user(flask_app):
    # pylint: disable=invalid-name,unused-argument
    from app.extensions import db

    admin_user_instance = utils.generate_user_instance(
        username='admin_user',
        is_admin=True
    )

    db.session.add(admin_user_instance)
    db.session.commit()
    yield admin_user_instance
    db.session.delete(admin_user_instance)
    db.session.commit()


import pytest
from utils import *

from fireplace.exceptions import GameOver

LORD_JARAXXUS = "EX1_323"
LORD_JARAXXUS_HERO = "EX1_323h"
LORD_JARAXXUS_WEAPON = "EX1_323w"
INFERNO = "EX1_tk33"
INFERNO_TOKEN = "EX1_tk34"

def test_jaraxxus():
	game = prepare_game(CardClass.WARRIOR, CardClass.WARRIOR)
	game.player1.hero.power.use()
	game.player1.give(LIGHTS_JUSTICE).play()
	assert game.player1.weapon.id == LIGHTS_JUSTICE
	game.end_turn()
	game.end_turn()

	assert game.player1.hero.health == 30
	assert game.player1.hero.armor == 2
	game.player1.give(LORD_JARAXXUS).play()
	assert game.player1.hero.id == LORD_JARAXXUS_HERO
	assert game.player1.weapon.id == LORD_JARAXXUS_WEAPON
	assert game.player1.hero.health == 15
	assert game.player1.hero.armor == 0
	assert game.player1.hero.power.id == INFERNO
	assert len(game.player1.field) == 0
	game.end_turn()
	game.end_turn()

	game.player1.hero.power.use()
	assert len(game.player1.field) == 1
	assert game.player1.field[0].id == INFERNO_TOKEN

def test_jaraxxus_cult_master():
	game = prepare_game()
	game.player1.discard_hand()
	game.player1.summon("EX1_595")
	game.player1.give(LORD_JARAXXUS).play()
	assert len(game.player1.field) == 1
	assert not game.player1.hand

def test_jaraxxus_knife_juggler():
	game = prepare_game()
	juggler = game.player1.summon("NEW1_019")
	game.player1.give(LORD_JARAXXUS).play()
	assert game.player2.hero.health == 30
	assert juggler.health == 2

def test_jaraxxus_molten_giant():
	game = prepare_game()
	jaraxxus = game.player1.give("EX1_323")
	molten = game.player1.give("EX1_620")
	jaraxxus.play()
	assert game.player1.hero.health == 15
	assert molten.cost == 20

def test_jaraxxus_mirror_entity():
	game = prepare_game()
	mirror = game.player1.give("EX1_294")
	mirror.play()
	game.end_turn()

	jaraxxus = game.player2.give(LORD_JARAXXUS)
	jaraxxus.play()
	assert not game.player1.secrets
	assert game.player2.hero.id == LORD_JARAXXUS_HERO
	assert len(game.player1.field) == 1
	assert game.player1.field[0].id == LORD_JARAXXUS

def test_jaraxxus_repentance():
	game = prepare_game()
	repentance = game.player1.give("EX1_379")
	repentance.play()
	game.end_turn()

	jaraxxus = game.player2.give(LORD_JARAXXUS)
	jaraxxus.play()
	assert not game.player1.secrets
	assert game.player2.hero.id == LORD_JARAXXUS_HERO
	assert game.player2.hero.health == game.player2.hero.max_health == 1

def test_jaraxxus_snipe():
	game = prepare_game()
	snipe = game.player1.give("EX1_609")
	snipe.play()
	game.end_turn()

	jaraxxus = game.player2.give(LORD_JARAXXUS)
	jaraxxus.play()
	assert len(game.player1.secrets) == 1
	assert game.player2.hero.health == 15

def test_jaraxxus_sacred_trial():
	game = prepare_game()
	trial = game.player1.give("LOE_027")
	trial.play()
	game.end_turn()

	game.player2.give(WISP).play()
	game.player2.give(WISP).play()
	game.player2.give(WISP).play()
	jaraxxus = game.player2.give(LORD_JARAXXUS)
	jaraxxus.play()
	# Will not trigger as 4th minion due to timing
	assert trial in game.player1.secrets
	assert not game.player2.hero.dead
	game.end_turn()
	game.end_turn()

	wisp4 = game.player2.summon(WISP)
	assert not wisp4.dead
	jaraxxus = game.player2.give(LORD_JARAXXUS)
	with pytest.raises(GameOver):
		jaraxxus.play()
		assert trial not in game.player1.secrets
		assert game.player2.hero.dead


import random
import numpy as np
from math import sqrt, log
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

x1_list = []
x2_list = []
y_list = []
counter = 0

def drawFunc(minX, minY, maxX, maxY, ax = None):
    #fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
    #ax.set_xlabel('x1')
    #ax.set_ylabel('x2')
    #ax.set_zlabel('f(x1,x2)')

    x1_array = np.arange(minX, maxX, 0.1)
    x2_array = np.arange(minY, maxY, 0.1)
    x1_array, x2_array = fill_arrays(x1_array, x2_array)
    R = fill_z(x1_array, x2_array)

    x1_array = np.arange(minX, maxX, 0.1)
    x2_array = np.arange(minY, maxY, 0.1)
    x1_array, x2_array = np.meshgrid(x1_array, x2_array)
    #R = f(x1_array, x2_array)

    #drawBoder(ax, x1_array, g1_1)
    #drawBoder(ax, x1_array, g2_1)
    #drawBoder(ax, x1_array, g3_1)
    #drawBoder(ax, x1_array, g4_1)

    #print(R)
    ax.plot_surface(x1_array, x2_array, R, alpha = 0.6)
    #plt.show()

def fill_arrays(x, y):
    final_y = []
    final_x = []
    for i in range(len(y)):
        final_y.append([])
        for j in range(len(x)):
            if (barier(x[j], y[i])):
                #if f(x[j], y[i]) > 50:
                    #print("i =", i, "j =", j)
                    #print("x =", x[j], "y =", y[i], "f =", f(x[j], y[i]))
                final_y[i].append(x[j])
            else: final_y[i].append(0)

    for i in range(len(x)):
        final_x.append([])
        for j in range(len(y)):
            if (barier(x[j], y[i])):
                final_x[i].append(y[j])
            else: final_x[i].append(0)

    #for i in range(len(final_x)):
    #    print(i,")", final_x[i])
    return final_y, final_x

def fill_z(x, y):
    z = []
    for i in range(len(x)):
        z.append([])
        for j in range(len(x[i])):
            if (x[i][j] != 0 and y[j][i] != 0):
                z[i].append(f(x[i][j], y[j][i]))
            else: z[i].append(0.0)
            #print("i =", i, "j =", j)
            #print("x =", x[i][j], "y =", y[j][i], "z =", z[i][j])

    #for i in range(len(z)):
    #    print(i,")", z[i])
    r = np.array(z)
    #for i in range(len(z)):
    #    r.__add__(np.array[z[i]])
    return r

def fill_F2(x, y):
    z = []
    for i in range(len(x)):
        z.append([])
        for j in range(len(x[i])):
            if (barier(x[i][j], y[i][j])):
                z[i].append(f(x[i][j], y[i][j]))
            else: z[i].append(0.0)

    r = np.array(z)
    #for i in range(len(z)):
    #    r.__add__(np.array[z[i]])
    #print(r)
    return r

def g1_1(x1):
    return (-3*x1 + 6) / 2

def g2_1(x1):
    return (-x1 - 3) / (-1)

def g3_1(x1):
    return (x1 - 7) / (-1)

def g4_1(x1):
    return (2*x1 - 4) / 3

def drawBoder(ax, x1, g):
    zs = np.arange(0, 80, 35)

    X, Z = np.meshgrid(x1, zs)
    Y = g(X)

    #fig = plt.figure()
    #ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(X, Y, Z, alpha = 0.4)

def show(x1_list, x2_list):
    N = int(x1_list.__len__())
    if (N <= 0):
        return

    fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
    ax.set_xlabel('x1')
    ax.set_ylabel('x2')
    ax.set_zlabel('f(x1,x2)')
    #x1_array = np.arange(min(x1_list) - 0.1, max(x1_list) + 0.1, 0.1)
    #x2_array = np.arange(min(x2_list) - 0.1, max(x2_list) + 0.1, 0.1)
    #x1_array, x2_array = np.meshgrid(x1_array, x2_array)
    #R = f(x1_array, x2_array)
    #ax.plot_surface(x1_array, x2_array, R, color='b', alpha=0.5)
    drawFunc(0, 0, 5, 5, ax)

    x1_list2 = []
    x2_list2 = []
    f_list = []

    ax.scatter(x1_list[0], x2_list[0], f(x1_list[0], x2_list[0]), c='black')
    x1_list2.append(x1_list[0])
    x2_list2.append(x2_list[0])
    f_list.append(f(x1_list[0], x2_list[0]))

    for n in range(1, N - 1):
        ax.scatter(x1_list[n], x2_list[n], f(x1_list[n], x2_list[n]), c='red')
        x1_list2.append(x1_list[n])
        x2_list2.append(x2_list[n])
        f_list.append(f(x1_list[n], x2_list[n]))

    ax.scatter(x1_list[N - 1], x2_list[N - 1], f(x1_list[N - 1], x2_list[N - 1]), c='green')
    x1_list2.append(x1_list[N - 1])
    x2_list2.append(x2_list[N - 1])
    f_list.append(f(x1_list[N - 1], x2_list[n]))

    ax.plot(x1_list2, x2_list2, f_list, color="black")

    plt.show()

# <---------- f
def f(x1, x2):
    return (x1-6)**2 +(x2-7)**2

def f_x1(x1, x2):
    return 2*x1 - 12

def f_x2(x1, x2):
    return 2*x2 - 14
# -------------->

# <---------- gi
def g1(x1, x2):
    return -3*x1 - 2*x2 + 6

def g2(x1, x2):
    return -x1 + x2 - 3

def g3(x1, x2):
    return x1 + x2 - 7

def g4(x1, x2):
    return 2*x1 - 3*x2 - 4
# -------------->

# <---------- gi_bool
def g1_bool(x1, x2):
    return -3*x1 - 2*x2 + 6 <= 0

def g2_bool(x1, x2):
    return -x1 + x2 - 3 <= 0

def g3_bool(x1, x2):
    return x1 + x2 - 7 <= 0

def g4_bool(x1, x2):
    return 2*x1 - 3*x2 - 4 <= 0

def barier(x1, x2):
    return (g1_bool(x1, x2) and g2_bool(x1, x2) and g3_bool(x1, x2) and g4_bool(x1, x2))

# -------------->

# <---------- X
def F(x1, x2, r):
    return f(x1,x2) + P(x1, x2, r)

def F_x1(x1, x2, r):
    return f_x1(x1, x2) + P_x1(x1, x2, r)

def F_x2(x1, x2, r):
    return f_x2(x1, x2) + P_x2(x1, x2, r)
# -------------->

# <-------------- P
def P(x1, x2, r):
    sum = 1/g1(x1, x2) + 1/g2(x1, x2) + 1/g3(x1, x2) + 1/g4(x1, x2)
    return -r*sum

def P_x1(x1, x2, r):
    sum = 3/(g1(x1, x2)**2) + 1/(g2(x1, x2)**2) - 1/(g3(x1, x2)**2) - 1/(g4(x1, x2)**2)
    return -r*sum

def P_x2(x1, x2, r):
    sum = 2/(g1(x1, x2)**2) - 1/(g2(x1, x2)**2) - 1/(g3(x1, x2)**2) + 3/(g4(x1, x2)**2)
    return -r*sum

# ------------>

def gradient(x1, x2, r):
    i = F_x1(x1, x2, r)
    j = F_x2(x1, x2, r)
    return [i, j]

def module_of_gradient(grad):
    i = 0; j = 1
    return sqrt(grad[i]**2 + grad[j]**2)

def method_of_gradient_descent_with_a_constant_step(x1, x2, e, M, r):
    global counter
    k = 0
    counter += 1
    x1_next = x1
    x2_next = x2
    while True:
        counter += 2
        grad = gradient(x1, x2, r)
        module_grad = module_of_gradient(grad)

        if ((module_grad < e) and (k >= M)):
            return (x1_next, x2_next)

        gamma = 0.1
        x1_next = x1 - gamma * grad[0]
        x2_next = x2 - gamma * grad[1]

        counter += 2
        while (F(x1_next, x2_next, r) - F(x1, x2, r) >= 0 or not barier(x1_next, x2_next)):
            gamma /= 4
            x1_next = x1 - gamma * grad[0]
            x2_next = x2 - gamma * grad[1]
            counter += 1

        #print(grad, 'x1 =', x1, 'x2 =', x2, 'x1_next =', x1_next, 'x2_next =', x2_next, 'gamma =', gamma)

        x1_list.append(x1); x2_list.append(x2)
        if ((sqrt(abs(x1_next - x1)**2 + abs(x2_next - x2)**2) <= e)
            & (abs(F(x1_next, x2_next, r) - F(x1, x2, r)) <= e)):
            return (x1_next, x2_next)

        x1 = x1_next
        x2 = x2_next
        k += 1

def barrier_function_method(x1, x2, r, C, e, M, k):
    min_x1, min_x2 = method_of_gradient_descent_with_a_constant_step(x1, x2, e, M, r)
    #print("x1 =", min_x1, "x2 =", min_x2)
    fine = P(min_x1, min_x2, r)
    #print("fine =", fine)
    if (abs(fine) <= e):
            return [(round(min_x1, round_num),
                     round(min_x2, round_num),
                     round(f(min_x1, min_x2), round_num)),
                    k]
    k += 1
    r = r/C
    return barrier_function_method(min_x1, min_x2, r, C, e, M, k)

round_num = 4
x1 = 2.5
x2 = 1
e = 0.0001
M = 100
r = 1
c = 10
k = 0

result = barrier_function_method(x1, x2, r, c, e, M, k)
print(f"Barrier function method: {result[0]}; count of iteractions = {result[1]}")
print('Count of compute function =', counter + 1)

show(x1_list, x2_list)
#drawFunc(0, 0, 5, 5)

6 Семестр/Оптимизация/Lab_6_grad.py


import requests
import json
import nestConfig

#AWS Constants
url = nestConfig.get_URL()
query = '''
mutation Mutation($id: String!) {
  checkIn(id: $id) {
    code
    message
  }
}
'''

def checkIn(nestID):
    #Ensure nest is connected to the backend
    content = json.dumps({'id':nestID}) #Assign nest name to be checked
    try:
        res = requests.post(url, json={'query': query, 'variables': content})
    except Exception as error:
        return None
    if res.status_code == 200:
        print(res.status_code)
    else:
        raise Exception("Query failed to run by returning code of {}.".format(res.text))
    return None


#!/usr/bin/env python3
"""Tools to define Templates.

Templates are very similar to plugins, but use jinja to transform `.enbt` template files upon installation.
"""
__author__ = "Miguel Hernández-Cabronero"
__since__ = "2021/08/01"

import sys
import argparse
import inspect
import os
import glob
import shutil
import tempfile
import jinja2
import stat

from .installable import Installable, InstallableMeta
import enb.config
from enb.config import options

class MetaTemplate(InstallableMeta):
    def __init__(cls, *args, **kwargs):
        if cls.__name__ != "Template":
            cls.tags.add("template")
        super().__init__(*args, **kwargs)

class Template(Installable, metaclass=MetaTemplate):
    """
    Base class to define templates. Subclasses must be defined in the __plugin__.py file of the template's
    source dir.

    - Templates copy the source dir's contents (except for __plugin__.py) and then transforms
      any `*.enbt` file applying jinja and removing that extension.

    - Templates may require so-called fields in order to produce output.
      These fields can be automatically taken from enb.config.ini (e.g., file-based configuration),
      passed as arguments to the template installation CLI, and programmatically.

    - One or more templates can be installed into an existing directory, the __plugin__.py file is not written
      by default to the installation dir.
    """
    # Map of required field names to their corresponding help
    required_fields_to_help = dict()

    # Files in the template's source dir ending with templatable_extension
    # are subject to jinja templating upon installation.
    templatable_extension = ".enbt"

    @classmethod
    def get_fields(cls, original_fields=None):
        try:
            return cls._fields
        except AttributeError:
            # If there are required fields, satisfy them or fail
            fields = dict(original_fields) if original_fields is not None else dict()
            if cls.required_fields_to_help:
                ini_cli_fields, unused_options = cls.get_field_parser().parse_known_args()
                # Syntax is "plugin install <template> <installation>, so
                # four non-parsed options are expected
                assert len(unused_options) >= 4, (sys.argv, ini_cli_fields, unused_options)
                unused_options = unused_options[4:]
                for field_name in cls.required_fields_to_help:
                    if field_name not in fields:
                        try:
                            fields[field_name] = getattr(ini_cli_fields, field_name)
                            assert fields[field_name] is not None
                        except (KeyError, AssertionError) as ex:
                            raise SyntaxError(
                                f"Missing field {repr(field_name)}. Help for {field_name}:\n"
                                f"{cls.required_fields_to_help[field_name]}\n\n"
                                f"Invoke again with --{field_name}=\"your value\" or with -h for additional help.\n") from ex
                if unused_options:
                    print(f"Warning: unused option{'s' if len(unused_options) > 1 else ''}. \n  - ", end="")
                    print('\n  - '.join(repr(o) for o in unused_options))
                    print(f"NOTE: You can use '' or \"\" to define fields with spaces in them.")
                    print()
            cls._fields = fields
            return fields

    @classmethod
    def install(cls, installation_dir, overwrite_destination=False, fields=None):
        """Install a template into the given dir. See super().install for more information.

        :param installation_dir: directory where the contents of the template are placed.
          It will be created if not existing.
        :param overwrite_destination: if False, a SyntaxError is raised if any of the
          destination contents existed prior to this call. Note that installation_dir
          can already exist, it is the files and directories moved into it that can
          trigger this SyntaxError.
        :param fields: if not None, it must be a dict-like object containing a field to field value
          mapping. If None, it is interpreted as an empty dictionary.
          Required template fields not present in fields will be then read from the CLI
          arguments. If those are not provided, then the default values read from `*.ini`
          configuration files. If any required field cannot not satisfied after this,
          a SyntaxError is raised.
        """
        # If there are required fields, satisfy them or fail
        fields = cls.get_fields(original_fields=fields)

        template_src_dir = os.path.dirname(os.path.abspath(inspect.getfile(cls)))
        for input_path in glob.glob(os.path.join(template_src_dir, "**", "*"), recursive=True):
            if "__pycache__" in input_path:
                continue
            if os.path.basename(input_path) == "__plugin__.py":
                continue

            # By default, the original structure and file names are preserved.
            output_path = os.path.abspath(input_path).replace(
                os.path.abspath(template_src_dir),
                os.path.abspath(installation_dir))

            # Directories are created when found
            if os.path.isdir(input_path):
                os.makedirs(output_path, exist_ok=True)
                continue

            input_is_executable = os.access(input_path, os.X_OK)

            # Files ending in '.enbt' will be identified as templates, processed and stripped of their extension.
            is_templatable = os.path.isfile(input_path) \
                             and os.path.basename(input_path).endswith(cls.templatable_extension)

            os.makedirs(os.path.dirname(output_path), exist_ok=True)
            if is_templatable:
                with tempfile.NamedTemporaryFile(mode="w+") as templated_file:
                    jinja_env = jinja2.Environment(
                        loader=jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(input_path))),
                        autoescape=jinja2.select_autoescape())
                    template = jinja_env.get_template(os.path.basename(input_path))
                    templated_file.write(template.render(**fields))
                    templated_file.flush()
                    templated_file.seek(0)
                    if os.path.exists(output_path[:-len(cls.templatable_extension)]) and not options.force:
                        raise ValueError(
                            f"Error installing template {cls.name}: output file {repr(output_path)} already exists "
                            f"and options.force={options.force}. Run with -f to overwrite.")
                    with open(output_path[:-len(cls.templatable_extension)], "w") as output_file:
                        output_file.write(templated_file.read())
                    if input_is_executable:
                        os.chmod(output_path[:-len(cls.templatable_extension)],
                                 os.stat(output_path[:-len(cls.templatable_extension)]).st_mode | stat.S_IEXEC)
            else:
                if os.path.exists(output_path) and not options.force:
                    raise ValueError(
                        f"Error installing template {cls.name}: output file {repr(output_path)} already exists "
                        f"and options.force={options.force}. Run with -f to overwrite.")
                shutil.copy(input_path, output_path)

        cls.build(installation_dir=installation_dir)
        print(f"Template {repr(cls.name)} successfully installed into {repr(installation_dir)}.")

    @classmethod
    def get_field_parser(cls):
        description = f"Template {repr(cls.name)} installation help."
        if cls.required_fields_to_help:
            description += f"\n\nFields are automatically read from the following paths (in this order):\n"
            description += "\n".join(enb.config.ini.used_config_paths)

            # defined_description = f"\n\nAlready refined fields:"
            defined_field_lines = []
            for field_name in sorted(cls.required_fields_to_help.keys()):
                try:
                    defined_field_lines.append(f"  {field_name} = {enb.config.ini.get_key('template', field_name)}")
                except KeyError:
                    pass
            if defined_field_lines:
                description += f"\n\nFile-defined fields:\n"
                description += "\n".join(defined_field_lines)

        parser = argparse.ArgumentParser(
            prog=f"enb plugin install {cls.name}",
            description=description,
            formatter_class=argparse.RawTextHelpFormatter)
        required_flags_group = parser.add_argument_group(
            "Required flags (use '' or \"\" quoting for fields with spaces)")
        for field_name, field_help in cls.required_fields_to_help.items():
            try:
                default_field_value = enb.config.ini.get_key("template", field_name)
            except KeyError:
                default_field_value = None
            if field_help[-1] != ".":
                field_help += "."
            required_flags_group.add_argument(
                f"--{field_name}",
                default=default_field_value,
                help=field_help,
                metavar=field_name)
        # This argument is for showing help to the user only, since it will have already been parsed
        # by enb.config.ini by the time this is called.
        parser.add_argument(f"--ini", nargs="*", required=False, type=str,
                            help="Additional .ini paths with a [field] section containing field = value lines")
        return parser