# -*- encoding:utf-8 -*-
'''
@time: 2019/12/21 9:48 下午
@author: huguimin
@email: 718400742@qq.com
'''
import os
import random
import math
import torch
import argparse
import numpy as np
from util.util_data_gcn import *
from models.word2vec.ecgcn import ECGCN
from models.word2vec.ecgat import ECGAT
from models.word2vec.fssgcn import ECClassifier
from models.word2vec.aggcn import AGClassifier
# from models.ecaggcn_no_dcn import ECClassifier
from sklearn import metrics
import torch.nn as nn
import time

class Model:

    def __init__(self, opt, idx):
        self.opt = opt

        self.embedding = load_embedding(opt.embedding_path)
        self.embedding_pos = load_pos_embedding(opt.embedding_dim_pos)
        self.split_size = math.ceil(opt.data_size / opt.n_split)

        self.global_f1 = 0
        # self.train, self.test = load_data(self.split_size, idx, opt.data_size) #意味着只能从一个角度上训练，应该换几种姿势轮着训练
        if opt.dataset == 'EC':
            self.train, self.test = load_percent_train(opt.per, self.split_size, idx, opt.data_size)
        elif opt.dataset == 'EC_en':
            self.train, self.test = load_data_en()
        else:
            print('DATASET NOT EXIST')
        # self.train, self.test = load_data(self.split_size, idx, opt.data_size)
        self.sub_model = opt.model_class(self.embedding, self.embedding_pos, self.opt).to(opt.device)

    def _reset_params(self):
        for p in self.sub_model.parameters():
            if p.requires_grad:
                if len(p.shape) > 1:
                    self.opt.initializer(p)
                else:
                    stdv = 1. / math.sqrt(p.shape[0])
                    torch.nn.init.uniform_(p, a=-stdv, b=stdv)

    def _print_args(self):
        n_trainable_params, n_nontrainable_params, model_params = 0, 0, 0
        for p in self.sub_model.parameters():
            n_params = torch.prod(torch.tensor(p.shape)).item()
            model_params += n_params
            if p.requires_grad:
                n_trainable_params += n_params
            else:
                n_nontrainable_params += n_params
        print('n_trainable_params: {0}, n_nontrainable_params: {1}, model_params: {2}'.format(n_trainable_params, n_nontrainable_params, model_params))
        print('> training arguments:')
        for arg in vars(self.opt):
            print('>>> {0}: {1}'.format(arg, getattr(self.opt, arg)))

    def _train(self, criterion, optimizer):

        max_test_pre = 0
        max_test_rec = 0
        max_test_f1 = 0
        global_step = 0
        continue_not_increase = 0

        for epoch in range(self.opt.num_epoch):
            print('>' * 100)
            print('epoch: ', epoch)
            n_correct, n_total = 0, 0
            increase_flag = False

            for train in get_train_batch_data(self.train, self.opt.batch_size, self.opt.keep_prob1, self.opt.keep_prob2):
                global_step += 1
                self.sub_model.train()
                optimizer.zero_grad()

                inputs = [train[col].to(self.opt.device) for col in self.opt.inputs_cols]
                targets = train['label'].to(self.opt.device)
                doc_len = train['doc_len'].to(self.opt.device)
                targets = torch.argmax(targets, dim=2)
                targets_flatten = torch.reshape(targets, [-1])

                outputs = self.sub_model(inputs)
                outputs_flatten = torch.reshape(outputs, [-1, self.opt.num_class])
                loss = criterion(outputs_flatten, targets_flatten)
                # loss = nn.functional.nll_loss(outputs_flatten, targets_flatten)
                outputs = torch.argmax(outputs, dim=-1)

                loss.backward()
                optimizer.step()

                if global_step % self.opt.log_step == 0:
                    train_acc, train_pre, train_rec, train_f1 = self._evaluate_prf_binary(targets, outputs, doc_len)
                    print('Train: loss:{:.4f}, train_acc: {:.4f}, train_pre:{:.4f}, train_rec:{:.4f}, train_f1: {:.4f}\n'.format(loss.item(), train_acc, train_pre, train_rec, train_f1))

                    test_acc, test_pre, test_rec, test_f1 = self._evaluate_acc_f1()
                    # if test_acc > max_test_acc:
                    #     max_test_acc = test_acc
                    if test_f1 > max_test_f1:
                        increase_flag = True
                        max_test_f1 = test_f1
                        max_test_pre = test_pre
                        max_test_rec = test_rec
                        if self.opt.save and test_f1 > self.global_f1:
                            self.global_f1 = test_f1
                            torch.save(self.sub_model.state_dict(), 'state_dict/'+self.opt.model_name+'_'+self.opt.dataset+'_test.pkl')
                            print('>>> best model saved.')
                    print('Test: test_acc: {:.4f}, test_pre:{:.4f}, test_rec:{:.4f}, test_f1: {:.4f}'.format(test_acc, test_pre, test_rec, test_f1))
            if increase_flag == False:
                continue_not_increase += 1
                if continue_not_increase >= 5:
                    print('early stop.')
                    break
            else:
                continue_not_increase = 0
        return max_test_pre, max_test_rec, max_test_f1

    def _evaluate_acc_f1(self):
        # switch model to evaluation mode
        self.sub_model.eval()
        targets_all, outputs_all, doc_len_all = None, None, None
        inference_time_list = []
        with torch.no_grad():
            for test in get_test_batch_data(self.test, self.opt.batch_size):
                inputs = [test[col].to(self.opt.device) for col in self.opt.inputs_cols]
                targets = test['label'].to(self.opt.device)
                doc_len = test['doc_len'].to(self.opt.device)
                targets = torch.argmax(targets, dim=2)#(32,75)
                if self.opt.infer_time:
                    torch.cuda.synchronize()
                    start_time = time.time()
                    outputs = self.sub_model(inputs)
                    torch.cuda.synchronize()
                    end_time = time.time()
                    inference_time = end_time - start_time
                    inference_time_list.append(inference_time/targets.shape[0])
                else:
                    outputs = self.sub_model(inputs)
                outputs = torch.argmax(outputs, dim=2)#(32, 75)

                if targets_all is None:
                    targets_all = targets
                    outputs_all = outputs
                    doc_len_all = doc_len
                else:
                    targets_all = torch.cat((targets_all, targets), dim=0)
                    outputs_all = torch.cat((outputs_all, outputs), dim=0)
                    doc_len_all = torch.cat((doc_len_all, doc_len), dim=0)

            test_acc, test_pre, test_rec, test_f1 = self._evaluate_prf_binary(targets_all, outputs_all, doc_len_all)
        infer_time = np.mean(np.array(inference_time_list))
        print('infer_time==================', str(infer_time))
        return test_acc, test_pre, test_rec, test_f1

    def _evaluate_prf_binary(self, targets, outputs, doc_len):
        """
        :param targets: [32,75]
        :param outputs: [32,75]
        :return:
        """
        tmp1, tmp2 = [], []
        for i in range(outputs.shape[0]):
            for j in range(doc_len[i]):
                tmp1.append(outputs[i][j].cpu())
                tmp2.append(targets[i][j].cpu())
        y_pred, y_true = np.array(tmp1), np.array(tmp2)
        acc = metrics.precision_score(y_true, y_pred, average='micro')
        p = metrics.precision_score(y_true, y_pred, average='binary')
        r = metrics.recall_score(y_true, y_pred, average='binary')
        f1 = metrics.f1_score(y_true, y_pred, average='binary')
        return acc, p, r, f1

    def run(self, folder, repeats=1):
        # Loss and Optimizer
        print(('-'*50 + 'Folder{}' + '-'*50).format(folder))
        criterion = nn.CrossEntropyLoss()
        # criterion = nn.functional.nll_loss()
        _params = filter(lambda p: p.requires_grad, self.sub_model.parameters())
        optimizer = self.opt.optimizer(_params, lr=self.opt.learning_rate, weight_decay=self.opt.l2reg)

        if not os.path.exists('log/'):
            os.mkdir('log/')

        f_out = open('log/' + self.opt.model_name + '_' + str(folder) + '_test.txt', 'a+', encoding='utf-8')

        max_test_pre_avg = 0
        max_test_rec_avg = 0
        max_test_f1_avg = 0
        for i in range(repeats):
            print('repeat: ', (i + 1))
            f_out.write('repeat: ' + str(i + 1))
            self._reset_params()
            max_test_pre, max_test_rec, max_test_f1 = self._train(criterion, optimizer)
            print('max_test_acc: {0}     max_test_hf1: {1}'.format(max_test_pre, max_test_f1))
            f_out.write('max_test_acc: {0}, max_test_f1: {1}'.format(max_test_pre, max_test_f1))
            max_test_pre_avg += max_test_pre
            max_test_rec_avg += max_test_rec
            max_test_f1_avg += max_test_f1
            print('#' * 100)
        print("max_test_acc_avg: {.4f}", max_test_pre_avg / repeats)
        print('max_test_acc_rec: {.4f}', max_test_rec_avg / repeats)
        print("max_test_f1_avg: {.4f}", max_test_f1_avg / repeats)
        f_out.write("max_test_pre_avg: {0}, max_test_rec_avg: {1}, max_test_f1_avg: {2}".format(max_test_pre_avg / repeats, max_test_rec_avg / repeats, max_test_f1_avg / repeats))
        f_out.close()
        return max_test_pre_avg / repeats, max_test_rec_avg / repeats, max_test_f1_avg / repeats

if __name__ == '__main__':
    # Hyper Parameters
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_name', default='fssgcn', type=str)
    parser.add_argument('--optimizer', default='adam', type=str)
    parser.add_argument('--initializer', default='xavier_uniform_', type=str)
    parser.add_argument('--learning_rate', default=0.001, type=float)
    parser.add_argument('--input_dropout', default=0.1, type=float)
    parser.add_argument('--gcn_dropout', default=0.1, type=float)
    parser.add_argument('--head_dropout', default=0.1, type=float)
    parser.add_argument('--keep_prob2', default=0.1, type=float)
    parser.add_argument('--keep_prob1', default=0.1, type=float)
    parser.add_argument('--alpha', default=0.3, type=float)
    parser.add_argument('--l2reg', default=0.00001, type=float)
    # parser.add_argument('--l2reg', default=0.000005, type=float)
    parser.add_argument('--num_epoch', default=100, type=int)
    parser.add_argument('--batch_size', default=32, type=int)
    parser.add_argument('--log_step', default=5, type=int)
    parser.add_argument('--embed_dim', default=200, type=int)
    parser.add_argument('--embedding_dim_pos', default=100, type=int)
    ###中文数据集的embedding文件
    parser.add_argument('--embedding_path', default='embedding.txt', type=str)
    ###英文数据集的embedding文件################################
    # parser.add_argument('--embedding_path', default='all_embedding_en.txt', type=str)
    #################################################

    parser.add_argument('--pos_num',default=138, type=int)
    parser.add_argument('--hidden_dim', default=100, type=int)

    parser.add_argument('--num_layers', default=3, type=int)
    parser.add_argument('--nheads', default=1, type=int)
    parser.add_argument('--sublayer_first', default=2, type=int)
    parser.add_argument('--sublayer_second', default=4, type=int)
    parser.add_argument('--sublayer', default=1, type=int)

    parser.add_argument('--no_rnn', default=False, type=bool)
    parser.add_argument('--rnn_layer', default=1, type=int)
    parser.add_argument('--rnn_hidden', default=100, type=int)
    parser.add_argument('--rnn_dropout', default=0.5, type=float)

    parser.add_argument('--no_pos', default=False, type=bool)
    parser.add_argument('--n_split', default=10, type=int)
    parser.add_argument('--per', default=1.0, type=float)

    parser.add_argument('--num_class', default=2, type=int)
    parser.add_argument('--save', default=True, type=bool)
    parser.add_argument('--seed', default=776, type=int)
    parser.add_argument('--device', default=None, type=str)
    parser.add_argument('--infer_time', default=False, type=bool)
    ####数据集为英文数据集
    # parser.add_argument('--dataset', default='EC_en', type=str)

    ####数据集为中文数据集
    parser.add_argument('--dataset', default='EC', type=str)

    opt = parser.parse_args()

    model_classes = {
        'ecgcn': ECGCN,
        'ecgat': ECGAT,
        'aggcn': AGClassifier,
        'fssgcn': ECClassifier
    }
    input_colses = {
        'ecgcn': ['content', 'sen_len', 'doc_len', 'doc_id', 'emotion_id', 'graph'],
        'ecgat': ['content', 'sen_len', 'doc_len', 'doc_id', 'emotion_id', 'graph'],
        'aggcn': ['content', 'sen_len', 'doc_len', 'doc_id', 'emotion_id', 'graph'],
        'fssgcn': ['content', 'sen_len', 'doc_len', 'doc_id', 'emotion_id', 'graph']
    }
    initializers = {
        'xavier_uniform_': torch.nn.init.xavier_uniform_,
        'xavier_normal_': torch.nn.init.xavier_normal,
        'orthogonal_': torch.nn.init.orthogonal_,
    }
    optimizers = {
        'adadelta': torch.optim.Adadelta,  # default lr=1.0
        'adagrad': torch.optim.Adagrad,  # default lr=0.01
        'adam': torch.optim.Adam,  # default lr=0.001
        'adamax': torch.optim.Adamax,  # default lr=0.002
        'asgd': torch.optim.ASGD,  # default lr=0.01
        'rmsprop': torch.optim.RMSprop,  # default lr=0.01
        'sgd': torch.optim.SGD,
    }
    opt.model_class = model_classes[opt.model_name]
    opt.inputs_cols = input_colses[opt.model_name]
    opt.initializer = initializers[opt.initializer]
    opt.optimizer = optimizers[opt.optimizer]
    if opt.dataset == 'EC':
        opt.max_doc_len = 75
        opt.max_sen_len = 45
        opt.data_size = 2105
        opt.hidden_dim = 100
        opt.rnn_hidden = 100
        opt.embed_dim = 200
        opt.embedding_path = 'embedding.txt'
    else:
        opt.max_doc_len = 45
        opt.max_sen_len = 130
        opt.data_size = 2105
        opt.hidden_dim = 150
        opt.rnn_hidden = 150
        opt.embed_dim = 300
        opt.embedding_path = 'all_embedding_en.txt'

    opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') \
        if opt.device is None else torch.device(opt.device)

    if opt.seed is not None:
        random.seed(opt.seed)
        np.random.seed(opt.seed)
        torch.manual_seed(opt.seed)
        torch.cuda.manual_seed(opt.seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    p, r, f1 = [], [], []

    for i in range(1):
        model = Model(opt, i)
        ###计算模型大
        model._print_args()
        p_t, r_t, f1_t = model.run(i)
        p.append(p_t)
        r.append(r_t)
        f1.append(f1_t)
    print("max_test_pre_avg: {:.4f}, max_test_rec_avg: {:.4f}, max_test_f1_avg: {:.4f}".format(np.mean(p), np.mean(r), np.mean(f1)))


import os, settings
from app import myApp
import uuid
from flask import request, render_template
from pdf_core import PdfHelper
from threading import Timer

@myApp.route('/', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        # create a list with all pdf files
        files = []
        for uploadedFile in request.files.getlist('file'):
            if allowed_file(uploadedFile.filename):
                files.append(uploadedFile)

        # join pdf files
        pdfHelper = PdfHelper()
        uniqueFilenamePath = os.path.join(settings.RESULT_PATH, str(uuid.uuid4()) + ".pdf")
        pdfHelper.merge_pdfs(files, uniqueFilenamePath)

        # remove the file after 10 min
        t = Timer(60*10, delete, (uniqueFilenamePath,))
        t.start();

        # close the files
        for uploadedFile in files:
            uploadedFile.close()

        return render_template('show_links.html', link=uniqueFilenamePath)
    return render_template('index.html')

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in settings.ALLOWED_EXTENSIONS

def delete(dest):
    if os.path.exists(dest):
        os.remove(dest)


# This is a sample Python script.

# Press ⌃R to execute it or replace it with your code.
# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings.
import os, requests, json

# python request examples
# https://www.pythonforbeginners.com/requests/using-requests-in-python

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press ⌘F8 to toggle the breakpoint.

def restexample01():
    github_url = "https://api.github.com/user/repos"
    data = json.dumps({'name': 'test', 'description': 'some test repo'})
    r = requests.post(github_url, data, auth=('user', '*****'))
    print(r.json)

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    restexample01()
    print_hi("PyCharm. It's end of the code")

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


from routersim.interface import LogicalInterface
from .messaging import FrameType
from .messaging import ICMPType, UnreachableType
from .mpls import MPLSPacket, PopStackOperation
from .observers import Event, EventType
from scapy.layers.inet import IP,ICMP,icmptypes
from copy import copy
import ipaddress

class ForwardingTable:

    def __init__(self, event_manager, parent_logger):
        self.fib = None
        self.event_manager = event_manager
        self.logger = parent_logger.getChild('forwarding')

    def __str__(self):
        return "Forwarding Table"

    def set_fib(self, fib):
        self.fib = fib
        self.logger.debug("Installed new forwarding table")

    def lookup_ip(self, ip_address):
        as_network = ipaddress.ip_network(ip_address)
        # ASSUMPTION: fib is sorted with highest prefix first
        # so we should always arrive at something more specific first
        # yes, this is very inefficient
        if self.fib is None:
            return None

        for prefix in self.fib[FrameType.IPV4]:
            if as_network.overlaps(prefix):
                self.event_manager.observe(
                    Event(
                        EventType.FORWARDING,
                        self,
                        f"Identified forwarding entry for {ip_address}"
                    )
                )
                return [self.fib[FrameType.IPV4][prefix]]

        return None

    def lookup_label(self, label):
        if self.fib is None:
            return None

        if self.fib is None or FrameType.MPLSU not in self.fib:
            return None

        return [self.fib[FrameType.MPLSU][str(label)]]

    def print_fib(self):
        print("** IPV4 FIB ***")
        for prefix in self.fib[FrameType.IPV4]:
            entry = self.fib[FrameType.IPV4][prefix]
            print(f"{entry}")
        print("")
        print("** MPLS FIB ***")
        for prefix in self.fib[FrameType.MPLSU]:
            entry = self.fib[FrameType.MPLSU][prefix]
            print(f"{entry}")

class PacketForwardingEngine():

    def __init__(self, forwarding_table: ForwardingTable, router):
        self.router = router
        self.forwarding = forwarding_table
        self.arp_cache = router.arp.cache
        self.logger = router.logger.getChild("pfe")

    # Intended for internal communications
    def accept_frame(self, frame, dest_interface=None):
        self.router.event_manager.observe(
            Event(
                EventType.PACKET_SEND,
                self.router, f"PFE Sending {frame.type}", object=frame, target=dest_interface,
                sub_type="LOCAL_SEND")
            )
        # parameter naming was confusing...
        self.process_frame(frame, dest_interface=dest_interface, from_self=True)

    def process_frame(self, frame, source_interface=None, from_self=False, dest_interface=None):

        def process_ip(pdu, dest_interface=None):
            if pdu.inspectable() and not from_self:
                self.router.process_packet(source_interface, pdu)
                return

            # should be an IPPacket
            potential_next_hops = self.forwarding.lookup_ip(
                pdu.dst
            )
            if potential_next_hops is not None:
                pdu.ttl -= 1
                # TODO: Fire event?
                hop_action = potential_next_hops[0]

                self.logger.info(f"Will apply action {hop_action.action}")

                if not isinstance(hop_action.action, str):
                    newpdu = hop_action.action.apply(pdu, self.router, self.router.event_manager)

                    self.logger.info(f"New pdu is {newpdu}")
                    if isinstance(newpdu, MPLSPacket):
                        hop_action.interface.phy.send(FrameType.MPLSU, newpdu)
                    else:
                        self.logger.warn("Didn't get back an MPLSPacket")
                else:
                    if hop_action.action == 'FORWARD' or dest_interface is not None:
                        # TODO: If we know the dest_interface should we be blindly sending on it?
                        # I'm not too happy about this quite yet
                        # really the link between the RE and PFE is wonky
                        if dest_interface is None:
                            self.logger.debug(f"Using {potential_next_hops[0].interface} for {pdu}")
                            dest_interface = potential_next_hops[0].interface
                        self.logger.debug(f"Using {dest_interface} for {pdu} (potential NH: {potential_next_hops[0]}")
                        self.send_encapsulated(
                            potential_next_hops[0].next_hop_ip,
                            FrameType.IPV4,
                            pdu,
                            dest_interface
                        )
                    elif hop_action.action == 'CONTROL':
                        if from_self:
                            self.logger.error(f"Unexpectedly have frame from self we need to forward {pdu}")
                            raise Exception(f"Unexpectedly have frame from self we need to forward {pdu}")
                        self.router.process_packet(source_interface, pdu)
                    elif hop_action.action == 'REJECT' and source_interface is not None:
                        #print(f"Sending reject from {source_interface.name}:{source_interface.address().ip} to {pdu.source_ip}")

                        packet = IP(
                            dst=pdu.src,
                            src=source_interface.address().ip
                        ) / ICMP(
                            type = ICMPType.DestinationUnreachable,
                            code=UnreachableType.NetworkUnreachable
                        ) / (
                                    pdu.dst,
                                    pdu.src,
                                    pdu.payload.payload  # IRL its first 8 bytes
                                )
                        source_interface.send_ip(packet)
                    else:
                        self.logger.info(f"**** Have action {hop_action.action}")
            else:
                self.logger.warn("**** Need to issue ICMP UNREACHABLE")
                pass
                # send unreachable

        pdu = copy(frame.pdu)
        if frame.type == FrameType.IPV4:
            self.logger.info("Calling process_ip")
            process_ip(pdu, dest_interface)
            # This means we're supposed to look at it
        # special case of control plane...
        elif frame.type == FrameType.ARP:
            # So, dilemma: Here we PROBABLY want to make sure
            # this only happens on switch interfaces?
            # would is also happen on routed interfaces?
            self.router.process_arp(source_interface, pdu)
            # TODO: If we're switching, we also want to forward it!
        elif frame.type == FrameType.CLNS:
            self.router.process['isis'].process_pdu(source_interface, frame.pdu)
        elif frame.type == FrameType.MPLSU:
            # pdu should be an MPLSPacket
            potential_next_hops = None
            try:
                potential_next_hops = self.forwarding.lookup_label(
                    pdu.label_stack[len(pdu.label_stack)-1]
                )
            except:
                if pdu.label_stack[0] == '3':
                    newpdu = PopStackOperation().apply(pdu, self.router, event_manager=self.router.event_manager)
                    if isinstance(newpdu, IP):
                        process_ip(newpdu)
                        return

                self.logger.warn(f"Unable to find {pdu.label_stack[0]}")

            if potential_next_hops is not None:
                fibentry = potential_next_hops[0]
                newpdu = fibentry.action.apply(pdu,
                                               self.router,
                                               event_manager=self.router.event_manager)
                if isinstance(newpdu, MPLSPacket):
                    fibentry.interface.parent.send(
                        FrameType.MPLSU, newpdu, logical=None)
                elif isinstance(newpdu, IP):
                    fibentry.interface.send_ip(newpdu)
                else:
                    print(f"Unknown de-encapsulated packet type!")
            else:
                self.logger.error(f"**** No action found for label {pdu.label_stack[0]}")

    def send_encapsulated(self,
                          next_hop: ipaddress.IPv4Address,
                          type: FrameType,
                          packet,
                          interface: LogicalInterface):
        if next_hop is None:
            dest_ip = packet.dst
            dest_ip_as_net = ipaddress.ip_network(f"{dest_ip}/32")
            if interface.address().network.overlaps(dest_ip_as_net):
                next_hop = dest_ip
            else:
                raise Exception("Valid IP is required")

        hw_address = self.arp_cache[next_hop]
        if hw_address is None:
            # TODO: Drop it?
            self.router.arp.request(next_hop, interface)
        else:
            interface.send(hw_address, type, packet)


# make a time series of instantaneous electric power consumption graph from a csv file

import csv
import glob
import re
import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from statistics import mean

# define variables
timestep = 0.01

def csv_to_graph(path):
    data = pd.read_csv(path, index_col=0, skipinitialspace=True)
    # comvert csv data to a list format data
    current = np.array(data['current'].values.tolist())
    # find the peak value from the list data
    peak_value_index = np.argmax(current)
    # extract useful values from arround the peak value
    arround_peak_value = current[peak_value_index-100:peak_value_index+500]

    # calucurate const value
    const_value = arround_peak_value[len(arround_peak_value)-400:len(arround_peak_value)]
    avg_const_value = round(mean(const_value),2)
    text_avg_const_value = "mean const value = " + str(avg_const_value)
    # make a time series graph
    count = np.arange(0, len(arround_peak_value)/100, timestep)
    plt.plot(count, arround_peak_value)
    plt.xlim(0.0, 6.0)
    plt.ylim(0.0, 10.0)
    plt.xlabel('t [s]')
    plt.ylabel('current [A]')
    font_dict = dict(style="italic",
                    size=16)
    bbox_dict = dict(facecolor="#ffffff",
                    edgecolor="#000000",
                    fill=True)
    plt.text(2.5, 9, text_avg_const_value, font_dict, bbox=bbox_dict)
    plt.grid()
    plt.show()

def make_result_file(path):
    # define variables
    peak_value = []
    mean_const_value = []
    # make file list
    file_list = glob.glob(path+'*.csv')
    # extract the peak value and average const value of each file
    # and append each value to the list
    for file in file_list:
        print(file)
        data = pd.read_csv(file, index_col=0, skipinitialspace=True)
        # comvert csv data to a list format data
        current = np.array(data['current'].values.tolist())
        # find the peak value from the list data
        peak_value_index = np.argmax(current)
        # extract useful values from arround the peak value
        arround_peak_value = current[peak_value_index-100:peak_value_index+500]
        # calucurate const value
        const_value = arround_peak_value[len(arround_peak_value)-400:len(arround_peak_value)]
        avg_const_value = round(mean(const_value),2)
        # calcurate mean value of peak value and average const value
        peak_value.append(np.max(current))
        mean_const_value.append(avg_const_value)
    # make a result file(write each value)
    file_name = path + 'result.txt'
    f = open(file_name, 'a')
    for i in range(len(file_list)):
        peak = peak_value[i]
        const = mean_const_value[i]
        f.write("FILE%s: Peak value: %s, Mean const value: %s \n" % (i, peak, const))
    mean_peak = round(mean(peak_value),2)
    mean_const = round(mean(mean_const_value),2)
    f.write("Mean peak value: %s, Mean const value: %s\n" % (mean_peak, mean_const))
    f.close()

# analyze the step down experiment data
def analyze_gradation_exp(file_list):
    # import csv format file
    for file in file_list:
        data = pd.read_csv(file, index_col=0, skipinitialspace=True)
        # comvert csv data to a list format data
        current = np.array(data['current'].values.tolist())
        # extract the peak value from the list data
        peak_value_index = np.argmax(current)
        start_index = peak_value_index
        # clip the time series data by 1 sec
        grad_data = []
        for i in range(int(len(current[start_index:])/99)):
            grad_data.append(current[start_index:start_index+99*i])
            start_index = start_index + 99
        # calcurate mean value of each data set
        for data in grad_data:
            mean_grad = mean(data)
            print(mean_grad)
        # write results on a text file

if __name__ == "__main__":
    # import csv format file
    """
    # useage: make a time series of power consumption graph
    path = "test.csv"
    csv_to_graph(path)
    """

    # useage: make result files
    path = 'C:/Users/is0232xf/OneDrive - 学校法人立命館/ソースコード/BIWAKO_unit_test/csv/diagonal/25%/'
    make_result_file(path)
    """
    files = os.listdir(path)
    # get subdirectory list
    files_dir = [f for f in files if os.path.isdir(os.path.join(path, f))]
    for subdir in files_dir:
        dir = path + subdir + '/'
        make_result_file(dir)

    """


from typing import Tuple

from sqlalchemy import and_, desc

from quizard_backend import db
from quizard_backend.utils.exceptions import raise_not_found_exception
from quizard_backend.utils.transaction import in_transaction

def dict_to_filter_args(model, **kwargs):
    """
    Convert a dictionary to Gino/SQLAlchemy's conditions for filtering.

    Example:
        A correct Gino's query is:
            User.query.where(
                and_(
                    User.role_id == 10,
                    User.location == "Singapore"
                )
            ).gino.all()

        The given `kwargs` is:
            {
                "role_id": 10,
                "location": "Singapore",
            }
        This function unpacks the given dictionary `kwargs`
        into `and_(*clauses)`.
    """
    return (getattr(model, k) == v for k, v in kwargs.items())

async def get_one(model, **kwargs):
    return (
        await model.query.where(and_(*dict_to_filter_args(model, **kwargs)))
        .limit(1)
        .gino.first()
    )

async def get_many(
    model,
    columns=None,
    after_id=None,
    limit=15,
    in_column=None,
    in_values=None,
    order_by="internal_id",
    descrease=False,
    **kwargs,
):
    # Get the `internal_id` value from the starting row
    # And use it to query the next page of results
    last_internal_id = 0
    if after_id:
        row_of_after_id = await model.query.where(model.id == after_id).gino.first()
        if not row_of_after_id:
            raise_not_found_exception(model, **kwargs)

        last_internal_id = row_of_after_id.internal_id

    # Get certain columns only
    if columns:
        query = db.select([*(getattr(model, column) for column in columns)])
    else:
        query = model.query

    query = query.where(
        and_(
            *dict_to_filter_args(model, **kwargs),
            model.internal_id < last_internal_id
            if descrease and last_internal_id
            else model.internal_id > last_internal_id,
            getattr(model, in_column).in_(in_values)
            if in_column and in_values
            else True,
        )
    )

    return (
        await query.order_by(
            desc(getattr(model, order_by)) if descrease else getattr(model, order_by)
        )
        .limit(limit)
        .gino.all()
    )

async def get_latest_quiz_attempts(model, user_id, limit=15, after_id=None, **kwargs):
    # Get the `internal_id` value from the starting row
    # And use it to query the next page of results
    last_internal_id = 0
    if after_id:
        row_of_after_id = await model.query.where(model.id == after_id).gino.first()
        if not row_of_after_id:
            raise_not_found_exception(model, **kwargs)

        last_internal_id = row_of_after_id.internal_id

    return (
        await db.status(
            db.text(
                """SELECT * FROM (
                    SELECT DISTINCT ON (quiz_attempt.quiz_id, quiz_attempt.user_id)
                        quiz_attempt.quiz_id,
                        quiz_attempt.user_id,
                        quiz_attempt.is_finished,
                        quiz_attempt.internal_id
                    FROM quiz_attempt
                    WHERE quiz_attempt.user_id = :user_id {}
                    ORDER BY
                        quiz_attempt.quiz_id,
                        quiz_attempt.user_id,
                        quiz_attempt.internal_id DESC
                    ) t
                    ORDER By t.internal_id DESC limit :limit;""".format(
                    "and quiz_attempt.internal_id < :last_internal_id"
                    if after_id
                    else ""
                )
            ),
            {"user_id": user_id, "limit": limit, "last_internal_id": last_internal_id},
        )
    )[1]

async def get_one_latest(model, **kwargs):
    return (
        await model.query.where(and_(*dict_to_filter_args(model, **kwargs)))
        .order_by(desc(model.internal_id))
        .limit(1)
        .gino.first()
    )

async def get_many_with_count_and_group_by(
    model, *, columns, in_column=None, in_values=None
):
    return (
        await db.select(
            [*[getattr(model, column) for column in columns], db.func.count()]
        )
        .where(
            getattr(model, in_column).in_(in_values)
            if in_column and in_values
            else True
        )
        .group_by(*[getattr(model, column) for column in columns])
        .gino.all()
    )

@in_transaction
async def create_one(model, **kwargs):
    return await model(**kwargs).create()

@in_transaction
async def update_one(row, **kwargs):
    if not kwargs:
        return row

    await row.update(**kwargs).apply()
    return row

@in_transaction
async def update_many(model, get_kwargs, update_kwargs):
    status: Tuple[str, list] = await model.update.values(**update_kwargs).where(
        and_(*and_(*dict_to_filter_args(model, **get_kwargs)))
    ).gino.status()
    return status[0]

@in_transaction
async def delete_many(model, **kwargs):
    status: Tuple[str, list] = await model.delete.where(
        and_(*dict_to_filter_args(model, **kwargs))
    ).gino.status()
    return status[0]


import sys
import pefile
import re

# Pega os headers de um executável
def get_headers(executable):
    pe = pefile.PE(executable)
    sections = []
    for section in pe.sections:
        sections.append(section.Name.decode('utf-8'))
    return sections

# Pega os headers dos argumentos de entrada
sections1 = get_headers(sys.argv[1])
sections2 = get_headers(sys.argv[2])

# Imprime a intersecção entre as listas
commonSections = list(set(sections1).intersection(sections2))
print("Seções comuns: [", end="")
for i, section in enumerate(commonSections):
    print("'{}'".format(section), end="")
    if i < len(commonSections) - 1:
        print(', ', end="")
print("]\n")

# Imprime a diferença da lista 1 para a lista 2
difference12 = list(set(sections1).difference(sections2))
print(re.sub(".*/", "", sys.argv[1]) + ": [", end="")
for i, section in enumerate(difference12):
    print("'{}'".format(section), end="")
    if i < len(difference12) - 1:
        print(', ', end="")
print("]\n")

# Imprime a diferença da lista 2 para a lista 1
difference21 = list(set(sections2).difference(sections1))
print(re.sub(".*/", "", sys.argv[2]) + ": [", end="")
for i, section in enumerate(difference21):
    print("'{}'".format(section), end="")
    if i < len(difference21) - 1:
        print(', ', end="")
print("]\n")


from __future__ import annotations

from typing import TYPE_CHECKING

from avilla.core.context import Context
from avilla.core.event import RelationshipCreated, RelationshipDestroyed
from avilla.core.selector import Selector
from avilla.core.trait.context import EventParserRecorder
from cai.client.events.group import (
    GroupLuckyCharacterChangedEvent,
    GroupLuckyCharacterClosedEvent,
    GroupLuckyCharacterInitEvent,
    GroupLuckyCharacterNewEvent,
    GroupLuckyCharacterOpenedEvent,
    GroupMemberJoinedEvent,
    GroupMemberLeaveEvent,
    GroupMemberMutedEvent,
    GroupMemberPermissionChangeEvent,
    GroupMemberSpecialTitleChangedEvent,
    GroupMemberUnMutedEvent,
    GroupNameChangedEvent,
    TransferGroupEvent,
)

if TYPE_CHECKING:
    from ..account import CAIAccount
    from ..protocol import CAIProtocol

event = EventParserRecorder["CAIProtocol", "CAIAccount"]

@event("GroupMemberJoinedEvent")
async def group_member_joined_event(
    protocol: CAIProtocol, account: CAIAccount, raw: GroupMemberJoinedEvent
):
    group = Selector().land(protocol.land.name).group(str(raw.group_id))
    member = group.member(str(raw.uin))
    context = Context(
        account=account,
        client=member,
        endpoint=group,
        scene=group,
        selft=group.member(account.id),
    )
    return RelationshipCreated(context, member, group, context.self), context

@event("GroupMemberLeaveEvent")
async def group_member_leave_event(
    protocol: CAIProtocol, account: CAIAccount, raw: GroupMemberLeaveEvent
):
    group = Selector().land(protocol.land.name).group(str(raw.group_id))
    member = group.member(str(raw.uin))
    context = Context(
        account=account,
        client=member,
        endpoint=group,
        scene=group,
        selft=group.member(account.id),
    )
    res = RelationshipDestroyed(context, member, group, context.self)
    if raw.operator and raw.operator != raw.uin:
        res.mediums.append(group.member(str(raw.operator)))
    return res, context


from lib2to3.pgen2 import token
from brownie import Test, accounts, interface
from eth_utils import to_wei
from web3 import Web3

def main():
    deploy()

def deploy():
    amount_in = Web3.toWei(1000000, "ether")

    # DAI address
    DAI = "0x6B175474E89094C44Da98b954EedeAC495271d0F"

    # DAI whale
    DAI_WHALE = "0xcffad3200574698b78f32232aa9d63eabd290703"

    # WETH
    WETH = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"

    # WETH whale
    WETH_WHALE = "0xeD1840223484483C0cb050E6fC344d1eBF0778a9"

    print("===Transferring gas cost covers===")
    # covering the transaction cost
    # accounts[0].transfer(DAI_WHALE, "1 ether")
    # accounts[0].transfer(WETH_WHALE, "1 ether")

    tokenA = interface.IERC20(DAI)
    tokenB = interface.IERC20(WETH)
    print("===Transferring the tokenA and tokenB amounts from whales to account[0]===")
    tokenA.transfer(accounts[0], Web3.toWei(2400, "ether"), {"from": DAI_WHALE})
    tokenB.transfer(accounts[0], Web3.toWei(1, "ether"), {"from": WETH_WHALE})

    contract = Test.deploy({"from": accounts[0]})

    tokenA.approve(contract.address, Web3.toWei(2400, "ether"), {"from": accounts[0]})
    tokenB.approve(contract.address, Web3.toWei(1, "ether"), {"from": accounts[0]})

    print("Adding liquidity...")
    tx = contract.addLiquidity(
        DAI,
        WETH,
        Web3.toWei(2400, "ether"),
        Web3.toWei(1, "ether"),
        {"from": accounts[0]},
    )
    tx.wait(1)

    print("Added Liquidity...")

    for i in tx.events["Log"]:
        print(i)

    print("=== Removing Liquidity ===")

    tx = contract.removeLiquidity(DAI, WETH, {"from": accounts[0]})

    tx.wait(1)

    for i in tx.events["Log"]:
        print(i)


import unittest
import numpy as np
from numpy import linalg
from task import img_rescaled, img_array_transposed, U, s, Vt

class TestCase(unittest.TestCase):
    def test_transpose(self):
        np.testing.assert_array_equal(img_array_transposed, np.transpose(img_rescaled, (2, 0, 1)),
                                      'The transposed array does not look right.')

    def test_svd(self):
        transposed_test = np.transpose(img_rescaled, (2, 0, 1))
        U_test, s_test, Vt_test = linalg.svd(transposed_test)
        np.testing.assert_array_equal(U, U_test,
                                      'Your decomposition does not look right. Go back to "SVD on One Matrix" to refresh the topic.')
        np.testing.assert_array_equal(s, s_test,
                                      'Your decomposition does not look right. Go back to "SVD on One Matrix" to refresh the topic.')
        np.testing.assert_array_equal(Vt, Vt_test,
                                      'Your decomposition does not look right. Go back to "SVD on One Matrix" to refresh the topic.')

Projects/SVD/Applying to All Colors/tests/test_task.py


# 1번 풀이
# import sys
# dx = [0,0,-1,1] # 우좌상하
# dy = [1,-1,0,0]

# def dfs(places, x, y,depth):
#     if depth == 3: # depth 3까지 찾아봤는데 거리두기 잘 지키는 경우 True
#         return True
#     for i in range(4):
#         nx = x + dx[i]
#         ny = y + dy[i]
#         if 0<= nx <5 and 0<= ny <5 and visited[nx][ny] == 0 and places[nx][ny] != 'X':
#             if places[nx][ny] == 'P':
#                 return False
#             else:
#                 visited[nx][ny] = 1
#                 if dfs(places,nx,ny,depth + 1):
#                     visited[nx][ny] = 0
#                 else:
#                     visited[nx][ny] = 0
#                     return False

#     return True

# def solution(places):
#     global visited
#     answer = []
#     for place in places:
#         flag = 0
#         visited = [[0] * 5 for _ in range(5)]
#         for i in range(5):
#             if flag == 1: # 이미 거리두기 안지키는 사람을 발견함
#                 break
#             for j in range(5):
#                 if place[i][j] == 'P' and not visited[i][j]:
#                     visited[i][j] = 1
#                     if dfs(place, i, j,1):
#                         continue
#                     else: # 거리두기 안지키는게 발견
#                         answer.append(0)
#                         flag = 1
#                         break
#         else:
#             answer.append(1)

#     return answer
#2번 풀이
import sys
dx = [0,0,-1,1] # 우좌상하
dy = [1,-1,0,0]

def dfs(place, x, y,depth):
    global check
    if depth == 3: # depth 3까지 찾아봤는데 거리두기 잘 지키는 경우 True
        return
    for i in range(4):
        nx = x + dx[i]
        ny = y + dy[i]
        if 0<= nx <5 and 0<= ny <5 and visited[nx][ny] == 0 and place[nx][ny] != 'X':
            if place[nx][ny] == 'P':
                check = 0
                return
            else:
                visited[nx][ny] = 1
                dfs(place,nx,ny,depth + 1)
                visited[nx][ny] = 0
    return

def solution(places):
    global visited
    global check
    answer = []
    for place in places:
        flag = 0
        check = 1 # 거리두기 잘지킴
        visited = [[0] * 5 for _ in range(5)]
        for i in range(5):
            if flag == 1: # 이미 거리두기 안지키는 사람을 발견함
                break
            for j in range(5):
                if place[i][j] == 'P' and not visited[i][j]:
                    visited[i][j] = 1
                    dfs(place,i,j,1)
                    if check:
                        continue
                    else: # 거리두기 안지키는게 발견
                        answer.append(0)
                        flag = 1
                        break
        else:
            answer.append(1)

    return answer
# 3번 풀이
from collections import deque
def bfs(place):
    dx = [0,0,-1,1] # 우좌상하
    dy = [1,-1,0,0]
    start = []
    q = deque()
    visited = [[0] * 5 for _ in range(5)]
    for i in range(5):
        for j in range(5):
            if place[i][j] == 'P' and not visited[i][j]:
                start.append((i,j))

    for s in start:
        i,j = s
        visited = [[0] * 5 for _ in range(5)]
        visited[i][j] = 1
        q.append(s)
        while q:
            x, y = q.popleft()
            if visited[x][y] < 3:
                for i in range(4):
                    nx = x + dx[i]
                    ny = y + dy[i]
                    if 0 <= nx < 5 and 0<= ny < 5 and place[nx][ny] != 'X' and not visited[nx][ny]:
                        if place[nx][ny] == 'P':
                            return 0
                        else:
                            visited[nx][ny] = visited[x][y] + 1
                            q.append((nx,ny))

    return 1

def solution(places):
    answer = []
    for place in places:
        answer.append(bfs(place))

    return answer
if __name__ == '__main__':
    places = [["POOPX", "OXPXP", "PXXXO", "OXXXO", "OOOPP"],
              ["POOOP", "OXXOX", "OPXPX", "OOXOX", "POXXP"],
              ["PXOPX", "OXOXP", "OXPOX", "OXXOP", "PXPOX"],
              ["OOOXX", "XOOOX", "OOOXX", "OXOOX", "OOOOO"],
              ["PXPXP", "XPXPX", "PXPXP", "XPXPX", "PXPXP"]]
    print(solution(places))

프로그래머스 문제/파이썬/level2/거리두기 확인하기.py

거리두기 확인하기.py


# adapated from munch 2.5.0
from collections.abc import Mapping

class Munch(dict):
    """A dictionary that provides attribute-style access.

    >>> b = Munch()
    >>> b.hello = 'world'
    >>> b.hello
    'world'
    >>> b['hello'] += "!"
    >>> b.hello
    'world!'
    >>> b.foo = Munch(lol=True)
    >>> b.foo.lol
    True
    >>> b.foo is b['foo']
    True

    A Munch is a subclass of dict; it supports all the methods a dict does...

    >>> sorted(b.keys())
    ['foo', 'hello']

    Including update()...

    >>> b.update({ 'ponies': 'are pretty!' }, hello=42)
    >>> print (repr(b))
    Munch({'ponies': 'are pretty!', 'foo': Munch({'lol': True}), 'hello': 42})

    As well as iteration...

    >>> sorted([ (k,b[k]) for k in b ])
    [('foo', Munch({'lol': True})), ('hello', 42), ('ponies', 'are pretty!')]

    And "splats".

    >>> "The {knights} who say {ni}!".format(**Munch(knights='lolcats', ni='can haz'))
    'The lolcats who say can haz!'

    See unmunchify/Munch.toDict, munchify/Munch.fromDict for notes about conversion.
    """

    def __init__(self, *args, **kwargs):  # pylint: disable=super-init-not-called
        self.update(*args, **kwargs)

    # only called if k not found in normal places
    def __getattr__(self, k):
        """Gets key if it exists, otherwise throws AttributeError.

        nb. __getattr__ is only called if key is not found in normal places.

        >>> b = Munch(bar='baz', lol={})
        >>> b.foo
        Traceback (most recent call last):
            ...
        AttributeError: foo

        >>> b.bar
        'baz'
        >>> getattr(b, 'bar')
        'baz'
        >>> b['bar']
        'baz'

        >>> b.lol is b['lol']
        True
        >>> b.lol is getattr(b, 'lol')
        True
        """
        try:
            # Throws exception if not in prototype chain
            return object.__getattribute__(self, k)
        except AttributeError:
            try:
                return self[k]
            except KeyError as exc:
                raise AttributeError(k) from exc

    def __setattr__(self, k, v):
        """Sets attribute k if it exists, otherwise sets key k. A KeyError
        raised by set-item (only likely if you subclass Munch) will
        propagate as an AttributeError instead.

        >>> b = Munch(foo='bar', this_is='useful when subclassing')
        >>> hasattr(b.values, '__call__')
        True
        >>> b.values = 'uh oh'
        >>> b.values
        'uh oh'
        >>> b['values']
        Traceback (most recent call last):
            ...
        KeyError: 'values'
        """
        try:
            # Throws exception if not in prototype chain
            object.__getattribute__(self, k)
        except AttributeError:
            try:
                self[k] = v
            except KeyError as exc:
                raise AttributeError(k) from exc
        else:
            object.__setattr__(self, k, v)

    def __delattr__(self, k):
        """Deletes attribute k if it exists, otherwise deletes key k. A KeyError
        raised by deleting the key--such as when the key is missing--will
        propagate as an AttributeError instead.

        >>> b = Munch(lol=42)
        >>> del b.lol
        >>> b.lol
        Traceback (most recent call last):
            ...
        AttributeError: lol
        """
        try:
            # Throws exception if not in prototype chain
            object.__getattribute__(self, k)
        except AttributeError:
            try:
                del self[k]
            except KeyError as exc:
                raise AttributeError(k) from exc
        else:
            object.__delattr__(self, k)

    def toDict(self):
        """Recursively converts a munch back into a dictionary.

        >>> b = Munch(foo=Munch(lol=True), hello=42, ponies='are pretty!')
        >>> sorted(b.toDict().items())
        [('foo', {'lol': True}), ('hello', 42), ('ponies', 'are pretty!')]

        See unmunchify for more info.
        """
        return unmunchify(self)

    @property
    def __dict__(self):
        return self.toDict()

    def __repr__(self):
        """Invertible* string-form of a Munch.

        >>> b = Munch(foo=Munch(lol=True), hello=42, ponies='are pretty!')
        >>> print (repr(b))
        Munch({'ponies': 'are pretty!', 'foo': Munch({'lol': True}), 'hello': 42})
        >>> eval(repr(b))
        Munch({'ponies': 'are pretty!', 'foo': Munch({'lol': True}), 'hello': 42})

        >>> with_spaces = Munch({1: 2, 'a b': 9, 'c': Munch({'simple': 5})})
        >>> print (repr(with_spaces))
        Munch({'a b': 9, 1: 2, 'c': Munch({'simple': 5})})
        >>> eval(repr(with_spaces))
        Munch({'a b': 9, 1: 2, 'c': Munch({'simple': 5})})

        (*) Invertible so long as collection contents are each repr-invertible.
        """
        return f"{self.__class__.__name__}({dict.__repr__(self)})"

    def __dir__(self):
        return list(self.keys())

    def __getstate__(self):
        """Implement a serializable interface used for pickling.

        See https://docs.python.org/3.6/library/pickle.html.
        """
        return {k: v for k, v in self.items()}

    def __setstate__(self, state):
        """Implement a serializable interface used for pickling.

        See https://docs.python.org/3.6/library/pickle.html.
        """
        self.clear()
        self.update(state)

    __members__ = __dir__  # for python2.x compatibility

    @classmethod
    def fromDict(cls, d):
        """Recursively transforms a dictionary into a Munch via copy.

        >>> b = Munch.fromDict({'urmom': {'sez': {'what': 'what'}}})
        >>> b.urmom.sez.what
        'what'

        See munchify for more info.
        """
        return munchify(d, cls)

    def copy(self):
        return type(self).fromDict(self)

    def update(self, *args, **kwargs):
        """
        Override built-in method to call custom __setitem__ method that may
        be defined in subclasses.
        """
        for k, v in dict(*args, **kwargs).items():
            self[k] = v

    def get(self, k, d=None):
        """
        D.get(k[,d]) -> D[k] if k in D, else d.  d defaults to None.
        """
        if k not in self:
            return d
        return self[k]

    def setdefault(self, k, d=None):
        """
        D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D
        """
        if k not in self:
            self[k] = d
        return self[k]

def munchify(x):
    """Recursively transforms a dictionary into a Munch via copy.

    >>> b = munchify({'urmom': {'sez': {'what': 'what'}}})
    >>> b.urmom.sez.what
    'what'

    munchify can handle intermediary dicts, lists and tuples (as well as
    their subclasses), but ymmv on custom datatypes.

    >>> b = munchify({ 'lol': ('cats', {'hah':'i win again'}),
    ...         'hello': [{'french':'salut', 'german':'hallo'}] })
    >>> b.hello[0].french
    'salut'
    >>> b.lol[1].hah
    'i win again'

    nb. As dicts are not hashable, they cannot be nested in sets/frozensets.
    """
    # Munchify x, using `seen` to track object cycles
    seen = dict()

    def munchify_cycles(obj):
        # If we've already begun munchifying obj, just return the already-created munchified obj
        try:
            return seen[id(obj)]
        except KeyError:
            pass

        # Otherwise, first partly munchify obj (but without descending into any lists or dicts) and save that
        seen[id(obj)] = partial = pre_munchify(obj)
        # Then finish munchifying lists and dicts inside obj (reusing munchified obj if cycles are encountered)
        return post_munchify(partial, obj)

    def pre_munchify(obj):
        # Here we return a skeleton of munchified obj, which is enough to save for later (in case
        # we need to break cycles) but it needs to filled out in post_munchify
        if isinstance(obj, Mapping):
            return Munch({})
        elif isinstance(obj, list):
            return type(obj)()
        elif isinstance(obj, tuple):
            type_factory = getattr(obj, "_make", type(obj))
            return type_factory(munchify_cycles(item) for item in obj)
        else:
            return obj

    def post_munchify(partial, obj):
        # Here we finish munchifying the parts of obj that were deferred by pre_munchify because they
        # might be involved in a cycle
        if isinstance(obj, Mapping):
            partial.update((k, munchify_cycles(obj[k])) for k in obj.keys())
        elif isinstance(obj, list):
            partial.extend(munchify_cycles(item) for item in obj)
        elif isinstance(obj, tuple):
            for item_partial, item in zip(partial, obj):
                post_munchify(item_partial, item)

        return partial

    return munchify_cycles(x)

def unmunchify(x):
    """Recursively converts a Munch into a dictionary.

    >>> b = Munch(foo=Munch(lol=True), hello=42, ponies='are pretty!')
    >>> sorted(unmunchify(b).items())
    [('foo', {'lol': True}), ('hello', 42), ('ponies', 'are pretty!')]

    unmunchify will handle intermediary dicts, lists and tuples (as well as
    their subclasses), but ymmv on custom datatypes.

    >>> b = Munch(foo=['bar', Munch(lol=True)], hello=42,
    ...         ponies=('are pretty!', Munch(lies='are trouble!')))
    >>> sorted(unmunchify(b).items()) #doctest: +NORMALIZE_WHITESPACE
    [('foo', ['bar', {'lol': True}]), ('hello', 42), ('ponies', ('are pretty!', {'lies': 'are trouble!'}))]

    nb. As dicts are not hashable, they cannot be nested in sets/frozensets.
    """

    # Munchify x, using `seen` to track object cycles
    seen = dict()

    def unmunchify_cycles(obj):
        # If we've already begun unmunchifying obj, just return the already-created unmunchified obj
        try:
            return seen[id(obj)]
        except KeyError:
            pass

        # Otherwise, first partly unmunchify obj (but without descending into any lists or dicts) and save that
        seen[id(obj)] = partial = pre_unmunchify(obj)
        # Then finish unmunchifying lists and dicts inside obj (reusing unmunchified obj if cycles are encountered)
        return post_unmunchify(partial, obj)

    def pre_unmunchify(obj):
        # Here we return a skeleton of unmunchified obj, which is enough to save for later (in case
        # we need to break cycles) but it needs to filled out in post_unmunchify
        if isinstance(obj, Mapping):
            return dict()
        elif isinstance(obj, list):
            return type(obj)()
        elif isinstance(obj, tuple):
            type_factory = getattr(obj, "_make", type(obj))
            return type_factory(unmunchify_cycles(item) for item in obj)
        else:
            return obj

    def post_unmunchify(partial, obj):
        # Here we finish unmunchifying the parts of obj that were deferred by pre_unmunchify because they
        # might be involved in a cycle
        if isinstance(obj, Mapping):
            partial.update((k, unmunchify_cycles(obj[k])) for k in obj.keys())
        elif isinstance(obj, list):
            partial.extend(unmunchify_cycles(v) for v in obj)
        elif isinstance(obj, tuple):
            for value_partial, value in zip(partial, obj):
                post_unmunchify(value_partial, value)

        return partial

    return unmunchify_cycles(x)


# import libraries
import sys
import nltk
nltk.download(['punkt', 'wordnet', 'stopwords'])

import re
import numpy as np
import pandas as pd
from sqlalchemy import create_engine
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline
import pickle

def load_data(database_filepath):
    """
    Loads table from database as a Pandas Dataframe
    and returns the following:
    X -- feature dataset containing the messages to be
         categorized
    y -- label dataset containing the 36 categories that each
         message is assigned to.
    category_names -- list containing category names

    Keyword arguments:
    database_filepath -- filepath (including file name)
                         of the database containing the
                         messages and categories
    """
    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql_table('messages_and_categories', engine)
    X = df['message']
    y = df.drop(columns=['id', 'message', 'original', 'genre'])
    category_names = list(y.columns)

    return X, y, category_names

def tokenize(text):
    """
    Cleans, tokenizes, lemmatizes messages in preparation for
    classification algorithm

    1) finds and replaces urls with a placeholder
    2) finds and replaces non alphanumeric characters with a space
    3) removes stop words from tokenized messages
    4) strips leading and trailing spaces and lowcases lemmatized
       tokens

    Keyword arguments:
    text -- raw message that will be cleaned, tokenized
    """
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    text = re.sub(r'\W+', ' ', text)

    tokens = word_tokenize(text)
    tokens = [t for t in tokens if t not in stopwords.words("english")]
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

def build_model():
    """
    Creates a pipeline and grid search for hyperparameter tuning
    returns pipeline with the specified parameter search space
    """
    pipeline = Pipeline([
    ('vect', CountVectorizer(tokenizer=tokenize)),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(estimator=AdaBoostClassifier()))
    ])

    # specify parameters for grid search
    parameters = {
        'vect__ngram_range': ((1, 1), (1, 2),(2,2)),
        'tfidf__use_idf': (True, False),
        'tfidf__norm': ('l1', 'l2'),
        'clf__estimator__learning_rate': [0.1, 0.5],
        'clf__estimator__n_estimators': [50, 60, 70]
        }
    # create grid search object
    cv = GridSearchCV(pipeline, param_grid=parameters, verbose=216)
    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """
    Generates predicted values for test data based on
    fit model. Outputs a classification report for each category.

    Keyword arguments:
    model -- fit model based on training data
    X_test, Y_test -- message and target category values for testing
    category_names -- list of possible categories for each message
    """
    Y_pred = model.predict(X_test)
    for i, label in enumerate(category_names):
        print(category_names[i])
        print(classification_report(Y_test[label], Y_pred[:,i]))

def save_model(model, model_filepath):
    """
    Export the classifier to a pickle file

    Keyword arguments:
    model -- final model
    model_filepath -- location and name of saved pickle file
    """
    with open(model_filepath, 'wb') as model_filepath:
        pickle.dump(model, model_filepath)

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state = 42)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)
        print(model.best_score_)
        print(model.best_params_)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


import sys
from aspartix_parser import Apx_parser
import itertools

def conflict_free(arguments, attacks):
    confl_free_sets = []
    combs = []

    for i in range(1, len(arguments) + 1):
        els = [list(x) for x in itertools.combinations(arguments, i)]
        combs.extend(els)
    combs_sorted = [list(combs_sorted) for combs_sorted in combs][::-1]
    # print("Combs: ", combs_sorted)
    for i in combs_sorted:
        att_count = 0
        # print(i)
        for att in attacks:
            # print(att)
            if set([str(att)[2], str(att)[4]]).issubset(set(i)) or any([i in item for item in
                                                                        confl_free_sets]):  # any(x in item for item in confl_free_sets for x in i):#(True if list(filter(lambda x:i in x,confl_free_sets)) else False):#(any([set(i).issubset(set(item)) in item for item in confl_free_sets])):
                break
            else:
                att_count += 1
            #    print(att_count)
            # if ((str(att)[2] and str(att)[4]) not in i) and (not any([i in item for item in confl_free_sets])):
            #     att_count += 1
            #     print(att_count)
            if att_count == len(attacks):
                confl_free_sets.append(i)
    return confl_free_sets

def admissible(confl_free, attacks):
    admissible_sets = []

    for ext in confl_free:
        count = 0
        # print(ext)
        for att in attacks:
            if str(att)[4] not in ext:
                count += 1
                # print(att, count)
            else:
                # print(att, count)
                for atr in attacks:
                    # print(att, atr, count, str(atr)[4], str(att)[2])
                    if (str(att)[2] == str(atr)[4]) and (str(atr)[2] in ext):
                        count += 1
                        # print(count)
        if count == len(attacks):
            admissible_sets.append(ext)

    return admissible_sets

# def complete(admissible_sets, attacks):
#     complete_ext = []
#     for adm in admissible_sets:
#         for att in attacks:
#             if (str(att)[2] in adm) and
#
#     return complete_ext

def preferred(admissible_sets):
    preferred_exts = []

    for adm in admissible_sets:
        count = 0
        # print(adm, count)
        for adm_t in admissible_sets:
            if set(adm).issubset(set(adm_t)) and adm_t != adm:
                pass
            else:
                count += 1
                # print(adm, adm_t, count)
            if count == len(admissible_sets):
                preferred_exts.append(adm)

    return preferred_exts

def stable_extensions(stable_exts):
    pass

if __name__ == '__main__':
    filepath = 'example.apx'
    if sys.argv[1:]:
        filepath = sys.argv[1]

    arguments = []
    attacks = []
    parser = Apx_parser(filepath)
    arguments, attacks = parser.read_file()
    parser.close()
    print(arguments, attacks)
    print("There are ", len(arguments), " arguments and ", len(attacks), " attacks.")
    # print(str(attacks[0])[4])
    confl_free = conflict_free(arguments, attacks)
    print("Conflict free extensions: ", "[]", sorted(confl_free))
    admissible_sets = admissible(confl_free, attacks)
    print("Admissible extensions: ", "[]", sorted(admissible_sets))
    # complete_ext = complete(admissible_sets, attacks)
    preferred_exts = preferred(admissible_sets)
    print("Preferred extensions: ", sorted(preferred_exts))
    stable_exts = stable_extensions(preferred_exts)
    print("Stable extensions: ", stable_exts)


import os
import json

import flask
from vrprot.alphafold_db_parser import AlphafoldDBParser

import vrprot

from . import map_uniprot
from . import settings as st
from . import util
from .classes import NodeTags as NT

def get_scales(uniprot_ids=[], mode=st.DEFAULT_MODE):
    return vrprot.overview_util.get_scale(uniprot_ids, mode)

def run_pipeline(proteins: list, parser: AlphafoldDBParser = st.parser, **kwargs):
    # create the output directory for the corresponding coloring mode if they do not exist
    # output_dir = os.path.join(st._MAPS_PATH, parser.processing)
    output_dir = os.path.join(st._MAPS_PATH)
    parser.update_output_dir(output_dir)

    # initialize the structures dictionary of the parser and check wether some processing files do already exist
    parser.init_structures_dict(proteins)
    for protein in proteins:
        parser.update_existence(protein)
    # run the batched process
    try:
        parser.fetch_pipeline(proteins, **kwargs)
        # batch([parser.fetch_pdb, parser.pdb_pipeline], proteins, parser.batch_size)
    except vrprot.exceptions.ChimeraXException as e:
        return {"error": "ChimeraX could not be found. Is it installed?"}
    result = get_scales(proteins, parser.processing)

    # update the existence of the processed files
    for protein in proteins:
        parser.update_existence(protein)

    return result

def fetch_from_request(request: flask.Request, parser: AlphafoldDBParser = st.parser):
    # get information from request
    pdb_id = request.args.get("id")
    if pdb_id is None:
        return {
            "error": "No PDB ID provided.",
            "example": f"{request.host}/vrprot/fetch?id=P69905",
        }

    # extract processing mode and alphafold version from request
    parser = util.parse_request(parser, request)
    # if mode is not part of the list of available modes, return an error
    if isinstance(parser, dict):
        return parser

    # create a list of proteins to be processed
    proteins = [pdb_id]
    return fetch(proteins, parser)

def fetch(proteins: list[str], parser: AlphafoldDBParser = st.parser):
    # run the batched process
    parser.not_fetched = set()
    parser.already_exists = set()
    result = run_pipeline(proteins, parser)
    # Try whether you can find an updated UniProt id
    second_try = {}
    if len(parser.not_fetched) > 0:
        try:
            mapped_ac = map_uniprot.main(
                parser.not_fetched,
                source_db=map_uniprot.Databases.uniprot_ac,
                target_db=map_uniprot.Databases.uniprot,
            )
            for re in mapped_ac["results"]:
                a, b = True, True
                while a and b:
                    a = re.get("from")
                    b = re.get("to")
                    b = b.get("uniProtKBCrossReferences")
                    for entry in b:
                        if entry.get("database") == "AlphaFoldDB":
                            b = entry.get("id")
                            second_try[b] = a
                            if a in parser.not_fetched:
                                parser.not_fetched.remove(a)
                            break
                    break
            result.update(run_pipeline(second_try, parser))
            tmp = parser.not_fetched.copy()
            for ac in tmp:
                if ac in second_try:
                    parser.not_fetched.remove(ac)
                    parser.not_fetched.add(second_try[ac])
        except Exception as e:
            print(e)
    return {
        "not_fetched": list(parser.not_fetched),
        "already_exists": list(parser.already_processed),
        "results": result,
        "alternative_ids": {v: k for k, v in second_try.items()},
    }

def for_project(
    project: str, request: flask.request, parser: AlphafoldDBParser = st.parser
):
    # get information from request

    if project is None:
        return {"error": "No project provided."}

    # extract processing mode and alphafold version from request
    parser = util.parse_request(parser, request)

    # if mode is not part of the list of available modes, return an error
    if isinstance(parser, dict):
        return parser

    # extract node data from the projects nodes.json file
    nodes_files = os.path.join(st._PROJECTS_PATH, project, "nodes.json")
    if not os.path.isfile(nodes_files):
        return {"error": "Project does not exist."}

    with open(nodes_files, "r") as f:
        nodes = json.load(f)["nodes"]

    # extract the uniprot ids from the nodes
    proteins = [",".join(node[NT.uniprot]) for node in nodes if node.get(NT.uniprot)]

    # run the batched process
    result = run_pipeline(proteins, parser, on_demand=False)
    return {"not_fetched": list(parser.not_fetched), "results": result}

def fetch_list_from_request(
    request: flask.Request, parser: AlphafoldDBParser = st.parser
):
    # get information from request
    pdb_ids = request.args.get("ids")
    if pdb_ids is None:
        return {
            "error": "No PDB IDs provided.",
            "example": f"http://{request.host}/vrprot/list?ids=P02655,At1g58602",
        }

    # extract processing mode and alphafold version from request
    parser = util.parse_request(parser, request)
    # if mode is not part of the list of available modes, return an error
    if isinstance(parser, dict):
        return parser

    # create a list of proteins to be processed
    proteins = [id for id in pdb_ids.split(",")]

    return fetch_list(proteins, parser)

def fetch_list(proteins: list[str], parser: AlphafoldDBParser = st.parser):
    # run the batched process
    result = run_pipeline(proteins, parser, on_demand=False)

    return {"not_fetched": list(parser.not_fetched), "results": result}


from math import sqrt
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

x1_list = []
x2_list = []
y_list = []
counter = 0

def show(x1_list, x2_list):
    N = int(x1_list.__len__())
    if (N <= 0):
        return

    fig, ax = plt.subplots(subplot_kw={"projection": "3d"})

    x1_array = np.arange(min(x1_list) - 1, max(x1_list) + 1, 0.01)
    x2_array = np.arange(min(x2_list) - 1, max(x2_list) + 1, 0.01)
    #x1_array = np.arange(-6, 3, 0.1)
    #x2_array = np.arange(-6, 6, 0.1)

    x1_array, x2_array = np.meshgrid(x1_array, x2_array)
    R = f(x1_array, x2_array)

    ax = Axes3D(fig)

    ax.set_xlabel('x1')
    ax.set_ylabel('x2')
    ax.set_zlabel('f(x1,x2)')
    ax.plot_surface(x1_array, x2_array, R, color='b', alpha=0.5)

    x1_list2 = []
    x2_list2 = []
    f_list = []

    ax.scatter(x1_list[0], x2_list[0], f(x1_list[0], x2_list[0]), c='black')
    x1_list2.append(x1_list[0])
    x2_list2.append(x2_list[0])
    f_list.append(f(x1_list[0], x2_list[0]))
    #print(x1_list[0], x2_list[0], f(x1_list[0], x2_list[0]))

    for n in range(1, N):

        ax.scatter(x1_list[n], x2_list[n], f(x1_list[n], x2_list[n]), c='red')
        x1_list2.append(x1_list[n])
        x2_list2.append(x2_list[n])
        f_list.append(f(x1_list[n], x2_list[n]))
        #print(x1_list[n], x2_list[n], f(x1_list[n], x2_list[n]))

    ax.scatter(x1_list[N - 1], x2_list[N - 1], f(x1_list[N - 1], x2_list[N - 1]), c='green')
    #print(x1_list[N - 1], x2_list[N - 1], f(x1_list[N - 1], x2_list[N - 1]))

    ax.plot(x1_list2, x2_list2, f_list, color="black")

    plt.show()

def f(x1, x2):
    return 3*x1**4 - x1*x2 + x2**4 - 7*x1 - 8*x2 + 2
def f_x1(x1, x2):
    return 12*x1**3 - x2 - 7
def f_x2(x1, x2):
    return 4*x2**3 - x1 - 8

def gradient(x1, x2):
    i = f_x1(x1, x2)
    j = f_x2(x1, x2)
    return [i, j]

def module_of_gradient(grad):
    i = 0; j = 1
    return sqrt(grad[i]**2 + grad[j]**2)

def dichotomy_mehod(a, b, epsilon, x1, x2, d1, d2):
    x = (a + b) / 2
    global counter
    counter += 2

    if (f(x1 + (x - epsilon)*d1, x2 + (x - epsilon)*d2) < f(x1 + (x + epsilon)*d1, x2 + (x + epsilon)*d2)):
        b = x
    else:
        a = x

    if(abs(b - a) >= 2 * epsilon):
        return dichotomy_mehod(a, b, epsilon, x1, x2, d1, d2)
    return x

def the_fletcher_reevse_method(x1, x2, e1, e2, M):
    global counter
    k = 0
    d_prev = [0, 0]
    grad_prev = 0
    while True:
        counter += 2
        grad = gradient(x1, x2)
        module_grad = module_of_gradient(grad)
        if ((module_grad < e1) | (k >= M)):
            return [(round(x1, round_num), round(x2, round_num), round(f(x1, x2), round_num)), k]

        B = 0

        if k % 2 == 1: B = module_of_gradient(grad)**2 / module_of_gradient(grad_prev)**2

        d = [-grad[0] + B * d_prev[0], -grad[1] + B * d_prev[1]]
        t = dichotomy_mehod(0, 0.1, e1, x1, x2, d[0], d[1])

        x1_next = x1 + t * d[0]
        x2_next = x2 + t * d[1]

        x1_list.append(x1); x2_list.append(x2)

        counter += 1
        if ((sqrt(abs(x1_next - x1)**2 + abs(x2_next - x2)**2) <= e2)
            & (abs(f(x1_next, x2_next) - f(x1, x2)) <= e2)):
            return [(round(x1_next, round_num),
                     round(x2_next, round_num),
                     round(f(x1_next, x2_next), round_num)),
                    k]

        x1 = x1_next; x2 = x2_next
        d_prev = d; grad_prev = grad
        k += 1

round_num = 3
x1 = -5
x2 = 3
e1 = 0.001
e2 = 0.001
M = 100

result = the_fletcher_reevse_method(x1, x2, e1, e2, M)
print(f"The Fletcher Reevse method: {result[0]}; count of iteractions = {result[1]}")
print('Count of compute function =', counter)

#show(x1_list, x2_list)

6 Семестр/Оптимизация/Lab_4_1.py


from tempfile import gettempdir
import urllib.request
import platform
import zipfile
from os.path import join
from os import walk

pth = "https://github.com/AequilibraE/aequilibrae/releases/download/V0.6.0.post1/mod_spatialite-NG-win-amd64.zip"

outfolder = gettempdir()

dest_path = join(outfolder, "mod_spatialite-NG-win-amd64.zip")
urllib.request.urlretrieve(pth, dest_path)

fldr = join(outfolder, "temp_data")
zipfile.ZipFile(dest_path).extractall(fldr)

if "WINDOWS" in platform.platform().upper():
    # We now set sqlite. Only needed in thge windows server in Github
    plats = {
        "x86": "https://sqlite.org/2020/sqlite-dll-win32-x86-3320100.zip",
        "x64": "https://sqlite.org/2020/sqlite-dll-win64-x64-3320100.zip",
    }

    outfolder = "C:/"
    zip_path64 = join(outfolder, "sqlite-dll-win64-x64-3320100.zip")
    urllib.request.urlretrieve(plats["x64"], zip_path64)

    zip_path86 = join(outfolder, "sqlite-dll-win32-x86-3320100.zip")
    urllib.request.urlretrieve(plats["x86"], zip_path86)

    root = "C:/hostedtoolcache/windows/Python/"
    file = "sqlite3.dll"
    for d, subD, f in walk(root):
        if file in f:
            if "x64" in d:
                zipfile.ZipFile(zip_path64).extractall(d)
            else:
                zipfile.ZipFile(zip_path86).extractall(d)
            print(f"Replaces {d}")


"""
Parser.py

Used to parse URLs into a linked list of dictionaries.
"""

from bs4 import BeautifulSoup
import requests
import re

class Node:  # pragma: no cover
    """
    Creates a Node that contains data, and a next node
    Data holds any object.
    Next points to the next node, and should always be a node.
    """

    def __init__(
            self, data):
        """Initialize Node Class"""
        self.data = data
        self.next = None

class LinkedList:  # pragma: no cover
    """
    Creates a Linked List, with a head, and a tail.
    Head only contains the first link in the list, and should be called at the
    beginning of scan.
    Tail only contains the last link in the list, and should not be called.
    """

    def __init__(
            self):
        """Initialize Linked List Class"""
        self.head = None
        self.tail = None

    def add_list_item(
            self, item):
        """Add an item to the Linked List"""
        if not isinstance(item, Node):
            item = Node(item)

        if self.head is None:
            self.head = item
        elif self.tail.data == item:
            return
        else:
            self.tail.next = item

        self.tail = item

def parse_url_feed(
        incoming) -> LinkedList:
    """
    Receives either a list of URLs or a single URL, and returns a Linked
    List of Dictionaries
    """
    total_feed = LinkedList()
    url_list = return_list(incoming)
    for url_entry in url_list:
        if not check_url(url_entry):
            raise Exception("Invalid URL. Must Be a RSS Feed URL ending in "
                            ".rss, .html, or .xml: " + url_entry)
        parse_value = find_parser(url_entry)
        response = requests.get(url_entry)
        soup = BeautifulSoup(response.content, parse_value)
        if soup.rss is not None:
            feed = rss_parse(soup)
            total_feed.add_list_item(feed)
        elif soup.find_all(re.compile("atom.xml")) is not None:
            feed = atom_parse(soup)
            total_feed.add_list_item(feed)
    return total_feed

def check_url(
        url: str) -> bool:
    """Checks to see if the URL given is parseable"""
    url = str(url)
    if len(url) == 0:
        return False
    result1 = re.search("rss?", url)
    result2 = re.search("xml?", url)
    result3 = re.search("tml?", url)
    result4 = re.search("feeds?", url)
    if result1 is not None:
        return True
    elif result2 is not None:
        return True
    elif result3 is not None:
        return True
    elif result4 is not None:
        return True
    else:
        return False

def find_parser(
        response: str) -> str:
    """Checks to see which parser to use"""
    if len(response) <= 3:
        raise Exception("Invalid URL Length")
    result = re.search("tml?", response)
    if result is not None:
        return "lxml"
    else:
        return "lxml-xml"

def return_list(
        incoming) -> list:
    """
    Checks to see if incoming is a String or a List. If a String, adds the
    string to a list and returns.
    """
    url_list = []
    if isinstance(incoming, str):
        url_list.append(incoming)
    elif isinstance(incoming, list):
        url_list = incoming
    return url_list

def rss_parse(
        soup: BeautifulSoup) -> LinkedList:  # pragma: no cover
    """
    When URL is an RSS feed, returns a linked list of dictionaries
    containing the titles and links
    """
    feed = LinkedList()
    tag = soup.rss
    tag = tag.channel
    channel_dict = {"RSS_String": tag.title.string, "Link": tag.link.string}
    feed.add_list_item(channel_dict)
    for item in tag.find_all(re.compile("item?")):
        feed_dict = {}
        for title in item.find_all(re.compile("title?")):
            for entry in title.find_all(string=True):
                feed_dict["RSS_String"] = entry
                feed_dict["RSS_String"] = truncate(feed_dict["RSS_String"])
        for link in item.find_all(re.compile("link?")):
            for entry in link.find_all(string=True):
                feed_dict["Link"] = entry
        feed.add_list_item(feed_dict)
    return feed

def atom_parse(
        soup: BeautifulSoup) -> LinkedList:  # pragma: no cover
    """
    When URL is an Atom feed, returns a linked list of dictionaries containing
    the titles and links
    """
    feed = LinkedList()
    tag = soup.feed
    for entry in tag.find_all("entry"):
        feed_dict = {}
        for title in entry.find_all("title"):
            for string in title.find_all(string=True):
                feed_dict["RSS_String"] = string
                feed_dict["RSS_String"] = truncate(feed_dict["RSS_String"])
        for link in entry.find_all(re.compile("link?")):
            feed_dict["Link"] = link.get('href')
        feed.add_list_item(feed_dict)
    return feed

def truncate(
        input_line: str) -> str:
    """
    When a string is over 80 characters long, string is limited to 79
    characters for readability in GUI window, An ellipsis (...) is added to
    denote unseen text
    """
    if len(input_line) >= 80:
        input_line = input_line[0:79]
        return input_line + "..."
    else:
        return input_line


#!/usr/bin/python

### File Information ###
"""
Rejector
"""

__author__ = 'duanqz@gmail.com'

import os
import fnmatch
from config import Config

class Rejector:
    """ Rejector:
        1. Check whether conflicts happen.
        2. Resolve conflicts automatically.
    """

    CONFILCT_START = "<<<<<<<"
    CONFLICT_MID   = "======="
    CONFILCT_END   = ">>>>>>>"

    def __init__(self, target):
        self.mTarget = target
        self.mConflictNum = 0

    def getConflictNum(self):
        if   fnmatch.fnmatch(self.mTarget, "*.xml"):
            self.resolveConflict()
        else:
            self.collectConflict()

        return self.mConflictNum

    def collectConflict(self):
        """ Check whether conflict happen or not in the target
        """

        self.mConflictNum = 0

        top = 0
        size = 0

        # delLinesNumbers record the lines of conflicts
        delLineNumbers = []
        needToDel = False

        targetFile = open(self.mTarget, "r+")

        lineNum = 0
        lines = targetFile.readlines()

        for line in lines:
            size = self.mConflictNum
            if line.startswith(Rejector.CONFILCT_START):

                top += 1

                # Modify the conflict in the original
                lines[lineNum] = "%s #Conflict %d\n" % (line.rstrip(), size)
                self.mConflictNum += 1

                #conflicts.append("#Conflict %d , start at line %d\n" % (size, lineNum))
                #conflicts[size] += line

                delLineNumbers.append(lineNum)

            elif line.startswith(Rejector.CONFILCT_END):

                # Modify the conflict in the original
                lines[lineNum] = "%s #Conflict %d\n" % (line.rstrip(), size-top)

                #conflicts[size-top] += line
                #conflicts[size-top] += "#Conflict %d , end at line %d\n\n" % (size-top, lineNum)

                delLineNumbers.append(lineNum)
                needToDel = False

                if top == 0: break;
                top -= 1

            else:
                if top > 0:
                    #conflicts[size-top] += line

                    if line.startswith(Rejector.CONFLICT_MID):
                        # Modify the conflict in the original
                        #lines[lineNum] = "%s #Conflict %d\n" % (line.rstrip(), size-top)
                        needToDel = True

                    if needToDel:
                        delLineNumbers.append(lineNum)

            lineNum += 1

        # Create a reject file if conflict happen
        if self.mConflictNum > 0:
            rejFilename = Rejector.createReject(self.mTarget)
            rejFile = open(rejFilename, "wb")
            rejFile.writelines(lines)
            rejFile.close()

        # Remove conflict blocks, and write back target.
        for lineNum in delLineNumbers[::-1]:
            del lines[lineNum]

        targetFile.seek(0)
        targetFile.truncate()
        targetFile.writelines(lines)
        targetFile.close()

        return self

    @staticmethod
    def createReject(target):
        relTarget = os.path.relpath(target, Config.PRJ_ROOT)
        rejFilename = os.path.join(Config.REJ_ROOT, relTarget)
        dirname = os.path.dirname(rejFilename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        return rejFilename

    def resolveConflict(self):

        rejFileHandle = open(self.mTarget, "r+")

        top = 0
        lineNum = 0

        delLineNumbers = []
        needToDel = True

        lines = rejFileHandle.readlines()
        for line in lines:
            if line.startswith(Rejector.CONFILCT_START):
                top += 1
                delLineNumbers.append(lineNum)

            elif line.startswith(Rejector.CONFILCT_END):
                top -= 1
                delLineNumbers.append(lineNum)
                needToDel = True

                if top < 0: break;
            else:
                if top > 0:
                    if needToDel:
                        delLineNumbers.append(lineNum)

                    if line.startswith(Rejector.CONFLICT_MID):
                        needToDel = False

            lineNum += 1

        for lineNum in delLineNumbers[::-1]:
            del lines[lineNum]

        rejFileHandle.seek(0)
        rejFileHandle.truncate()
        rejFileHandle.writelines(lines)
        rejFileHandle.close()


import os.path
import unittest
from pathlib import Path

from sflkit.analysis.analysis_type import AnalysisType
from sflkit.analysis.spectra import Spectrum
from sflkit.analysis.suggestion import Location

from tests4py import framework
from tests4py.constants import DEFAULT_WORK_DIR
from utils import BaseTest

class TestSFL(BaseTest):
    @unittest.skip
    def test_middle(self):
        project_name = "middle"
        bug_id = 2
        report = framework.default.tests4py_checkout(project_name, bug_id)
        if report.raised:
            raise report.raised
        src = Path(report.location)
        dst = DEFAULT_WORK_DIR / "sfl"
        report = framework.sfl.tests4py_sfl_instrument(src, dst)
        if report.raised:
            raise report.raised

        dst_src = dst / "src"
        dst_src_middle = dst_src / "middle"
        dst_src_middle___init___py = dst_src_middle / "__init__.py"
        dst_tests = dst / "tests"
        dst_tests_test_middle_py = dst_tests / "test_middle.py"
        dst_gitignore = dst / ".gitignore"
        dst_license = dst / "LICENSE"
        dst_readme_md = dst / "README.md"
        dst_setup_cfg = dst / "setup.cfg"
        dst_setup_py = dst / "setup.py"

        src_src = src / "src"
        src_src_middle = src_src / "middle"
        src_src_middle___init___py = src_src_middle / "__init__.py"
        src_tests = src / "tests"
        src_tests_test_middle_py = src_tests / "test_middle.py"
        src_gitignore = src / ".gitignore"
        src_license = src / "LICENSE"
        src_readme_md = src / "README.md"
        src_setup_cfg = src / "setup.cfg"
        src_setup_py = src / "setup.py"

        exist_files = [
            dst_src_middle___init___py,
            dst_tests_test_middle_py,
            dst_gitignore,
            dst_license,
            dst_readme_md,
            dst_setup_cfg,
            dst_setup_py,
        ]
        exist_dirs = [dst_src, dst_src_middle, dst_tests]

        for d in exist_dirs:
            self.assertTrue(d.exists())
            self.assertTrue(d.is_dir())

        for f in exist_files:
            self.assertTrue(f.exists())
            self.assertTrue(f.is_file())

        for d, s in [
            (dst_tests_test_middle_py, src_tests_test_middle_py),
            (dst_gitignore, src_gitignore),
            (dst_license, src_license),
            (dst_readme_md, src_readme_md),
            (dst_setup_cfg, src_setup_cfg),
            (dst_setup_py, src_setup_py),
        ]:
            with open(d, "r") as fp:
                d_content = fp.read()
            with open(s, "r") as fp:
                s_content = fp.read()
            self.assertEqual(s_content, d_content, f"{d} has other content then {s}")

        for d, s in [
            (dst_src_middle___init___py, src_src_middle___init___py),
        ]:
            with open(d, "r") as fp:
                d_content = fp.read()
            with open(s, "r") as fp:
                s_content = fp.read()
            self.assertNotEqual(
                s_content, d_content, f"{d} has the same content then {s}"
            )

        report = framework.sfl.tests4py_sfl_events(dst)
        if report.raised:
            raise report.raised

        report = framework.sfl.tests4py_sfl_analyze(dst, src, predicates="line")
        if report.raised:
            raise report.raised

        suggestions = report.analyzer.get_sorted_suggestions(
            base_dir=src,
            type_=AnalysisType.LINE,
            metric=Spectrum.Ochiai,
        )
        self.assertAlmostEqual(
            0.7071067811865475, suggestions[0].suspiciousness, delta=0.0000001
        )
        self.assertEqual(1, len(suggestions[0].lines))
        self.assertIn(
            Location(os.path.join("src", "middle", "__init__.py"), 6),
            suggestions[0].lines,
        )


from typing import Dict, Tuple
from copy import deepcopy
import torch
from config import tqc_config
from modules import Actor, TruncatedQuantileEnsembledCritic

class TQC:
    def __init__(self,
                 cfg: tqc_config,
                 actor: Actor,
                 critic: TruncatedQuantileEnsembledCritic) -> None:
        self.cfg = cfg
        self.device = cfg.device
        self.tau = cfg.tau
        self.discount = cfg.discount
        self.batch_size = cfg.batch_size

        self.target_entropy = -float(actor.action_dim)

        self.log_alpha = torch.tensor([0.0], dtype=torch.float32, device=self.device, requires_grad=True)

        self.alpha_optimizer = torch.optim.AdamW([self.log_alpha], lr=cfg.alpha_lr)
        self.alpha = self.log_alpha.exp().detach()

        self.actor = actor.to(self.device)
        self.actor_optim = torch.optim.AdamW(self.actor.parameters(), lr=cfg.actor_lr)

        self.critic = critic.to(self.device)
        self.critic_target = deepcopy(critic).to(self.device)
        self.critic_optim = torch.optim.AdamW(self.critic.parameters(), lr=cfg.critic_lr)

        self.quantiles_total = critic.num_critics * critic.num_quantiles
        self.quantiles2drop = cfg.quantiles_to_drop_per_critic * cfg.num_critics
        self.top = self.quantiles_total - self.quantiles2drop

        huber_tau = torch.arange(self.cfg.num_quantiles, device=self.device).float() / self.top + 1 / (2 * self.top)
        self.huber_tau = huber_tau[None, None, :, None]

        self.total_iterations = 0

    def train(self,
              states: torch.Tensor,
              actions: torch.Tensor,
              rewards: torch.Tensor,
              next_states: torch.Tensor,
              dones: torch.Tensor) -> Dict[str, float]:
        self.total_iterations += 1

        # critic step
        critic_loss = self.critic_loss(states, actions, rewards, next_states, dones)

        self.critic_optim.zero_grad()
        critic_loss.backward()
        self.critic_optim.step()

        # actor step
        actor_loss, batch_entropy, qz_values = self.actor_loss(states)

        self.actor_optim.zero_grad()
        actor_loss.backward()
        self.actor_optim.step()

        # alpha step
        alpha_loss = self.alpha_loss(states)

        self.alpha_optimizer.zero_grad()
        alpha_loss.backward()
        self.alpha_optimizer.step()

        self.alpha = self.log_alpha.exp().detach()

        self.soft_critic_update()

        return {
            "critic_loss": critic_loss.item(),
            "actor_loss": actor_loss.item(),
            "actor_batch_entropy": batch_entropy,
            "qz_values": qz_values,
            "alpha": self.alpha.item(),
            "alpha_loss": alpha_loss.item()
        }

    def actor_loss(self, states: torch.Tensor) -> Tuple[torch.Tensor, float, float]:
        actions, log_prob = self.actor(states, need_log_prob=True)

        qz_values = self.critic(states, actions).mean(dim=2).mean(dim=1, keepdim=True)

        loss = self.alpha * log_prob - qz_values
        batch_entropy = -log_prob.mean().item()

        return loss.mean(), batch_entropy, qz_values.mean().item()

    def critic_loss(self,
                    states: torch.Tensor,
                    actions: torch.Tensor,
                    rewards: torch.Tensor,
                    next_states: torch.Tensor,
                    dones: torch.Tensor) -> torch.Tensor:
        with torch.no_grad():
            next_actions, next_log_prob = self.actor(next_states, need_log_prob=True)

            next_z = self.critic_target(next_states, next_actions)

            sorted_next_z = torch.sort(next_z.reshape(self.batch_size, -1)).values
            sorted_next_z_top = sorted_next_z[:, :self.top]
            sorted_next_z_top = sorted_next_z_top - self.alpha * next_log_prob.unsqueeze(-1)

            quantiles_target = rewards + self.discount * (1.0 - dones) * sorted_next_z_top

        current_z = self.critic(states, actions)
        loss = self.quantile_huber_loss(current_z, quantiles_target)

        return loss

    def quantile_huber_loss(self, quantiles: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
        pairwise_diff = target[:, None, None, :] - quantiles[:, :, :, None]
        abs_val = pairwise_diff.abs()

        huber_loss = torch.where(abs_val > 1.0,
                                 abs_val - 0.5,
                                 pairwise_diff.pow(2) / 2)

        loss = torch.abs(self.huber_tau - (pairwise_diff < 0).float()) * huber_loss
        return loss.mean()

    def alpha_loss(self, state: torch.Tensor) -> torch.Tensor:
        with torch.no_grad():
            action, log_prob = self.actor(state, need_log_prob=True)

        loss = -self.log_alpha * (log_prob + self.target_entropy)
        return loss.mean()

    def soft_critic_update(self):
        for param, tgt_param in zip(self.critic.parameters(), self.critic_target.parameters()):
            tgt_param.data.copy_(self.tau * param.data + (1 - self.tau) * tgt_param.data)