"""
Transducer Training script
"""
#system utils
import sys
import argparse
import importlib
import os
import os.path
import math

import numpy as np
#torch related
import torch
import torch.optim as optim
from torch._six import inf

#supposed to be after kaldi import
from utils.logger import Logger
from utils.spec_augment import SpecAugment

from kaldi.matrix import _matrix_ext, DoubleMatrix
from kaldi.util import io
#from warprnnt_pytorch import RNNTLoss
from warp_rnnt import RNNTLoss

from trainer.bmuf import BmufTrainer

MASTER_NODE = 0

def run_one_epoch(epoch, model, log_f,
                  args, bmuf_trainer, training):
    """
    Run one epoch of training

    Args:
        epoch (int): zero based epoch index
        model (torch.nn.module): model
        log_f (file): logging file
        args : arguments from outer
        bmuf_trainer : initialized bmuf_trainer
        training (bool): training or validation
    """
    log_f.write('===> Epoch {} <===\n'.format(epoch))
    total_num_batches = args.num_epochs * args.num_batches_per_epoch
    num_batches_processed = epoch * args.num_batches_per_epoch
    lr = args.initial_lr * math.exp(num_batches_processed * \
                                    math.log(args.final_lr /\
                                             args.initial_lr) /\
                                    total_num_batches)
    log_f.write('===Using Learning Rate {}===\n'.format(lr))
    optimizer = optim.SGD(model.parameters(), lr,
                          momentum=args.momentum,
                          nesterov=True)

    loss_logger = Logger(args.log, args.log_per_n_frames, ['Loss'])
    transducer_loss = RNNTLoss(blank=0, reduction='sum').apply
    #transducer_loss = RNNTLoss(blank=0, reduction='sum')
    #for spec augment
    spec_augmentor = None
    if args.spec_augment:
        spec_augmentor = SpecAugment(args.max_freq_span, args.max_time_span)
    #for batchnorm and dropout
    if training:
        model.train()
    else:
        model.eval()
        optimizer = None

    for num_done, (data_cpu, target_cpu, len_cpu, ali_lens_cpu) in \
        enumerate(args.dataloader(args.data_lst, args.rir,
                                  args.noise, args)):
        if training:
            optimizer.zero_grad()

        if data_cpu is not None:
            #forward
            len_batch = len_cpu.cuda(args.local_rank)
            len_batch = len_batch - args.model_lctx - args.model_rctx
            len_batch = len_batch // args.model_stride + \
                        torch.ne((len_batch%args.model_stride), 0).int()
            ali_lens = ali_lens_cpu.cuda(args.local_rank)
            data_batch = data_cpu.cuda(args.local_rank)
            target_batch = target_cpu.long().cuda(args.local_rank)
            if args.cmvn_stats:
                #cmn#
                if args.cmn:
                    data_batch -= data_batch.mean(dim=1).unsqueeze(dim=1)
                data_batch += args.offset.unsqueeze(dim=0).unsqueeze(dim=0).float()
                data_batch *= args.scale.unsqueeze(dim=0).unsqueeze(dim=0).float()
            if spec_augmentor is not None:
                spec_augmentor.apply(data_batch)

            outputs = model.forward(data_batch, target_batch, len_batch, True)

            loss = transducer_loss(outputs, target_batch.int(),
                                   len_batch, ali_lens)
            loss = loss.sum()
        else: #empty batch
            loss = torch.FloatTensor([0.0])

        if training:
            if data_cpu is not None:
                loss.backward()
                if args.grad_clip > 0:
                    torch.nn.utils.clip_grad_norm_(model.parameters(),
                                                   args.grad_clip,
                                                   norm_type=inf)
                optimizer.step()

            if num_done != 0 and num_done % args.sync_period == 0:
                if not bmuf_trainer.update_and_sync():
                    return float('nan')
                num_batches_processed = (epoch * args.num_batches_per_epoch \
                                         + num_done)
                lr = args.initial_lr * math.exp(num_batches_processed * \
                                                math.log(args.final_lr /\
                                                         args.initial_lr) /\
                                                total_num_batches)
                optimizer = optim.SGD(model.parameters(), lr,
                                      momentum=args.momentum,
                                      nesterov=True)
        #destruct all cpu torch tensor
        #manully to prevent leaks if they
        #are created in different process
        del data_cpu, target_cpu, len_cpu, ali_lens_cpu

        labels = ali_lens.sum().item()
        loss_logger.update_and_log(labels, [loss.item()])

    if training:
        if not bmuf_trainer.update_and_sync():
            return float('nan')
        tot_loss, tot_num = loss_logger.summarize_and_log()
    else:
        tot_loss, tot_num = loss_logger.summarize_and_log()

    #aggregate across workers
    loss_tensor = torch.FloatTensor([tot_loss, float(tot_num)])
    loss_tensor = loss_tensor.cuda(args.local_rank)
    bmuf_trainer.sum_reduce(loss_tensor)
    bmuf_trainer.broadcast(loss_tensor)
    reduced_loss = loss_tensor[0] / loss_tensor[1]
    return reduced_loss.item()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Transducer training')

    parser.add_argument('nnet_proto', type=str,
                        help='pytorch NN proto definition filename')
    parser.add_argument('data_lst', type=str,
                        help='list of mrk, seq, ali files for data')
    parser.add_argument('log', type=str,
                        help='log file for the job')
    parser.add_argument('output_dir', type=str,
                        help='path to save the final model')
    parser.add_argument('--init_model', type=str, default=None,
                        help='initial model')

    #rir and noise
    parser.add_argument('--rir_lst', type=str, default=None,
                        help='mrk and seq files for rir')
    parser.add_argument('--noise_lst', type=str, default=None,
                        help='mrk and seq files for noise')

    #Model Options
    parser.add_argument('--encoder_type', type=str, default='rnn',
                        choices=['rnn', 'transformer'],
                        help="""Type of encoder layer to use.""")
    parser.add_argument('--decoder_type', type=str, default='rnn',
                        choices=['rnn', 'transformer'],
                        help='Type of decoder layer to use.')

    parser.add_argument('--layers', type=int, default=-1,
                        help='Number of layers in enc/dec.')
    parser.add_argument('--enc_layers', type=int, default=2,
                        help='Number of layers in the encoder')
    parser.add_argument('--dec_layers', type=int, default=2,
                        help='Number of layers in the decoder')

    parser.add_argument('--rnn_size', type=int, default=512,
                        help='Size of LSTM hidden states')

    parser.add_argument('--rnn_type', type=str, default='LSTM',
                        choices=['LSTM'],
                        help="""rnn type""")

    parser.add_argument('--embd_dim', type=int, default=300,
                        help='embeddings dimension for decoder')
    parser.add_argument('--output_dim', type=int, default=8000,
                        help='output dimension of neural network')

    parser.add_argument('--model_lctx', type=int, default=0,
                        help='model left context')
    parser.add_argument('--model_rctx', type=int, default=0,
                        help='model right context')
    parser.add_argument('--model_stride', type=int, default=1,
                        help='model stride, ie., subsampling in the model')

    parser.add_argument('--brnn', action="store_true",
                        help="valid only when encoder type is rnn")
    parser.add_argument('--cmn', action="store_true",
                        help="apply cepstrum mean normalizaiton per utterance")
    parser.add_argument('--cmvn_stats', type=str, default=None,
                        help='cmvn_stats file')

    parser.add_argument('--optim', type=str, default='sgd',
                        choices=['sgd', 'adam', 'adadelta'],
                        help="""optimizer to use """)

    parser.add_argument('--grad_clip', type=float, default=-1.0,
                        help='gradient clipping threshold')

    parser.add_argument('--initial_lr', type=float, default=1.0,
                        help='initial learning rate')
    parser.add_argument('--final_lr', type=float, default=1.0,
                        help='initial learning rate')
    parser.add_argument('--momentum', type=float, default=0.9,
                        help='momentum for SGD')
    parser.add_argument('--num_epochs', type=int, default=15,
                        help='number of epochs for training')
    parser.add_argument('--num_batches_per_epoch', type=int, default=1000,
                        help='number of batches per work per epoch')
    parser.add_argument('--dropout', type=float, default=0.3,
                        help="Dropout probability; applied in LSTM stacks.")
    parser.add_argument('--padding_idx', type=int, default=-1,
                        help='padding index for targets')
    parser.add_argument('--loader', choices=['otf_utt'],
                        default='otf_utt',
                        help='loaders')
    parser.add_argument('--log_per_n_frames', type=int, default=1024*1024,
                        help='logging per n frames')
    parser.add_argument('--seed', type=int, default=777,
                        help='random seed')
    parser.add_argument('--cuda', action='store_true',
                        help='use CUDA')

    #BMUF related
    parser.add_argument('--local_rank', type=int,
                        help='local process ID for parallel training')
    parser.add_argument('--block_momentum', type=float, default=0.9,
                        help='block momentum for BMUF')
    parser.add_argument('--block_lr', type=float, default=1.0)
    parser.add_argument('--sync_period', type=int, default=100)
    #Spec Augment related
    parser.add_argument('--spec_augment', action='store_true',
                        help='enable spec augmentation')
    parser.add_argument('--max_freq_span', type=int, default=15,
                        help='max frequency span to dropout input')
    parser.add_argument('--max_time_span', type=int, default=35,
                        help='max time span to dropout input')

    args, unk = parser.parse_known_args()

    # import loader
    loader_module = importlib.import_module('loader.' + args.loader \
                                             + '_loader')
    loader_module.register(parser)
    args = parser.parse_args()
    args.input_dim = loader_module.get_inputdim(args)
    args.dataloader = loader_module.dataloader

    # number of workers
    world_size = int(os.environ['WORLD_SIZE'])
    # set cuda device
    assert args.cuda
    assert torch.cuda.is_available()
    torch.cuda.set_device(args.local_rank)

    #this is just an example for those who
    #want to do on-the-fly noise and reverberation
    #both rir and noise are list of AudioSegments
    #see trainer/loader/audio.py for more details
    rir = []
    #noise adding augmentor has constrain
    #that noise lengths need to be longer
    #than utterance, assuming 100 fps
    #min_noise_len = args.max_len / 100 * args.sample_rate
    noise = []
    args.rir, args.noise = rir, noise

    #modify data and label rspec according to local_rank
    args.data_lst = args.data_lst.replace('WORKER-ID', str(args.local_rank))

    #modify log file according to local_rank
    args.log = args.log.replace('WORKER-ID', str(args.local_rank))
    log_f = open(args.log, 'w')
    args.log = log_f

    #manual seed for reproducibility
    nnet_module = importlib.import_module("model."+args.nnet_proto)
    frame_ctx = args.lctx + 1 + args.rctx
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    if args.init_model is None:
        model = nnet_module.Net(args, args.input_dim,
                                args.output_dim)

    else:
        model = torch.load(args.init_model,
                           map_location=lambda storage, loc: storage)

    model.cuda(args.local_rank)

    #initialize BMUF trainer
    bmuf_trainer = BmufTrainer(MASTER_NODE, args.local_rank, world_size,
                               model, args.block_momentum, args.block_lr)

    num_param = 0
    for param in model.parameters():
        num_param += param.numel()
    #print model proto
    log_f.write('*'*60+'\n')
    log_f.write('*'*60+'\n')
    log_f.write('*'*60+'\n')
    log_f.write('model proto: {}\n'
                'input  dim: {},\toutput dim: {},\n'
                'hidden dim: {},\tnum of enc_layers: {}\n'
                'num of dec_layers: {},\trnn_type: {}\n'
                'model size: {} M\n'
                .format(args.nnet_proto, args.input_dim,
                        args.output_dim, args.rnn_size,
                        args.enc_layers, args.dec_layers,
                        args.rnn_type, num_param/1000/1000))
    log_f.write('*'*60+'\n')
    log_f.write('*'*60+'\n')
    log_f.write('*'*60+'\n')
    log_f.flush()

    if torch.cuda.is_available():
        if args.cuda:
            torch.cuda.manual_seed(args.seed)
            model.cuda()
        else:
            print('WARNING: You have a CUDA device, '
                  'so you should probably run with --cuda')
    #load cmvn states
    if args.cmvn_stats:
        with io.Input(args.cmvn_stats, binary=False) as ki:
            stats = DoubleMatrix().read_(ki.stream(), ki.binary)
            cmvn = _matrix_ext.double_matrix_to_numpy(stats)
            mean = cmvn[0][:-1] / cmvn[0][-1]
            var = cmvn[1][:-1] / cmvn[0][-1] - (mean*mean)
            floor = 1.0e-20
            if min(abs(var)) < floor:
                print('problematic cmvn_stats, variance too small')
                sys.exit()
            args.offset = torch.from_numpy(-mean).cuda(args.local_rank)
            args.offset = args.offset.repeat(args.lctx + args.rctx + 1)
            args.scale = torch.from_numpy(1.0 / np.sqrt(var))
            args.scale = args.scale.cuda(args.local_rank)
            args.scale = args.scale.repeat(args.lctx + args.rctx + 1)

    #start training with run_one_epoch, params:
    #(model, args, bmuf_trainer, train or valid)
    for epoch in range(0, args.num_epochs):
        train_loss = run_one_epoch(epoch, model, log_f,
                                   args, bmuf_trainer, True)
        #save current model
        current_model = '{}/model.epoch.{}.{}'.format(args.output_dir,
                                                      epoch, args.local_rank)
        with open(current_model, 'wb') as f:
            torch.save(model, f)

    log_f.write('Training Finished')


import pytest

from django.test import TestCase

from typing import Dict

from agile.models import Agile
from agile.serializers import AgileSerializer

@pytest.mark.django_db
class TestAgileSerializer:
    @pytest.fixture
    def agile_fixture(self, db) -> Agile:
        return Agile.objects.create(name="name", description="description")

    @pytest.fixture
    def default_value(self, agile_fixture) -> Dict[str, Dict[str, str]]:
        return {
            "page": agile_fixture,
            "data": {
            "name": "Responding to change over following a plan.",
            "description": "Circumstances change and sometimes customers demand extra.",
        },
        }

    def test_valid(self, default_value):
        serializer = AgileSerializer(data=default_value["data"])
        assert serializer.is_valid()

    def test_save(self, default_value):
        serializer = AgileSerializer(data=default_value["data"])
        serializer.is_valid()
        instance = serializer.save()
        assert instance.name == default_value["data"]["name"]
        assert instance.description == default_value["data"]["description"]


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@文件    :TCGA_dataTomatrix.py
@说明    :
@时间    :2020/12/09 20:29:34
@作者    :陆柏成
@版本    :1.0
@Email   :lu_baicheng@163.com
'''

import os
import sys
from collections import OrderedDict
import json
import re
import glob
import gzip
import time
import shutil
from multiprocessing import Pool
os.chdir(r'E:\实验室\TCGA\TCGA数据\PRAD')
import asyncio
def TCGA_data_transformer(json_file):
    gene_group = OrderedDict()
    tcga_file = OrderedDict()
    list_1 = []
    normal = []
    tumor = []
    with open(json_file, "r") as f1:
        data = json.load(f1)
        for i in data:
            file_name = i["file_name"]
            file_name = re.sub("\.gz", "", file_name)
            tcga_id = i["associated_entities"][0]["entity_submitter_id"]
            tcga_file[tcga_id] = file_name

            ID = tcga_id.split("-")
            if ID[3] == "11A":
                normal.append(tcga_id)
            else:
                tumor.append(tcga_id)

    list_1.extend(normal)
    list_1.extend(tumor)
    return list_1,tcga_file

def un_gz(file_name):
    # 获取文件的名称，去掉后缀名
    f_name = file_name.replace(".gz", "")
    # 开始解压
    g_file = gzip.GzipFile(file_name)
    # 读取解压后的文件，并写入去掉后缀名的同名文件（即得到解压后的文件）
    open(f_name, "wb+").write(g_file.read())
    g_file.close()

def get_count_file():
    for root, dirs, files in os.walk('./'):
        for item in files:
            if re.search('gz',item):
                # i +=1
                # print(item,i)
                file_Dir=os.path.join(root,item)

                un_gz(file_Dir)
                file_read=re.sub('.gz','',file_Dir)
                shutil.move(file_read,'./')

def readfile(file):
    file_obj = open(file)
    try:
        while True:
            line = file_obj.readline().strip("\n")
            if not line:
                break
            arry = line.split("\t")
            if arry[0] in tcga_matrix:
                tcga_matrix[arry[0]].append(arry[1])
            else:
                tcga_matrix[arry[0]] = [arry[1]]
    finally:
        file_obj.close()

if __name__ == "__main__":
    tcga_matrix = OrderedDict()
    s1=time.time()
    case_list,tcga_dic=TCGA_data_transformer(r'E:\实验室\TCGA\TCGA数据\metadata.cart.2020-12-09.json')
    get_count_file()

    for i in case_list:
        readfile(tcga_dic[i])
    with open('tgca_matrix.txt','w') as f_w:
        f_w.write('id'+'\t'+'\t'.join(case_list[0:len(case_list)])+'\n')
        for k,v in tcga_matrix.items():
            f_w.write(k+'\t'+'\t'.join(v[0:len(v)])+'\n')
        f_w.close()

    s2=time.time()
    print(s2-s1)


import random
from collections import deque
from random import shuffle
from typing import Any, Deque, List, Optional

import pyperclip

class TreeNode:

    @staticmethod
    def from_structure(list: Optional[List[Any]]):
        if not list:
            return None

        val_queue = deque(list)
        list = None

        root = TreeNode(val_queue.popleft())
        node_queue = deque((root,))
        while val_queue:
            node = node_queue.popleft()

            if not val_queue:
                break

            left_val = val_queue.popleft()
            if left_val != None:
                node.left = TreeNode(left_val)
                node_queue.append(node.left)

            if not val_queue:
                break

            right_val = val_queue.popleft()
            if right_val != None:
                node.right = TreeNode(right_val)
                node_queue.append(node.right)

        return root

    @staticmethod
    def fromList(list: Optional[List[Any]]):
        if not list:
            return None

        res = TreeNode(list[0])
        for val in list[1:]:
            res.add_node(val)
        return res

    def __init__(self, val: Any = 0, left=None, right=None):
        self.val: int = val
        self.left: Optional[TreeNode] = left
        self.right: Optional[TreeNode] = right

    def __repr__(self):
        l = []
        node_stack: Deque[Optional[TreeNode]] = deque([self])
        while node_stack:
            node = node_stack.popleft()
            if not node:
                l.append(None)
                continue

            l.append(node.val)
            node_stack.append(node.left)
            node_stack.append(node.right)

        i = 0   # to ignore unbound warning
        for i in range(len(l)-1, -1, -1):
            if l[i] != None:
                break
        return str(l[:i+1]).replace("None", "null")

    def __bool__(self):
        return True

    def pystr(self):
        return str(self).replace("null", "None")

    def in_order_str(self) -> str:
        def helper(root):
            nonlocal sofar
            if not root:
                return
            helper(root.left)
            sofar.append(root.val)
            helper(root.right)

        sofar = []
        helper(self)
        return str(sofar)

    def pre_order_str(self) -> str:
        def helper(root):
            nonlocal sofar
            if not root:
                return
            sofar.append(root.val)
            helper(root.left)
            helper(root.right)

        sofar = []
        helper(self)
        return str(sofar)

    def add_node(self, val: Any):
        if val < self.val:
            if self.left:
                self.left.add_node(val)
            else:
                self.left = TreeNode(val)
        else:
            if self.right:
                self.right.add_node(val)
            else:
                self.right = TreeNode(val)

if __name__ == "__main__":
    null = None
    random.seed(2)

    l = list(range(1, 1000))
    random.shuffle(l)
    x = TreeNode.fromList(l)

    # def zfill(root):
    #     if not root:
    #         return
    #     root.val = 0
    #     zfill(root.left)
    #     zfill(root.right)
    # zfill(x)
    # s = str(x).replace("None", "null")
    # pyperclip.copy(s)


# Learning
"""
>> Inversion : Formally speaking, two elements a[i] and a[j] form an inversion if a[i] > a[j] and i < j
"""

class Solution:
    """
    Problem Statement : https://www.geeksforgeeks.org/counting-inversions/
    We Will be Solving this problem using `Enhanced Merge sort `
    Enhanced Merge sort Algorithm:
    >> The idea is similar to merge sort, divide the array into two equal or almost equal halves in each step until the base case is reached.
    >> Create a function merge that counts the number of inversions when two halves of the array are merged,
        create two indices i and j, i is the index for the first half, and j is an index of the second half.
        if a[i] is greater than a[j], then there are (mid – i) inversions. because left and right subarrays are sorted,
        so all the remaining elements in left-subarray (a[i+1], a[i+2] … a[mid]) will be greater than a[j].
    >> Create a recursive function to divide the array into halves and find the answer by summing the number of inversions is the first half,
        the number of inversion in the second half and the number of inversions by merging the two.
    >> The base case of recursion is when there is only one element in the given half.
    >> Print the answer

    Time Complexity: O(n log n), The algorithm used is divide and conquer,
    So in each level, one full array traversal is needed, and there are log n levels, so the time complexity is O(n log n).
    Space Complexity: O(n), Temporary array.
    """

    def merge(self, arr, temp_arr, left, mid, right):
        i = left  # Starting index of left subarray
        j = mid + 1  # Starting index of right subarray
        k = left  # Starting index of to be sorted subarray
        inv_count = 0
        # Conditions are checked to make sure that i and j don't exceed their subarray limits.
        while i <= mid and j <= right:
            # There will be no inversion if arr[i] <= arr[j]
            if arr[i] <= arr[j]:
                temp_arr[k] = arr[i]
                k += 1
                i += 1
            else:
                # Inversion will occur
                temp_arr[k] = arr[j]
                inv_count += (mid - i + 1)
                k += 1
                j += 1
        # Copy the remaining elements of left subarray into temporary array
        while i <= mid:
            temp_arr[k] = arr[i]
            k += 1
            i += 1
        # Copy the remaining elements of right subarray into temporary array
        while j <= right:
            temp_arr[k] = arr[j]
            k += 1
            j += 1

        # Copy the sorted subarray into Original array
        for loop_var in range(left, right + 1):
            arr[loop_var] = temp_arr[loop_var]

        return inv_count

    def _mergeSort(self, arr, temp_arr, left, right):
        # A variable inv_count is used to store inversion counts in each recursive call
        inv_count = 0
        # we will make a recursive call if and only if we have more than one elements

        if left < right:
            # mid is calculated to divide the array into two subaaray
            # Floor division is must in case of python
            mid = (left + right) // 2
            # It will calculate inversion counts in the left subarray
            inv_count += self._mergeSort(arr, temp_arr, left, mid)

            # It will calculate inversion counts in right subarray
            inv_count += self._mergeSort(arr, temp_arr, mid + 1, right)
            # It will merge 2 subarray in a sorted array
            inv_count += self.merge(arr, temp_arr, left, mid, right)
        return inv_count

    def mergeSort(self, arr):
        # A temp_arr is created to store sorted array in merge function
        temp_arr = [0] * len(arr)
        return self._mergeSort(arr, temp_arr, 0, len(arr) - 1)

if __name__ == "__main__":
    arr = [1, 20, 6, 4, 5]
    result = Solution().mergeSort(arr)
    print("Number of inversions are", result)


# 819. Most Common Word
import collections, re
class Solution:
    def mostCommonWord(self, paragraph: str, banned: List[str]) -> str:
        punct = set('!?\',;.')
        clean_parag = []
        for c in paragraph.lower():
            if c not in punct: clean_parag.append(c)
            else: clean_parag.append(' ')
        clean_parag = ''.join(clean_parag).split()
        for word, cnt in sorted(collections.Counter(clean_parag).items(), key=lambda item: item[1], reverse=True):
            if word not in banned:
                return word

    def mostCommonWord(self, paragraph: str, banned: List[str]) -> str:
        ban = set(banned)
        words = re.findall(r'\w+', paragraph.lower())
        return collections.Counter(w for w in words if w not in banned).most_common(1)[0][0]

leetcode/0-250/348-819. Most Common Word.py


# import required libraries
from module import XCentralizer, YCentralizer, YScaler, GestureTransformer
from keras.models import load_model
from datetime import datetime
import asyncio
import websockets
import json
import time
import timeit
import pandas as pd
import numpy as np
import h5py
import threading
import tello
from sklearn.pipeline import make_pipeline, make_union
import math
import warnings
from scipy.interpolate import interp1d
from os import listdir
import re
from sklearn.externals import joblib

warnings.filterwarnings("ignore")

# set static parameters
show_current_framelength = False
ms_per_frame_original = 120
gesture_length = 2000
ms_per_frame_interpolated = 50
add_interpol_frames = 3
allow_model_selection = False
debug_mode = False

# get model type
if allow_model_selection:
    print('')
    print("Which model type do you want to use? 1 = delta, 2 = posture, 3 = gesture")
    model_type_id = input()
    if int(model_type_id) == 1:
        model_type = 'delta'
    if int(model_type_id) == 2:
        model_type = 'posture'
    if int(model_type_id) == 3:
        model_type = 'gesture'
    print('The "' + model_type + '" model will be used.')
else:
    model_type_id = 3
    model_type = 'gesture'

# get interpolation yes / no
if allow_model_selection:
    if int(model_type_id) == 3:
        print('')
        print("Do you want to use interpolation? y = yes, n = no")
        use_interpolation_id = input()
        if str(use_interpolation_id)=='y':
            use_interpolation = True
            print('The PoseNet wireframes will be interpolated.')
            interpolation = 'ip'
        else:
            use_interpolation = False
            print('The PoseNet wireframes will not be interpolated.')
            interpolation = 'nip'
    print('')
else:
    use_interpolation = False
    interpolation = 'nip'

# get model instance
if allow_model_selection:
    if int(model_type_id) == 3:
        file_names = listdir('../models/alternative_models')

        grp_model_type = '(?P<model_type>model_'+ model_type + ')'
        grp_estimator = '(?P<estimator>[^_]+)'
        grp_ip = '(?P<ip_nip>' + interpolation + '\d*)'
        grp_model_params = '(?P<model_parameters>.+)'
        grp_suffix = '(?P<suffix>pkl|h5)'
        pattern = '(?P<file_name>' + grp_model_type + '_' + grp_estimator + '_' + grp_ip + '_' + grp_model_params + '.' + grp_suffix + ')'
        reg = re.compile(pattern)

        matches = []
        for file_name in file_names:
            match = reg.search(file_name)
            if match:
                matches.append(match)

        groups = []
        for i, match in enumerate(matches):
            group = match.groupdict()
            groups.append(group)

        models = []
        for grp in groups:
            models.append(grp["file_name"])

        estimators = []
        for grp in groups:
            estimators.append(grp["estimator"])

        print('Which model instance do you want to use?')
        for i in range(len(models)):
            print(str(i) + ' | ' + models[i])
        model_instance = input()
        print('')

        estimator = estimators[int(model_instance)]
        model_file_name = 'alternative_models/' + models[int(model_instance)]
else:
    estimator = 'lstm'
    model_file_name = 'model_gesture.h5'

# decide if drone or virtual flight is used
print("Do you connect to the drone or do a virtual flight? d = drone, v = virtual flight")
virtual_flight_id = input()
if str(virtual_flight_id)=='d':
    virtual_flight = False
    print('Connecting to drone...')
else:
    virtual_flight = True
    print('You will see the flight commands printed on the screen.')
print('')

# connect to drone and set status flags
drone_last_action = time.time()
drone_status = 'grounded'
if not virtual_flight:
    drone = tello.Tello('192.168.10.2', 8888)

movements = {0: 'not detected',
             1: 'takeoff',
             2: 'move',
             3: 'flip',
             4: 'left',
             5: 'right',
             6: 'land'}

# load model and initiate pipeline
if (model_type == 'posture'):
    model = load_model('../models/model_' + model_type + '.h5')
    cols_x = ['leftShoulder_x',
              'rightShoulder_x',
              'leftWrist_x',
              'rightWrist_x',
              'leftHip_x',
              'rightHip_x']
    cols_y = [col.replace('x', 'y') for col in cols_x]
    processing_pipeline = make_pipeline(XCentralizer(cols_x),
                                        YCentralizer(cols_y),
                                        YScaler())

if (model_type == 'gesture'):
    cols_x = ['leftShoulder_x',
              'rightShoulder_x',
              'leftWrist_x',
              'rightWrist_x',
              'leftHip_x',
              'rightHip_x',
              'leftElbow_x',
              'rightElbow_x']
    cols_y = [col.replace('x', 'y') for col in cols_x]

    if model_file_name.endswith('pkl'):
        model = joblib.load('../models/' + model_file_name)
    elif model_file_name.endswith('h5'):
        model = load_model('../models/' + model_file_name)

    if use_interpolation:
        start_time = timeit.default_timer()
        ms_since_start = timeit.default_timer()
        cols = sorted(cols_x + cols_y + ['ms_since_start'])
        cols_without_ms = sorted(cols_x + cols_y)
        processing_pipeline = make_pipeline(GestureTransformer(byrow=True,feature_names=cols_without_ms))
    else:
        cols = sorted(cols_x + cols_y)
        processing_pipeline = make_pipeline(GestureTransformer(byrow=True,feature_names=cols))
    pose_df = pd.DataFrame(columns=cols)

async def consumer_handler(websocket, path):

    global model_type
    print(model_type)

    print('Accepting incoming snapshots. Waiting for take-off signal.')
    try:
        async for pose_json in websocket:
            if show_current_framelength:
                if 'start' in locals():
                    start = stop
                else:
                    start = timeit.default_timer()
                stop = timeit.default_timer()
                ms_since_last_frame = str(round(1000*(stop-start)))
                print('Snapshot captured in ' + ms_since_last_frame + 'ms')
            pose_dict = json_to_dict(pose_json)
            if len(pose_dict) == 0:
                print('No wireframes detected. Please ensure that PoseNet detects wireframes.')
            else:
                if model_type == 'delta':
                    steer_drone(predict_movement_delta(pose_dict))
                if model_type == 'posture':
                    steer_drone(predict_movement_model_posture(pose_dict))
                if model_type == 'gesture':
                    steer_drone(predict_movement_model_gesture(pose_dict))
    except:
        print('Websocket connection terminated. Please re-connect.')
        raise

def steer_drone(movement):
    global drone_last_action
    global drone_status

    if ((movement == 1) & (drone_status == 'grounded')):
        threading.Thread(target=drone_takeoff).start()
        drone_last_action = time.time()
        time.sleep(4)
    if ((drone_status != 'grounded') & ((time.time() - drone_last_action) > 1.7)):
        drone_last_action = time.time()
        if movement == 2:
            threading.Thread(target=drone_move).start()
        if movement == 3:
            threading.Thread(target=drone_flip).start()
        if movement == 4:
            threading.Thread(target=drone_left).start()
        if movement == 5:
            threading.Thread(target=drone_right).start()
        if movement == 6:
            threading.Thread(target=drone_land).start()

def drone_takeoff():
    global drone_status
    drone_status = 'flying'
    show_movement(1)
    print('Ready to take flight commands in five seconds.')
    if not virtual_flight:
        drone.takeoff()

def drone_move():
    show_movement(2)
    if not virtual_flight:
        drone.move_forward(2)

def drone_flip():
    show_movement(3)
    if not virtual_flight:
        drone.flip('r')

def drone_left():
    show_movement(4)
    if not virtual_flight:
        drone.rotate_ccw(70)

def drone_right():
    show_movement(5)
    if not virtual_flight:
        drone.rotate_cw(70)

def drone_land():
    global drone_status
    drone_status = 'grounded'
    show_movement(6)
    if not virtual_flight:
        drone.land()

def show_movement(movement):

    if movement == 1:  # take_off
        print('                                                  ')
        print('                        XX                        ')
        print('                      XXXXXX                      ')
        print('                    XXXXXXXXXX                    ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                XXXXXXXXXXXXXXXXXX                ')
        print('              XXXXXXXXXXXXXXXXXXXXXX              ')
        print('            XXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX          ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                  XXXXXXXXXXXXXX                  ')
        print('                                                  ')
        print('--------------------------------------------------')
    if movement == 2:  # move
        print('                                                  ')
        print('           X             X                        ')
        print('           XXX           XXX                      ')
        print('           XXXXX         XXXXX                    ')
        print('           XXXXXXX       XXXXXXX                  ')
        print('           XXXXXXXXX     XXXXXXXXX                ')
        print('           XXXXXXXXXXX   XXXXXXXXXXX              ')
        print('           XXXXXXXXXXXXX XXXXXXXXXXXXX            ')
        print('           XXXXXXXXXXX   XXXXXXXXXXX              ')
        print('           XXXXXXXXX     XXXXXXXXX                ')
        print('           XXXXXXX       XXXXXXX                  ')
        print('           XXXXX         XXXXX                    ')
        print('           XXX           XXX                      ')
        print('           X             X                        ')
        print('                                                  ')
        print('--------------------------------------------------')
    if movement == 3:  # flip
        print('                                                  ')
        print('                 XXXXX     XXXXX                  ')
        print('              XXXXXX         XXXXXX               ')
        print('           XXXXX                 XXXXX            ')
        print('         XXXXX                     XXXXX          ')
        print('        XXXXX                       XXXXX         ')
        print('        XXXX                         XXXX         ')
        print('       XXXX                           XXXX        ')
        print('        XXXX                         XXXX         ')
        print('        XXXXX                       XXXXX         ')
        print('         XXXXX                     XXXXX          ')
        print('           XXXXX                 XXXXX            ')
        print('              XXXXXX         XXXXXX               ')
        print('                 XXXXX     XXXXX                  ')
        print('                                                  ')
        print('--------------------------------------------------')
    if movement == 4:  # left
        print('                                                  ')
        print('                      XXXX                        ')
        print('                    XXXX                          ')
        print('                  XXXX                            ')
        print('                XXXX                              ')
        print('              XXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('            XXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('            XXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('              XXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('                XXXX                              ')
        print('                  XXXX                            ')
        print('                    XXXX                          ')
        print('                      XXXX                        ')
        print('                                                  ')
        print('--------------------------------------------------')
    if movement == 5:  # right
        print('                                                  ')
        print('                      XXXX                        ')
        print('                        XXXX                      ')
        print('                          XXXX                    ')
        print('                            XXXX                  ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXX                ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXX              ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXX              ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXX                ')
        print('                            XXXX                  ')
        print('                          XXXX                    ')
        print('                        XXXX                      ')
        print('                      XXXX                        ')
        print('                                                  ')
        print('--------------------------------------------------')
    if movement == 6:  # land
        print('                                                  ')
        print('                                                  ')
        print('                   XXXXXXXXXX                     ')
        print('                XXXXXXXXXXXXXXXX                  ')
        print('             XXXXXXXXXXXXXXXXXXXXXX               ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('          XXXXXXXXXXXXXXXXXXXXXXXXXXXX            ')
        print('             XXXXXXXXXXXXXXXXXXXXXX               ')
        print('                XXXXXXXXXXXXXXXX                  ')
        print('                   XXXXXXXXXX                     ')
        print('                                                  ')
        print('                                                  ')
        print('--------------------------------------------------')

def json_to_dict(pose_json):

    x = json.loads(pose_json)
    pose_dict = {}

    if len(x['poses']) > 0:
        for i in range(8):
            pose_dict[x['poses'][0]['keypoints'][i+5]['part'] +
                    '_x'] = x['poses'][0]['keypoints'][i + 5]['position']['x']
            pose_dict[x['poses'][0]['keypoints'][i+5]['part'] +
                    '_y'] = x['poses'][0]['keypoints'][i + 5]['position']['y']

        if model_type != 'gesture':
            del pose_dict['leftElbow_x']
            del pose_dict['leftElbow_y']
            del pose_dict['rightElbow_x']
            del pose_dict['rightElbow_y']

    return pose_dict

def predict_movement_delta(pose_dict):

    movement = 0  # no movement detected

    leftArm_x = pose_dict['leftWrist_x'] - pose_dict['leftShoulder_x']
    rightArm_x = pose_dict['rightShoulder_x'] - pose_dict['rightWrist_x']
    leftArm_y = pose_dict['leftShoulder_y'] - pose_dict['leftWrist_y']
    rightArm_y = pose_dict['rightShoulder_y'] - pose_dict['rightWrist_y']

    vertical_threshold = 50
    horizontal_threshold = 40
    undetected_threshold = 30

    if ((leftArm_y > vertical_threshold) & (rightArm_y > vertical_threshold) & (abs(leftArm_x) < undetected_threshold) & (abs(rightArm_x) < undetected_threshold)):
        movement = 1  # takeoff

    if ((abs(leftArm_y) < undetected_threshold) & (abs(rightArm_y) < undetected_threshold) & (leftArm_x > horizontal_threshold) & (rightArm_x > horizontal_threshold)):
        movement = 2  # move

    if ((abs(leftArm_x) < undetected_threshold) & (abs(rightArm_x) < undetected_threshold) & (abs(leftArm_y) < undetected_threshold) & (abs(rightArm_y) < undetected_threshold)):
        movement = 3  # flip

    if ((leftArm_y < -vertical_threshold) & (abs(rightArm_y) < undetected_threshold) & (abs(leftArm_x) < undetected_threshold) & (rightArm_x > horizontal_threshold)):
        movement = 4  # left

    if ((abs(leftArm_y) < undetected_threshold) & (rightArm_y < -vertical_threshold) & (leftArm_x > horizontal_threshold) & (abs(rightArm_x) < undetected_threshold)):
        movement = 5  # right

    if ((leftArm_y < -vertical_threshold) & (rightArm_y < -vertical_threshold) & (abs(leftArm_x) < undetected_threshold) & (abs(rightArm_x) < undetected_threshold)):
        movement = 6  # land

    return movement

def predict_movement_model_posture(pose_dict):

    pose_df = pd.DataFrame(pose_dict, index=[0])
    processing_pipeline.fit_transform(pose_df)
    movement = np.argmax(model.predict(pose_df)[0])

    return movement

def predict_movement_model_gesture(pose_dict):

    global pose_df
    global start_time
    global ms_since_start
    global estimator
    global debug_mode
    global use_interpolation
    movement = 0

    steps = math.ceil(gesture_length/ms_per_frame_original) + 1
    steps_ip = math.ceil(gesture_length/ms_per_frame_interpolated) + 1

    if use_interpolation:
        pose_new = pd.DataFrame(pose_dict, index=[0])
        pose_new.at[0,'ms_since_start'] = round(1000*(timeit.default_timer()-start_time))
        pose_df = pose_df.append(pose_new)
        pose_df.reset_index(inplace=True, drop=True)

        # keep only latest two seconds and the additional frames needed for interpolation
        cut_off = pose_df.iloc[pose_df['ms_since_start'].argmax()]['ms_since_start'] - 2000
        keep_df = pose_df.loc[pose_df['ms_since_start']>=cut_off]
        discarded_df = pose_df.loc[pose_df['ms_since_start']<cut_off]
        pose_df = pd.concat([discarded_df.tail(add_interpol_frames), keep_df]).reset_index(drop=True)

        if discarded_df.shape[0] > add_interpol_frames:
            pose_ip_df = interpolate(pose_df, ms_per_frame_interpolated)
            if debug_mode:
                file_name = 'model_input_' + datetime.now().strftime('%Y%m%d_%H%M%S%f') + '.csv'
                pose_ip_df.to_csv('model_inputs/' + file_name,  index=False)
            if model_file_name.endswith('pkl'):
                pose_np = pose_ip_df.drop(['ms_since_start'], axis=1).values.reshape(steps_ip, -1)
            elif model_file_name.endswith('h5'):
                pose_np = pose_ip_df.drop(['ms_since_start'], axis=1).values.reshape(1, steps_ip, -1)
            pose_np = processing_pipeline.fit_transform(pose_np)
            if estimator in ['svm', 'rf']:
                pose_np = pose_np.reshape(1, -1)
            if estimator == 'svm':
                movement = model.predict(pose_np)[0]
            else:
                movement = np.argmax(model.predict(pose_np)[0])
    else:
        pose_df = pose_df.append(pd.DataFrame(pose_dict, index=[0]))
        if len(pose_df) > steps:
            pose_df = pose_df.iloc[1:]
        if len(pose_df) == steps:
            if debug_mode:
                file_name = 'model_input_' + datetime.now().strftime('%Y%m%d_%H%M%S%f') + '.csv'
                pose_df.to_csv('model_inputs/' + file_name,  index=False)
            pose_np = pose_df.values.reshape(1, steps, len(cols))
            pose_np = processing_pipeline.fit_transform(pose_np)
            if estimator in ['svm', 'rf']:
                pose_np = pose_np.reshape(1, -1)
            if estimator == 'svm':
                movement = model.predict(pose_np)[0]
            else:
                movement = np.argmax(model.predict(pose_np)[0])

    return movement

def interpolate(pose_df, ms_per_frame_interpolated):

    features = list(pose_df.filter(regex = '_(x|y)$', axis = 1).columns)
    t = pose_df["ms_since_start"].values

    interpolators = {}
    for feat in features:
        f = pose_df[feat].values

        cub_f = interp1d(t, f,
                         kind = 'cubic',
                         fill_value = (f[0],f[-1]),
                         bounds_error = False,
                         assume_sorted=True)

        interpolators[feat] = cub_f

    n = int(np.ceil(gesture_length/ms_per_frame_interpolated) + 1)
    t_delta = sorted([ms_per_frame_interpolated * i for i in range(0,-n,-1)])
    t_new = t_delta + t[-1]

    interp_df = pd.DataFrame(columns = features)
    interp_df["ms_since_start"] = t_new

    for feat in features:
        interp_df[feat] = interpolators[feat](t_new)

    return interp_df

# start websocket server
start_server = websockets.serve(consumer_handler, 'localhost', 8080)
print('Websocket server started. Please connect PoseNet on port 8080.')

asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()


''' This program create Bezier curves

    Press Add Button to be allowed to create new lines
    You can extend the lines for having differents curves
    Keep in mind that if you wanna drag a point, the Add Button
    must be off.
    The others buttons speak by them-selves

    strongly inspired by PM 2Ring 2018.09.21
'''

import tkinter as tk
from collections import deque

def flat(x0, y0, x1, y1, tol):
    return abs(x0*y1 - x1*y0) < tol * abs(x0 * x1 + y0 * y1)

def bezier_points(x0, y0, x1, y1, x2, y2, x3, y3, tol=0.001):
    ''' Draw a cubic Bezier cirve by recursive subdivision
        The curve is subdivided until each of the 4 sections is
        sufficiently flat, determined by the angle between them.
        tol is the tolerance expressed as the tangent of the angle
    '''
    if (flat(x1-x0, y1-y0, x2-x0, y2-y0, tol) and
        flat(x2-x1, y2-y1, x3-x1, y3-y1, tol)):
        return [x0, y0, x3, y3]

    x01, y01 = (x0 + x1) / 2., (y0 + y1) / 2.
    x12, y12 = (x1 + x2) / 2., (y1 + y2) / 2.
    x23, y23 = (x2 + x3) / 2., (y2 + y3) / 2.
    xa, ya = (x01 + x12) / 2., (y01 + y12) / 2.
    xb, yb = (x12 + x23) / 2., (y12 + y23) / 2.
    xc, yc = (xa + xb) / 2., (ya + yb) / 2.

    # Double the tolerance angle
    tol = 2. / (1. / tol - tol)
    return (bezier_points(x0, y0, x01, y01, xa, ya, xc, yc, tol)[:-2] +
        bezier_points(xc, yc, xb, yb, x23, y23, x3, y3, tol))

class Dot:
    ''' Movable Canvas circles '''
    dots = {}
    def __init__(self, canvas, x, y, color, rad=1, tags=""):
        self.x, self.y = x, y
        self.canvas = canvas
        self.id = canvas.create_oval(x-rad, y-rad, x+rad, y+rad,
            outline=color, fill='white', tags=tags)
        Dot.dots[self.id] = self

    def update(self, x, y):
        dx, dy = x - self.x, y - self.y
        self.x, self.y = x, y
        self.canvas.move(self.id, dx, dy)

    def __repr__(self):
        return f'Dot({self.id}, {self.x}, {self.y})'

    def delete(self):
        self.canvas.delete(self.id)
        del Dot.dots[self.id]
class Line:
    '''Line between circles'''
    lines = {}
    def __init__(self, canvas, x1, y1, x2, y2, color):
        self.x1, self.y1 = x1, y1
        self.x2, self.y2 = x2, y2
        self.canvas = canvas
        self.id = canvas.create_line(x1, y1, x2, y2, fill=color)
        Line.lines[self.id] = self

    def __repr__(self):
        return f'Line({self.id})'

    def delete(self):
        self.canvas.delete(self.id)
        del Line.lines[self.id]

class GUI:
    def __init__(self):
        self.root = root = tk.Tk()
        width = root.winfo_screenwidth() * 4 // 5
        height = root.winfo_screenheight() * 4 // 5
        root.geometry(f'{width}x{height}')
        root.title('Bezier curve drawing')
        root.rowconfigure(0, weight=1)
        root.columnconfigure(0, weight=1)

        self.canvas = canvas = tk.Canvas(root)
        canvas.grid(row=0, column=0, sticky='nsew')

        # The Bezier control points, and groups for each curve
        self.lines, self.dots, self.middots, self.groups = [], [], [], []

        # Various Canvas-related items: the current Dot being dragged, the
        # lines comprising the convex hull, the lines of the Bezier curve
        # and the animated spot.
        self.drag_item = self.curve = None

        # Add bindings for clicking & dragging any object with the "dot" tag
        canvas.tag_bind("dot", "<ButtonPress-1>", self.on_dot_press)
        canvas.tag_bind("dot", "<B1-Motion>", self.on_dot_motion)
        canvas.tag_bind("dot", "<ButtonRelease-1>", self.on_dot_release)

        canvas.tag_bind("mdot", "<ButtonPress-1>", self.on_mdot_press)
        canvas.tag_bind("mdot", "<B1-Motion>", self.on_mdot_motion)
        canvas.tag_bind("mdot", "<ButtonRelease-1>", self.on_dot_release)

        canvas.bind("<ButtonPress-1>", self.add_dot)
        canvas.bind("<ButtonRelease-1>", lambda evt: self.draw_lines())

        frame = tk.Frame(root, bd=1, relief=tk.SUNKEN)
        frame.grid(row=1, column=0, sticky='nsew')

        # Button to add new Dots to the canvas
        self.add_btn = tk.Button(frame, text="Add", relief="sunken",
            command=self.add_dot_cb)
        self.add_btn.pack(side=tk.LEFT)
        self.add_btn.state = True

        self.hide_btn = tk.Button(frame, text="Hide", relief="raised",\
            command=self.hide_cb)
        self.hide_btn.pack(side=tk.LEFT)
        self.hide_btn.state = False

        tk.Button(frame, text="Delete", command=self.delete_cb).pack(side=tk.LEFT)
        tk.Button(frame, text="Clear", command=self.clear_cb).pack(side=tk.LEFT)

        root.mainloop()

    def add_dot_cb(self):
        self.add_btn.state ^= 1
        self.add_btn["relief"] = "sunken" if self.add_btn.state else "raised"

    def add_dot(self, event):
        '''Add a new point'''
        if not self.add_btn.state:
            return
        x, y = self.canvas.canvasx(event.x), self.canvas.canvasy(event.y)
        color = "blue"
        self.dots.append(Dot(self.canvas, x, y, color, rad=4, tags="dot"))

    def clear_cb(self):
        for element in self.dots + self.middots + self.lines:
            element.delete()
        self.dots.clear()
        self.middots.clear()
        self.lines.clear()

        if self.curve:
            self.canvas.delete(self.curve)
            self.curve = None
        if not self.add_btn.state:
            self.add_dot_cb()

    def delete_cb(self):
        self.dots.pop().delete()
        if len(self.dots) % 2:
            self.lines.pop().delete()
            self.middots.pop().delete()
        self.draw_lines()

    def hide_cb(self):
        self.hide_btn.state ^= 1
        self.hide_btn["relief"] = "sunken" if self.hide_btn.state else "raised"
        state = tk.HIDDEN if self.hide_btn.state else tk.NORMAL
        for item in self.dots + self.middots + self.lines:
            self.canvas.itemconfig(item.id, state = state)

#######################################################

    def on_dot_press(self, event):
        ''' Beginning drag of a dot '''
        # Record the item being dragged
        x, y = self.canvas.canvasx(event.x), self.canvas.canvasy(event.y)
        self.drag_item = Dot.dots[self.canvas.find_closest(x, y)[0]]
        self.canvas.config(cursor = 'fleur')

    def on_dot_motion(self, event):
        ''' Handle dragging of a dot '''
        if self.drag_item is None:
            return
        id = self.dots.index(self.drag_item)
        # Get the mouse location
        x, y = self.canvas.canvasx(event.x), self.canvas.canvasy(event.y)
        # Move the Dot to the new location
        self.drag_item.update(x, y)
        self.canvas.tag_raise(self.drag_item.id)
        if id % 2 or not id == len(self.dots):
            if 1 < id < len(self.dots)-2:
                mp = self.middots[id//2 - 1]
                pair = self.dots[id-1] if id % 2 else self.dots[id+1]
                x, y = 2*mp.x - x, 2*mp.y - y
                pair.update(x, y)
                self.canvas.tag_raise(pair.id)
        self.draw_lines()

    def on_mdot_press(self, event):
        # Record the item being dragged
        x, y = self.canvas.canvasx(event.x), self.canvas.canvasy(event.y)
        self.drag_item = Dot.dots[self.canvas.find_closest(x, y)[0]]

    def on_mdot_motion(self, event):
        ''' Handle dragging of an mdot object '''
        if self.drag_item is None:
            return
        # Find the control points of the segment containing this mdot
        idx = self.middots.index(self.drag_item) * 2 + 2
        d0, d1 = self.dots[idx:idx+2]
        dx, dy = d0.x - self.drag_item.x, d0.y - self.drag_item.y,
        # Get the mouse location
        x, y = self.canvas.canvasx(event.x), self.canvas.canvasy(event.y)
        self.drag_item.update(x, y)
        d0.update(x+dx, y+dy)
        d1.update(x-dx, y-dy)

        self.draw_lines()

    def on_dot_release(self, event):
        ''' End drag of an object '''
        # Reset the drag information
        self.drag_item = None
        self.canvas.config(cursor = 'arrow')

################################################################

    def draw_lines(self):
        ''' Draw the lines of the Bezier curve '''
        if len(self.dots) < 2:
            return
        if self.lines:
            for i in range(len(self.lines)):
                self.lines.pop().delete()
        coords = []
        for dot in self.dots:
            coords.extend((dot.x, dot.y))
        # draw the lines
        max = (len(coords))//4 * 4
        for i in range(0, max, 4):
            self.lines.append(Line(self.canvas, *coords[i:i+4], "blue"))
            self.canvas.tag_lower(self.lines[-1].id)

        self.draw_curve()

    def draw_curve(self):
        ''' Draw midpoints and the curve'''
        if len(self.dots) < 4:
            return

        dot_queue = deque(self.middots)

        # Draw midpoints
        coords = []
        for i, dot in enumerate(self.dots):
            x, y = dot.x, dot.y
            if i % 2 and 1 < i < len(self.dots)-2:
                px, py = coords[-2:]
                if dot_queue:
                    mdot = dot_queue.popleft()
                    mx, my = (px+x)*0.5, (py+y)*0.5
                    mdot.update(mx, my)
                    self.canvas.tag_raise(mdot.id)
                else:
                    mx, my = (px+x)*0.5, (py+y)*0.5
                    mdot = Dot(self.canvas, mx, my, "white", tags="mdot", rad=4)
                    self.middots.append(mdot)
                coords.extend((mx, my))
            coords.extend((x, y))

        # Draw the Bezier curves
        points = []
        self.groups.clear()
        maxi = (len(coords)) // 6 * 6
        for i in range(0, maxi, 6):
            group = coords[i:i+8]
            #print(group)
            if len(group) < 8:
                break
            points.extend(bezier_points(*group)[:-2])
            self.groups.append(group)

        if self.curve:
            self.canvas.coords(self.curve, points)
        else:
            self.curve = self.canvas.create_line(points, fill="red")

if __name__ == "__main__":
    GUI()


from Dataset.DataOperate import MySet,MySet_npy, get_data_list
from Utils.Utils import DiceLoss,metrics
from Utils.focal_loss import focal_loss
from Model.DMFNet_16x import My_DFMNet,New_DMFNet,New_DMFNet_NewRecons,New_Att_DMFNet,DMF_FPN,Deep_Attentive_DMFNet
from Model.transformer.vit_seg_modeling import VisionTransformer
import time
import os
import numpy as np
import cv2
import SimpleITK as itk
import torch
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from torch.autograd import Variable
from torch.nn import functional as F
from torch.optim import lr_scheduler
torch.set_printoptions(profile="full")
np.set_printoptions(threshold=np.inf)

def get_k_fold_data(k, i, X):  ###此过程主要是步骤（1）
    # 返回第i折交叉验证时所需要的训练和验证数据，分开放，X_train为训练数据，X_valid为验证数据
    assert k > 1
    fold_size = len(X) // k  # 每份的个数:数据总条数/折数（组数）
    train_list, val_list = None, None
    for j in range(k):
        idx = slice(j * fold_size, (j + 1) * fold_size)  # slice(start,end,step)切片函数
        ##idx 为每组 valid
        part = X[idx]
        if j == i:  ###第i折作valid
            val_list = part
        elif train_list is None:
            train_list = part
        else:
            train_list = train_list + part  # dim=0增加行数，竖着连接
    return train_list, val_list
def k_fold(k, X, train_log, val_seg_log, val_cls_log, val_path, train_path,tensorboard_dir):
    for i in range(k):
        writer = SummaryWriter(tensorboard_dir+'/fold%d'%i)
        total = 0
        positive = 0
        torch.cuda.empty_cache()
        net =VisionTransformer()
        net = torch.nn.DataParallel(net, device_ids).cuda()
        optimizer = torch.optim.Adam(net.parameters(), lr=1e-4,weight_decay=0.05)
        scheduler = lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.75)
        train_list, val_lsit = get_k_fold_data(k, i, X)  # 获取k折交叉验证的训练和验证数据
        train_set = MySet_npy(train_list)
        train_loader = DataLoader(train_set, batch_size=1, shuffle=True)
        val_set = MySet_npy(val_lsit)
        val_loader = DataLoader(val_set, batch_size=1, shuffle=False)
        for batch_idx, (image, mask, label, name) in enumerate(train_loader):
            label = Variable(label[0].cuda())
            l_class = label.data.cpu().numpy()[0]
            if l_class == 1:
                positive += 1
            total += 1
        positive_weight = positive / total
        criterion_mse = focal_loss(alpha=positive_weight)
        criterion_bce = torch.nn.BCELoss()
        l2_loss = torch.nn.MSELoss()
        criterion_dice = DiceLoss()
        ### 每份数据进行训练,体现步骤三####
        train(i, net, train_loader, val_loader, optimizer, criterion_bce, criterion_dice, criterion_mse,l2_loss, scheduler,
              train_log, val_seg_log, val_cls_log, val_path,train_path,writer)

def train(k, net, train_loader, val_loader, optimizer, criterion_bce, criterion_dice, criterion_mse,l2_loss, scheduler,train_log,val_seg_log,val_cls_log, val_path,train_path,writer):
    for epoch in range(0, 80):
        best_dice = 0.
        best_recall = 0.
        epoch_start_time = time.time()
        print("Epoch: {}".format(epoch))
        epoch_loss = 0.
        total_num = 1
        true_num = 0
        p_num=0
        p_thresh=0
        p_min = 1
        for batch_idx, (image,mask,label,name) in enumerate(train_loader):
            print(name)
            start_time = time.time()
            image = Variable(image.cuda())
            mask = Variable(mask.cuda())
            label = Variable(label.cuda())

            output= net(image)

            output = F.sigmoid(output)
            #recons1_l2loss = l2_loss(recons,image)
            loss0_bce = criterion_bce(output[0], mask)
            loss0_dice = criterion_dice(output, mask)

            #loss = 0.8 * loss0_dice + 4.0 * cls_loss + 0.3 * loss0_bce + 0.3 * loss1_dice + 0.2 * loss1_bce
            loss = 1.0 * loss0_dice + 0.5 * loss0_bce
            #loss = 1.0 * loss0_dice + 4.0 * cls_loss + 0.5 * loss0_bce

            epoch_loss += loss.item()
            print('Fold: {} | Epoch: {} | Batch: {} | Patient: {:20}----->Train loss: {:4f} | Cost Time: {}\n'.format(k, epoch, batch_idx,name[0],loss.item(),time.time() - start_time))
            open(train_log, 'a').write('Fold: {} | Epoch: {} | Batch: {} | Patient: {:30}----->Train loss: {:4f} | Cost Time: {}\n'.format(k, epoch, batch_idx,name[0],loss.item(),time.time() - start_time))
            loss.backward()
            optimizer.step()
        print("Fold: {} | Epoch: {} | Loss: {} | Acc: {} | Time: {}\n".format(k, epoch, epoch_loss / (batch_idx + 1),true_num/total_num,time.time() - epoch_start_time))
        open(train_log, 'a').write("Fold: {} | Epoch: {} | Loss: {} | Acc: {} | Time: {}\n".format(k, epoch, epoch_loss / (batch_idx + 1),true_num/total_num,time.time() - epoch_start_time))
        writer.add_scalar('train_loss', epoch_loss / (batch_idx + 1), global_step=epoch)
        writer.add_scalar('train_acc', true_num/total_num, global_step=epoch)
        scheduler.step()
        # begin to eval
        net.eval()
        total_num = 1
        true_num = 0
        tp = 0
        fp = 0
        tn = 0
        fn = 0
        true_num1 = 0
        tp1 = 0
        fp1 = 0
        tn1 = 0
        fn1 = 0
        dice = 0.
        jaccard = 0.
        seg_precision = 0.
        seg_sensitivity = 0.
        seg_specificity = 0.
        hd95 = 0.
        with torch.no_grad():
            for batch_idx, (image, mask, label, name) in enumerate(val_loader):
                image = Variable(image.cuda())
                mask = Variable(mask.cuda())
                label = Variable(label.cuda())
                #val_predict1, val_predict, cls_out = net(image)
                val_predict = net(image)

                val_predict = F.sigmoid(val_predict)
                mask = mask.data.cpu().numpy()
                val_predict = val_predict.data.cpu().numpy()
                image = image.data.cpu().numpy()
                save_predict = np.squeeze(val_predict)
                val_seg = np.zeros_like(save_predict)
                val_seg[save_predict > 0.4] = 1

                seg_output = itk.GetImageFromArray(val_seg)
                posibility_output = itk.GetImageFromArray(save_predict)
                image_output = itk.GetImageFromArray(image)
                mask_output = itk.GetImageFromArray(mask)

                save_dir = val_path + '/' + "fold_%d" % k + '/' + name[0]
                if not os.path.exists(save_dir):
                    os.makedirs(save_dir)
                save_seg_dir = save_dir + "/%s_seg.nii" % epoch
                itk.WriteImage(seg_output, save_seg_dir)
                save_posibility_dir = save_dir + "/%s_posibility.nii" % epoch
                itk.WriteImage(posibility_output, save_posibility_dir)
                save_mask_dir = save_dir + "/label.nii"
                itk.WriteImage(mask_output, save_mask_dir)
                save_image_dir = save_dir + "/image.nii"
                itk.WriteImage(image_output, save_image_dir)
                print("val_seg.shape")
                print(val_seg.shape)
                print("mask.shape")
                print(mask[0][0].shape)
                dice_tmp, hd_tmp, jaccard_tmp, seg_precision_tmp, seg_sensitivity_tmp, seg_specificity_tmp = metrics(
                    val_seg, mask[0][0])
                open(val_seg_log, 'a').write(
                    "Fold: {} | Epoch: {} | Patient: {:20} | Dice: {:5f} | HD95: {:5f} | Jaccard: {:5f} | Seg_precision: {:5f} | Seg_sensitivity: {:5f} | Seg_specificity: {:5f}\n".format(
                        k, epoch, name[0], dice_tmp, hd_tmp, jaccard_tmp, seg_precision_tmp, seg_sensitivity_tmp,
                        seg_specificity_tmp))
                dice = dice + dice_tmp
                jaccard = jaccard + jaccard_tmp
                seg_precision = seg_precision + seg_precision_tmp
                seg_sensitivity = seg_sensitivity + seg_sensitivity_tmp
                hd95 = hd_tmp + hd95
                seg_specificity = seg_specificity_tmp + seg_specificity

        #epoch cls metrics

        # epoch seg metrics
        dice = dice / (1 + batch_idx)
        jaccard = jaccard / (1 + batch_idx)
        seg_precision = seg_precision / (1 + batch_idx)
        seg_sensitivity = seg_sensitivity / (1 + batch_idx)
        hd95 = hd95 / (1 + batch_idx)
        seg_specificity = seg_specificity / (1 + batch_idx)

        print("Fold {} Epoch {} dice: {} hd95: {} jaccard: {} seg_precision: {} seg_sensitivity: {} seg_specificity: {}\n".format(
                k, epoch, dice, hd95, jaccard, seg_precision, seg_sensitivity, seg_specificity))
        open(val_seg_log, 'a').write(
            "Fold {} Epoch {} dice: {} hd95: {} jaccard: {} seg_precision: {} seg_sensitivity: {} seg_specificity: {}\n".format(
                k, epoch, dice, hd95, jaccard, seg_precision, seg_sensitivity, seg_specificity))

        writer.add_scalar('dice', dice, global_step=epoch)
        writer.add_scalar('jaccard', jaccard, global_step=epoch)
        writer.add_scalar('hd95', hd95, global_step=epoch)
        writer.add_scalar('val_seg_precision', seg_precision, global_step=epoch)
        writer.add_scalar('val_seg_sensitivity', seg_sensitivity, global_step=epoch)
        writer.add_scalar('val_seg_specificity', seg_specificity, global_step=epoch)

        if dice > best_dice:
            best_dice = dice
            torch.save(net.state_dict(), check_point + '/Best_Dice.pth')

        torch.save(net.state_dict(), check_point + '/model_fold{}_epoch{}.pth'.format(k, epoch))

os.environ["CUDA_VISIBLE_DEVICES"] = "3"
device_ids = [0]
train_list = get_data_list("/home/ubuntu/liuyiyao/3D_breast_Seg/Dataset/miccai_data_64*256*256_patch", ratio=0.8)
train_log = './Transformer_0325/train/trainLog.txt'
val_seg_log = './Transformer_0325/valid/valsegLog.txt'
val_cls_log = './Transformer_0325/valid/valclsLog.txt'
val_path = './Transformer_0325/valid'
train_path = './Transformer_0325/train'
check_point = './Transformer_0325/checkpoints'
tensorboard_dir = './Transformer_0325/tensorboard_log'

if not os.path.exists(check_point):
    os.makedirs(check_point)
if not os.path.exists(val_path):
    os.makedirs(val_path)
if not os.path.exists(train_path):
    os.makedirs(train_path)
if not os.path.exists(tensorboard_dir):
    os.makedirs(tensorboard_dir)
information_line = '=' * 20 + ' Transformer_0325-TRAIN ' + '=' * 20 + '\n'
open(train_log, 'w').write(information_line)
information_line = '=' * 20 + ' Transformer_0325-VAL-SEG ' + '=' * 20 + '\n'
open(val_seg_log, 'w').write(information_line)
information_line = '=' * 20 + ' Transformer_0325-VAL-CLS ' + '=' * 20 + '\n'
open(val_cls_log, 'w').write(information_line)
K = 5
k_fold(K, train_list, train_log,val_seg_log, val_cls_log, val_path,train_path,tensorboard_dir)


import sys
from PyQt5.QtWidgets import (QCheckBox, QWidget,
							QPushButton, QFileDialog, QApplication,
							QPlainTextEdit, QHBoxLayout, QVBoxLayout)
from PyQt5.QtCore import Qt, QSize
from PyQt5.QtGui import QImage, QPalette, QBrush
from completeness import run_detection

class ChangeDetectorApp(QWidget):

	image1 = ""
	image2 = ""
	result_path = ""
	AppSize = (600, 300)
	NUMBER_OF_SLICES = 9

	def __init__(self):
		super().__init__()
		self.AppTitle = "Satellite Images Change Detector"
		self.initUI()
		self.setFixedSize(*self.AppSize)
		self.setWindowTitle(self.AppTitle)
		self.show()
		self.output.insertPlainText(">>> Program Started\n")

	def initUI(self):

		#set background image of main window
		oImage = QImage("background.jpg")
		sImage = oImage.scaled(QSize(*self.AppSize))
		palette = QPalette()
		palette.setBrush(10, QBrush(sImage))
		self.setPalette(palette)

		#settings of image 1 button
		image1 = QPushButton('First Image', self)
		image1.setToolTip('Click to choose first photo')
		image1.resize(image1.sizeHint())
		image1.clicked.connect(self.openFile1NameDialog)

		#settings of image 2 button
		image2 = QPushButton('Second Image', self)
		image2.setToolTip('Click to choose second photo')
		image2.resize(image2.sizeHint())
		image2.clicked.connect(self.openFile2NameDialog)

		#settings of result button
		self.result = QPushButton('Result path', self)
		self.result.setToolTip('Click to choose where to save the result')
		self.result.resize(self.result.sizeHint())
		self.result.clicked.connect(self.saveFileDialog)
		self.result.setEnabled(False)

		#settings of save result check box
		self.save_res = QCheckBox('Save Result', self)
		self.save_res.setToolTip('Click this to save the output result on the disk')
		self.save_res.stateChanged.connect(self.saveResult)
		self.save_res.setAutoFillBackground(True)
		p = self.save_res.palette()
		p.setColor(self.save_res.backgroundRole(), Qt.white)
		self.save_res.setPalette(p)

		#settings of detect button
		detect = QPushButton('Detect', self)
		detect.setToolTip('Click to detect changes between inputs')
		detect.resize(detect.sizeHint())
		detect.clicked.connect(self.detectClicked)

		# creating a plainText box for output messages
		self.output = QPlainTextEdit(self)

		#settings of quit button
		quit = QPushButton('Quit', self)
		quit.setToolTip('Click to quit the app')
		quit.resize(quit.sizeHint())
		quit.clicked.connect(QApplication.instance().quit)

		# setting layout of main window
		resultBox = QHBoxLayout()
		resultBox.setSpacing(10)
		resultBox.addWidget(self.result)
		resultBox.addWidget(self.save_res)

		leftBox = QVBoxLayout()
		leftBox.addWidget(image1)
		leftBox.addWidget(image2)
		leftBox.addLayout(resultBox)
		leftBox.addStretch(1)
		leftBox.addWidget(detect)

		rightBox = QVBoxLayout()
		rightBox.addWidget(self.output)
		rightBox.addStretch(1)
		rightBox.addWidget(quit)

		layout = QHBoxLayout()
		layout.addLayout(leftBox)
		layout.addStretch(1)
		layout.addLayout(rightBox)

		self.setLayout(layout)

	def detectClicked(self):
		if self.image1 == "":
			self.output.insertPlainText("! Error: Enter first image\n")
		elif self.image2 == "":
			self.output.insertPlainText("! Error: Enter second image\n")
		else:
			self.output.insertPlainText(">>> Detection Started\n")
			run_detection(self.image1, self.image2, self.NUMBER_OF_SLICES)

	def openFile1NameDialog(self):
		options = QFileDialog.Options()
		options |= QFileDialog.DontUseNativeDialog
		fileName, _ = QFileDialog.getOpenFileName(self,"QFileDialog.getOpenFileName()", "","All Files (*);;PNG Files (*.png);;JPG Files (*.jpg);;JPEG Files (*.jpeg)", options=options)
		if fileName:
			self.image1 = fileName
			self.output.insertPlainText(">>> First image is {}\n".format(self.image1))

	def openFile2NameDialog(self):
		options = QFileDialog.Options()
		options |= QFileDialog.DontUseNativeDialog
		fileName, _ = QFileDialog.getOpenFileName(self,"QFileDialog.getOpenFileName()", "","All Files (*);;PNG Files (*.png);;JPG Files (*.jpg);;JPEG Files (*.jpeg)", options=options)
		if fileName:
			self.image2 = fileName
			self.output.insertPlainText(">>> Second image is {}\n".format(self.image2))

	def saveFileDialog(self):
		options = QFileDialog.Options()
		options |= QFileDialog.DontUseNativeDialog
		fileName, _ = QFileDialog.getSaveFileName(self,"QFileDialog.getSaveFileName()","","All Files (*);;PNG Files (*.png)", options=options)
		if fileName:
			self.result_path = fileName
			self.output.insertPlainText(">>> Result save to {}\n".format(self.result_path))

	def saveResult(self, state):
		if state == Qt.Checked:
			self.result.setEnabled(True)
			self.result_path = ""
		else:
			self.result.setEnabled(False)
			self.result_path = ""

if __name__ == '__main__':
	app = QApplication(sys.argv)
	ex = ChangeDetectorApp()
	sys.exit(app.exec_())


import rospy
from std_msgs.msg import Int32

n = 0

def cb(message):
    global n
    n = message.data

rospy.init_node('gusu0')
sub = rospy.Subscriber('count_up', Int32, cb)
pub = rospy.Publisher('gusu0', Int32, queue_size=1)
rate = rospy.Rate(10)
while not rospy.is_shutdown():
    if n % 2 == 0:
        n = 0;
    pub.publish(n)
    rate.sleep()


import vim

from gpt import default as gpt, GptError
from gpt import get_text_for_completion

class MyException(Exception):
    pass

def error(msg):
    raise MyException(msg)

def set_error(msg):
    msg = str(msg).replace("'", "''")
    vim.command(f"let s:error_msg = '{msg}'")

def empty():
    available = ' '.join(commands.keys())
    print(f"Available commands: {available}")

def status(args):
    if len(args) != 0:
        error('status command has no args')
    gpt.print_status()

def complete(args):
    if len(args) != 0:
        error('complete command has no args')

    buf = vim.current.buffer[:]
    l, c = vim.current.window.cursor

    text = get_text_for_completion(buf, l)
    insertion = gpt.complete_text(text)
    insertion.append('<' * 42)
    vim.current.buffer[l:l] = insertion

def ask(args):
    if len(args) == 0:
        error('Please, type a question after ask')

    text = ' '.join(args)
    for line in gpt.complete_text(text, max_tokens=2048):
        print(line)

commands = {
    'complete' : complete,
    'ask' :      ask,
    'status'   : status,
}

def execute(args):
    if len(args) == 0:
        return empty()

    cmd, *args = args
    func = commands.get(cmd)
    if func is None:
        error(f"Unknown command '{cmd}'")

    func(args)

def entry(cmd):
    vim.command('let s:error_msg = ""')
    try:
        args = str(cmd).split()
        execute(args)
    except (MyException, GptError) as e:
        set_error(e)


#!/usr/bin/python3
#  Python script that takes in a URL and an email, sends a POST request
# to the passed URL with the email as a parameter, and displays the
# body of the response
import requests
import sys

if __name__ == "__main__":

    url = "https://swapi.co/api/people/?search={}".format(sys.argv[1])

    req = requests.get(url)
    resp = req.json()
    print("Number of results: {}".format(resp['count']))
    for role in resp['results']:
        print(role['name'])


import string
from ps4a import get_permutations

def load_words(file_name):
    '''
    file_name (string): the name of the file containing
    the list of words to load

    Returns: a list of valid words. Words are strings of lowercase letters.

    Depending on the size of the word list, this function may
    take a while to finish.
    '''

    # inFile: file
    inFile = open(file_name, 'r')
    # wordlist: list of strings
    wordlist = []
    for line in inFile:
        wordlist.extend([word.lower() for word in line.split(' ')])
    return wordlist

def is_word(word_list, word):
    '''
    Determines if word is a valid word, ignoring
    capitalization and punctuation

    word_list (list): list of words in the dictionary.
    word (string): a possible word.

    Returns: True if word is in word_list, False otherwise

    Example:
    >>> is_word(word_list, 'bat') returns
    True
    >>> is_word(word_list, 'asdf') returns
    False
    '''
    word = word.lower()
    word = word.strip(" !@#$%^&*()-_+={}[]|\:;'<>?,./\"")
    return word in word_list

WORDLIST_FILENAME = 'words.txt'

VOWELS_LOWER = 'aeiou'
VOWELS_UPPER = 'AEIOU'
CONSONANTS_LOWER = 'bcdfghjklmnpqrstvwxyz'
CONSONANTS_UPPER = 'BCDFGHJKLMNPQRSTVWXYZ'

# The code above this line had been provided by MIT

class SubMessage(object):
    def __init__(self, text):
        '''
        Initializes a SubMessage object

        text (string): the message's text

        A SubMessage object has two attributes:
            self.message_text (string, determined by input text)
            self.valid_words (list, determined using helper function load_words)
        '''
        self.message_text = text
        self.valid_words = load_words(WORDLIST_FILENAME)

    def get_message_text(self):
        '''
        Used to safely access self.message_text outside of the class

        Returns: self.message_text
        '''
        return self.message_text

    def get_valid_words(self):
        '''
        Used to safely access a copy of self.valid_words outside of the class.
        This helps you avoid accidentally mutating class attributes.

        Returns: a COPY of self.valid_words
        '''
        words = self.valid_words.copy()
        return words

    def build_transpose_dict(self, vowels_permutation):
        '''
        vowels_permutation (string): a string containing a permutation of vowels (a, e, i, o, u)

        Creates a dictionary that can be used to apply a cipher to a letter.
        The dictionary maps every uppercase and lowercase letter to an
        uppercase and lowercase letter, respectively. Vowels are shuffled
        according to vowels_permutation. The first letter in vowels_permutation
        corresponds to a, the second to e, and so on in the order a, e, i, o, u.
        The consonants remain the same. The dictionary should have 52
        keys of all the uppercase letters and all the lowercase letters.

        Example: When input "eaiuo":
        Mapping is a->e, e->a, i->i, o->u, u->o
        and "Hello World!" maps to "Hallu Wurld!"

        Returns: a dictionary mapping a letter (string) to
                 another letter (string).
        '''
        vowel_dict = {}
        vowels_permutation_lower = vowels_permutation.lower()
        vowels_permutation_upper = vowels_permutation.upper()

        for i in range(5):
            vowel_dict[VOWELS_LOWER[i]] = vowels_permutation_lower[i]
            vowel_dict[VOWELS_UPPER[i]] = vowels_permutation_upper[i]

        return vowel_dict

    def apply_transpose(self, transpose_dict):
        '''
        transpose_dict (dict): a transpose dictionary

        Returns: an encrypted version of the message text, based
        on the dictionary
        '''
        words = self.get_message_text()

        # if the letter is a vowel, replace it with the corresponding value associated with that vowel in the given dict
        for index in range(len(words)):
            if words[index] in VOWELS_LOWER or words[index] in VOWELS_UPPER:
                words = words[:index] + transpose_dict[words[index]] + words[index+1:]
            else:
                continue

        return words

class EncryptedSubMessage(SubMessage):
    def __init__(self, text):
        '''
        Initializes an EncryptedSubMessage object

        text (string): the encrypted message text

        An EncryptedSubMessage object inherits from SubMessage and has two attributes:
            self.message_text (string, determined by input text)
            self.valid_words (list, determined using helper function load_words)
        '''
        SubMessage.__init__(self,text)

    def decrypt_message(self):
        '''
        Attempt to decrypt the encrypted message

        Idea is to go through each permutation of the vowels and test it
        on the encrypted message. For each permutation, check how many
        words in the decrypted text are valid English words, and return
        the decrypted message with the most English words.

        If no good permutations are found (i.e. no permutations result in
        at least 1 valid word), return the original string. If there are
        multiple permutations that yield the maximum number of words, return any
        one of them.

        Returns: the best decrypted message

        Hint: use your function from Part 4A
        '''

        permsofvowels = get_permutations('aeiou')
        wordcount_dict = {}                        # stores permutations as keys and the number of valid words generated by applying that permutation to the text as values

        for i in permsofvowels:
            dict = super().build_transpose_dict(i)
            permappliedtext = super().apply_transpose(dict)
            words = permappliedtext.split()
            for word in words:
                if is_word(self.valid_words, word):
                    wordcount_dict[i] = wordcount_dict.get(i,0) + 1

        highestvalue = 0
        for value in wordcount_dict.values():
            if value > highestvalue:
                highestvalue = value

        if highestvalue > 0:
            bestperm = [shiftval for shiftval,numofwords in wordcount_dict.items() if numofwords == highestvalue][0]

            finaldict = super().build_transpose_dict(bestperm)
            finaltext = super().apply_transpose(finaldict)
            return finaltext

        else:
            return self.message_text()

if __name__ == '__main__':

    welcome = """
    Hello. This is another version of the 2-in-1 encryption & decryption service provided by ps4b.
    In the case of encryption, when you give me a string with the 5 vowels in any order. I replace all the vowels in
    your given text according to the order you specify.
    For instance if you give me 'ouiea', 'a' in your text will be changed to 'o', 'e' will be changed to 'u', and so on...
    In the case of encryption, if you give me a text with the vowels all mixed up according to a certain ordering of the vowels,
    I can give you back the decrypted text.
    For instance, if you give me 'Hallu Wurld', I will give you back 'Hello World'
    """
    print(welcome)

    while True:
        enc_or_dec = input("Do you want to encrypt a message or decrypt it? Please enter 'e' for encryption and 'd' for decryption: ")
        textinput = input("Please enter your text: ")
        if (enc_or_dec == 'e' or enc_or_dec == 'd') and isinstance(textinput,str):
            break
        else:
            print("Invalid input! Please try again.")
            continue

    if enc_or_dec == 'e':
        perm = input("Please enter your unique order of vowels to be applied to your text: ")
        while True:
            if isinstance(perm,str) and len(perm) == 5:
                break
            else:
                print("Invalid input! Please try again.")
                continue

        print("You have chosen to encrypt the text '" + textinput + "' with the following order of vowels: " + perm + ".")
        message = SubMessage(textinput)
        enc_dict = message.build_transpose_dict(perm)
        print("Encrypted text:", message.apply_transpose(enc_dict))

    else:
        print("You have chosen to decrypt the text '" + textinput + "'")
        message = EncryptedSubMessage(textinput)
        print("Decrypted text:", message.decrypt_message())


import math
import random
import time
from math import floor

class Data:
    def __init__(self, key=None):
        self.key = key
        self.literal = str(key)

class ChainedHash:
    def __init__(self, p = 27, m = 10000):
        self.p = p
        self.m = m
        self.a = random.randint(0, p)
        self.b = random.randint(1, p)
        self.L = [[]] * m

    def insert(self, x):
        h = self.hash(x.key)
        if x in self.L[h]:
            self.L[h].remove(x)
        self.L[h].append(x)

    def search(self, k):
        h = self.hash(k)
        for i in self.L[h]:
            if i.key == k:
                return i
        return None

    def delete(self, x):
        h = self.hash(x.key)
        if x in self.L[h]:
            self.L[h].remove(x)

    def hash1(self, k):
        return k % m

    def hash2(self, k):
        KNUTH = 0.6180339887
        return floor(m * ((k * KNUTH) % 1))

    def hash3(self, k):
        return ((a*k + b) % p) % m

    def hash(self, k):
        return self.hash1(k)

def random_list(min, max, elements):
    list = random.sample(range(min, max), elements)
    return list

def test(n, p, m):
    l = random_list(1, n + 1, n)
    start_time = time.clock()

    L = ChainedHash(p, m)

    for i in l:
        x = Data(i)
        L.insert(x)

    end_time = time.clock() - start_time

    print("n:", n, "m:", m, "p:", p, "duration:", end_time)

if __name__ == "__main__":
    for n in [10000, 50000, 100000]:
        for p in [23, 9973, 99991]:
            for m in [p, p//2, p//4]:
                test(n, p, m)


class Tree:

    def __init__(self, value):
        self.value = value
        self.left = None
        self.right = None

    def Inorder(self, Root):
        if Root is None:
            return
        self.Inorder(Root.left)
        print(Root.value, end=' ')
        self.Inorder(Root.right)

    def Insert(self, value):
        if self is None:
            self = Tree(value)
        elif value < self.value:
            if self.left is None:
                self.left = Tree(value)
            else:
                self.left.Insert(value)
        else:
            if self.right is None:
                self.right = Tree(value)
            else:
                self.right.Insert(value)

Root = Tree(6)
Root.Insert(4)
Root.Insert(2)
Root.Insert(5)
Root.Insert(9)
Root.Insert(8)
Root.Insert(10)

print("Inorder traversal after insertion: ", end='')
Root.Inorder(Root)


def ordena(t):
    return  int(t[2]), t[1], t[0]
while True:
    try:
        lista = []
        N = int(input())
        for n in range(N):
            lista.append(input().split(" "))
        lista = sorted(lista, key=ordena)
        for i in lista:
            print(i[0])
    except EOFError:
        break


# 子集II
class Solution:
    def subsetsWithDup(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        res = [[]]
        nums.sort()

        for num in nums:
            for temp in res[:]:
                x = temp[:]
                x.append(num)
                if x not in res:
                    res.append(x)
        return res

print(Solution().subsetsWithDup([1, 2, 2]))


import random
#Step 1

word_list = ["ardvark", "baboon", "camel"]
stages = ['''
  +---+
  |   |
  O   |
 /|\  |
 / \  |
      |
=========
''', '''
  +---+
  |   |
  O   |
 /|\  |
 /    |
      |
=========
''', '''
  +---+
  |   |
  O   |
 /|\  |
      |
      |
=========
''', '''
  +---+
  |   |
  O   |
 /|   |
      |
      |
=========''', '''
  +---+
  |   |
  O   |
  |   |
      |
      |
=========
''', '''
  +---+
  |   |
  O   |
      |
      |
      |
=========
''', '''
  +---+
  |   |
      |
      |
      |
      |
=========
''']

#TODO-1 - Randomly choose a word from the word_list and assign it to a variable called chosen_word.
selected_word = random.choice(word_list)
print(selected_word)

# print as many blanks as letters:
representation = ["_"] * len(selected_word)
#print(representation) #used for debugging

end_of_game = False

while not end_of_game:
  #TODO-2 - Ask the user to guess a letter and assign their answer to a variable called guess. Make guess lowercase.
  guess_letter = input("Please guess a letter\n").lower()
  #TODO-3 - Check if the letter the user guessed (guess) is one of the letters in the chosen_word.

  hits = []
  selected_word_list = list(selected_word)

  for i in range(0, len(selected_word)):
    if guess_letter == selected_word_list[i]:
      hits.append(i)

  # print(hits) # used for debugging

  # replace guessed items in representation
  if len(hits) == 0:
    actual_stage = stages.pop(-1)
    print(actual_stage)
    if len(stages) == 0:
      end_of_game = True
      print("You lose")
  else:
    for i in hits:
      representation[i] = guess_letter

  print(representation)
  if not "_" in representation:
    end_of_game = True
    print("You won!")


"""
Origin: QE by John Stachurski and Thomas J. Sargent
Filename: ifp_savings_plots.py
Authors: John Stachurski, Thomas J. Sargent
LastModified: 11/08/2013

"""

from matplotlib import pyplot as plt
from quantecon import compute_fixed_point
from ifp import ConsumerProblem

# === solve for optimal consumption === #
m = ConsumerProblem(r=0.03, grid_max=4)
v_init, c_init = m.initialize()

# Coleman Operator takes in (c)?
c = compute_fixed_point(m.coleman_operator, c_init)
a = m.asset_grid
R, z_vals = m.R, m.z_vals

# === generate savings plot === #
fig, ax = plt.subplots()
ax.plot(a, R * a + z_vals[0] - c[:, 0], label='low income')
ax.plot(a, R * a + z_vals[1] - c[:, 1], label='high income')
ax.plot(a, a, 'k--')
ax.set_xlabel('current assets')
ax.set_ylabel('next period assets')
ax.legend(loc='upper left')
plt.show()


from flask import Flask, request, jsonify, render_template, redirect, url_for,send_file
from os import path
import json
from newspaper import fulltext
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
import re
from summarizer import Summarizer
import PyPDF2
import fpdf
import docx
import pathlib
from urllib.parse import urlparse
import requests
import os
from transformers import *

app = Flask(__name__)

#--------------------helper functions for inner processing--------------------#

def clean_and_process(text):
    # cleaning the text

    clean_abstract = []
    clean_abstract = re.sub(r'\d', ' ', text)
    clean_abstract = re.sub(r'\W', ' ', clean_abstract)
    clean_abstract = re.sub(r'\s+', ' ', text)
    text = re.sub(r'\[[0-9]*\]', ' ', text)

    # saving the wordcloud for pictorial  representation

    wordcloud = WordCloud(max_font_size=100, max_words=100,
                          background_color="white").generate(str(text))
    plt.figure()
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.savefig('static/img/wordcloud/wordcloud.png')
    #url_for('static',filename='img/wordcloud/wordcloud.png')
    return text

def BERT(cleaned_text):

    #custom_config = AutoConfig.from_pretrained('bert-large-uncased-config.json')
    #custom_config.output_hidden_states=True
    #custom_tokenizer = AutoTokenizer.from_pretrained('./bert-base-uncased-vocab.txt')
    #custom_model = AutoModel.from_pretrained('bert-large-uncased-pytorch_model.bin', config=custom_config)

    model = Summarizer()
    result = model(cleaned_text, min_length=30)
    full = "".join(result)
    return full

def url_validator(url):
    try:
        result = urlparse(url)
        return all([result.scheme, result.netloc, result.path])
    except:
        return False

def dump(text):
    mydoc = docx.Document()
    mydoc.add_heading("Summary", 0)
    mydoc.add_paragraph(text)
    mydoc.add_picture("static/img/wordcloud/wordcloud.png", width=docx.shared.Inches(5), height=docx.shared.Inches(6))

    mydoc.save('static/download/file.docx')

    #-----------------------------Routing methods----------------------------#

@app.route('/', methods=['GET', 'POST'])
def home():

    ################ for removing the previous wordcloud img on every request,so server dont gets loaded#############
    wordcloud = pathlib.Path("static/img/wordcloud/wordcloud.png")
    if wordcloud.is_file():
        os.remove(wordcloud)

    file = pathlib.Path("static/download/file.docx")
    if file.is_file():
        os.remove(file)

    return render_template('index.html')

@app.route('/PDF', methods=['GET', 'POST'])
def PDF():

    ################ for removing the previous wordcloud img on every request,so server dont gets loaded#############
    wordcloud = pathlib.Path("/static/img/wordcloud/wordcloud.png")
    if wordcloud.is_file():
        os.remove(wordcloud)

    file = pathlib.Path("static/download/file.docx")
    if file.is_file():
        os.remove(file)

    return render_template('PDF.html')

@app.route('/PDF_result', methods=['GET', 'POST'])
def PDF_result():

    ################ for removing the previous wordcloud img on every request,so server dont gets loaded#############
    wordcloud = pathlib.Path("static/img/wordcloud/wordcloud.png")
    if wordcloud.is_file():
        os.remove(wordcloud)

    if request.method == 'POST':
        if(not request.files['name']):
            return "
 Please for the sake of lord , provide a file
"
        else:
            f = request.files['name']
            f.save(f.filename)

            # for checking the file format
            suffix = pathlib.Path(f.filename).suffix

            # for PDF file
            if suffix == ".pdf":
                pdf = open(f.filename, 'rb')

                pdfReader = PyPDF2.PdfFileReader(pdf)
                num = pdfReader.numPages
                # indexing of pages is from (0 to num1)
                pageobj = pdfReader.pages
                text = ""
                for i in range(num):  # iterating over every page
                    # we have the raw text in the text variable
                    text += pageobj[i].extractText()
                pdf.close()
                cleaned_text = clean_and_process(text)

                # summarizer BERT model

                summary = BERT(cleaned_text)
                dump(summary)
                os.remove(f.filename)

            elif suffix == ".docx":  # for WORD file
                doc = docx.Document(f.filename)
                paras = doc.paragraphs
                text = ""
                for para in paras:
                    text += para.text

                cleaned_text = clean_and_process(text)
                # summarizer BERT model

                summary = BERT(cleaned_text)
                dump(summary)
                os.remove(f.filename)

            elif suffix == ".txt":  # for TXT file

                with open(f.filename, 'r') as file:
                    text = file.read().replace('\n', '')

                cleaned_text = clean_and_process(text)
                # summarizer BERT model

                summary = BERT(cleaned_text)
                dump(summary)
                os.remove(f.filename)

            else:
                return "
 Error! Please upload  PDF/WORD/TXT file
"

    return render_template('PDF_result.html', summary=summary)

@app.route('/RAW', methods=['GET', 'POST'])
def RAW():
    ################ for removing the previous wordcloud img on every request,so server dont gets loaded#############
    wordcloud = pathlib.Path("static/img/wordcloud/wordcloud.png")
    if wordcloud.is_file():
        os.remove(wordcloud)

    file = pathlib.Path("static/download/file.docx")
    if file.is_file():
        os.remove(file)

    return render_template('RAW.html')

@app.route('/RAW_result', methods=['GET', 'POST'])
def RAW_result():
    ################ for removing the previous wordcloud img on every request,so server dont gets loaded#############
    wordcloud = pathlib.Path("static/img/wordcloud/wordcloud.png")
    if wordcloud.is_file():
        os.remove(wordcloud)

    if request.method == 'POST':

        url = request.form.get('name')  # get the URL from the user

        if url_validator(url) is True:  # url validation:
            text = fulltext(requests.get(url).text)
            cleaned_text = clean_and_process(text)
            # summarizer BERT model

            summary = BERT(cleaned_text)
            dump(summary)

        else:
            return "
Error! Please upload a correct URL
"

    return render_template('RAW_result.html', summary=summary)

@app.route('/download', methods=['GET', 'POST'])
def download():
    file = pathlib.Path("static/download/file.docx")
    if file.is_file():

        return send_file(file, as_attachment=True)

#-------------------Routing ends----------------------------#

if __name__ == "__main__":
    app.run(host='0.0.0.0',port=8080,debug=True)