#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Aplicações distribuídas - Projeto 1 - lock_pool.py
Grupo: 28
Números de aluno: 49990, 50057, 50039
"""

# Zona para fazer imports
import sys, time
from sock_utils import *

# Programa principal
class resource_lock:
    def __init__(self):
        """
        Define e inicializa as características de um LOCK num recurso.
        """
        self.blocked = False
        self.inactive = False
        self.block_count = 0 #quantas vezes já foi bloqueado
        self.client_id = 0   #id do cliente
        self.time_limit = 0  #tempo
        self.start_time = 0

    def lock(self, client_id, time_limit):
        """
        Bloqueia o recurso se este não estiver bloqueado ou inativo, ou mantém o bloqueio
        se o recurso estiver bloqueado pelo cliente client_id. Neste caso renova
        o bloqueio do recurso até time_limit.
        Retorna True se bloqueou o recurso ou False caso contrário.
        """
        resource = False
        if not self.blocked and not self.inactive: # se n estiver bloqueado ou inativo
            self.blocked = True
            self.client_id = client_id
            self.time_limit = time_limit
            self.start_time = time.time()
            self.block_count +=1
            resource = True

        elif self.client_id == client_id and not self.inactive:
            self.time_limit = time_limit
            self.start_time = time.time()
            resource = True

        else:
            resource = False

        return resource

    def urelease(self):
        """
        Liberta o recurso incondicionalmente, alterando os valores associados
        ao bloqueio.
        """
        self.blocked = False # desbloqueia o recurso

    def release(self, client_id):
        """
        Liberta o recurso se este foi bloqueado pelo cliente client_id,
        retornando True nesse caso. Caso contrário retorna False.
        """
        resource = False

        if self.client_id == client_id and self.inactive == False: #verifica se é o client_id
            resource = True
            self.blocked = False

        return resource

    def test(self):
        """
        Retorna o estado de bloqueio do recurso ou inativo, caso o recurso se
        encontre inativo.
        """
        state = True
        if self.blocked:
            state = False
        elif self.inactive:
            state = "disable"

        return state

    def stat(self):
        """
        Retorna o número de vezes que este recurso já foi bloqueado em k.
        """
        return self.block_count

    def disable(self):
        """
        Coloca o recurso inativo/indisponível incondicionalmente, alterando os
        valores associados à sua disponibilidade.
        """
        self.blocked = False
        self.inactive = True

        #Perguntar!!!!!!!!!!!!!!!!!!!!!!!

###############################################################################

class lock_pool:
    def __init__(self, N, K, Y, T):
        """
        Define um array com um conjunto de locks para N recursos. Os locks podem
        ser manipulados pelos métodos desta classe.
        Define K, o número máximo de bloqueios permitidos para cada recurso. Ao
        atingir K, o recurso fica indisponível/inativo.
        Define Y, o número máximo permitido de recursos bloqueados num dado
        momento. Ao atingir Y, não é possível realizar mais bloqueios até que um
        recurso seja libertado.
		Define T, o tempo máximo de concessão de bloqueio.
        """
        self.lock_list = []
        self.K = K
        self.Y = Y
        self.T = T
        for i in range(N):
            self.lock_list.append(resource_lock())   #adiciona à lista o objeto

    def clear_expired_locks(self):
        """
        Verifica se os recursos que estão bloqueados ainda estão dentro do tempo
        de concessão do bloqueio. Liberta os recursos caso o seu tempo de
        concessão tenha expirado.
        """
        for recurso in self.lock_list:
            if time.time() - recurso.start_time > recurso.time_limit:
                recurso.urelease()

    def lock(self, resource_id, client_id, time_limit):
        """
        Tenta bloquear o recurso resource_id pelo cliente client_id, até ao
        instante time_limit.
        O bloqueio do recurso só é possível se o recurso estiver ativo, não
        bloqueado ou bloqueado para o próprio requerente, e Y ainda não foi
        excedido. É aconselhável implementar um método __try_lock__ para
        verificar estas condições.
        Retorna True em caso de sucesso e False caso contrário.
        """
        resp = False
        if self.stat_y() >= self.Y:
            resp = False
        else:
            resp = self.lock_list[resource_id].lock(client_id, self.T)

        return resp

    def release(self, resource_id, client_id):
        """
        Liberta o bloqueio sobre o recurso resource_id pelo cliente client_id.
        True em caso de sucesso e False caso contrário.
        """
        return self.lock_list[resource_id].release(client_id)  #liberta o recurso

    def test(self,resource_id):
        """
        Retorna True se o recurso resource_id estiver desbloqueado e False caso
        esteja bloqueado ou inativo.
        """
        return self.lock_list[resource_id].test() #checa o estado do recurso

    def stat(self,resource_id):
        """
        Retorna o número de vezes que o recurso resource_id já foi bloqueado, dos
        K bloqueios permitidos.
        """
        return self.lock_list[resource_id].stat()

    def stat_y(self):
        """
        Retorna o número de recursos bloqueados num dado momento do Y permitidos.
        """
        recursos_bloqueados = 0

        for id_recurso in range(len(self.lock_list)):
            if self.test(id_recurso) == False :
                recursos_bloqueados = recursos_bloqueados + 1
        return recursos_bloqueados

    def stat_n(self):
        """
        Retorna o número de recursos disponíneis em N.
        """
        recursos_disponiveis = 0
        for id_recurso in range(len(self.lock_list)):
            if self.test(id_recurso) == True:
                recursos_disponiveis = recursos_disponiveis + 1
        return recursos_disponiveis

    def disable(self):
        for id_recurso in range(len(self.lock_list)):
            if self.stat(id_recurso) >=  self.K:
                self.lock_list[id_recurso].disable()
            else:
                pass

    def __repr__(self):
        """
        Representação da classe para a saída standard. A string devolvida por
        esta função é usada, por exemplo, se uma instância da classe for
        passada à função print.
        """
        output = ""
        for id_recurso in range(len(self.lock_list)):
            recurso = self.lock_list[id_recurso]
            if self.test(id_recurso) == False:
                output += "recurso " + str(id_recurso) + " bloqueado pelo cliente " + str(recurso.client_id) + " ate " + time.ctime(recurso.start_time + recurso.time_limit) + "\n"
            elif self.test(id_recurso) == "disable":
                output += "recurso " + str(id_recurso) + " inativo\n"
            else:
                output += "recurso " + str(id_recurso) + " desbloqueado\n"

        #
        # Acrescentar na output uma linha por cada recurso bloqueado, da forma:
        # recurso <número do recurso> bloqueado pelo cliente <id do cliente> até
        # <instante limite da concessão do bloqueio>
        #
        # Caso o recurso não esteja bloqueado a linha é simplesmente da forma:
        # recurso <número do recurso> desbloqueado
        # Caso o recurso não esteja inativo a linha é simplesmente da forma:
        # recurso <número do recurso> inativo
        #
        return output


import random

START_RANDOM_RANGE = 1
END_RANDOM_RANGE = 100
START_RANDOM_STEP = 1
END_RANDOM_STEP = 10
PROGRESSION_LEN = 10
RULES = "What number is missing in the progression?"

def get_question():
    start_number = random.randint(START_RANDOM_RANGE, END_RANDOM_RANGE)
    step = random.randint(START_RANDOM_STEP, END_RANDOM_STEP)
    hidden_number = random.randint(0, PROGRESSION_LEN - 1)

    question = ""

    for i in range(0, PROGRESSION_LEN):
        if i == hidden_number:
            answer = str(start_number)
            question += ".. "
        else:
            question += str(start_number) + " "

        start_number = start_number + step

    return (question, answer)


import io
import concurrent.futures
import time

parsed_list = []
source_list = []

def process(word):
    reveresed_word = word[::-1]
    if reveresed_word == word:
        parsed_list.append(word)

if __name__ == "__main__":
    try:
     with io.open("words.txt",encoding="utf-8") as file:
        for word in file:
             word = word.replace("\n","").replace('"',"").replace(",","").replace("(","").replace(")","").replace(".","")
             if word != " " and word != "" and len(word) != 1:
               source_list.append(word)
    except FileNotFoundError:
        print("please run the writer.py file first")
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
     for idx,word in enumerate(source_list):
        if idx%2 == 0:
          executor.submit(process,word)
        else:
          executor.submit(process,word)
    print(parsed_list)


# C4.5 决策树实现及其剪枝
from math import log
import pandas as pd
import numpy as np

class Tree:
    def __init__(self, label, feature=None, feature_index=None):
        # 定义每个节点的属性
        self.label = label  # 标签值
        self.tree = {}  # 子树
        self.feature = feature  # 被分割的属性，叶子节点没有该属性
        self.feature_index = feature_index
        self.result = {'label:': self.label, 'feature': self.feature,'feature_index':self.feature_index, 'tree': self.tree}

    def __repr__(self):
        return '{}'.format(self.result)
    def add_node(self, val, node):
        self.tree[val] = node
    def predict(self,data):
        if self.feature_index == None:
            return self.label
        return self.tree[data[self.feature_index]].predict(data)

class C45:
    def __init__(self, eta=0):
        self.decision_tree = None
        self.eta = eta

    def entropy(self, dataframe):
        # 计算数据集的熵
        data_length = len(dataframe)
        mark_dict = {}
        for i in range(data_length):
            label = dataframe[i][-1]
            if label not in mark_dict:
                mark_dict[label] = 1
            else:
                mark_dict[label] += 1
        ent = -sum((p / data_length) * log(p / data_length, 2) for p in mark_dict.values())
        return ent

    def condition_entropy(self, dataframe, dim):
        # 1.根据维度划分数据集
        data_len = len(dataframe)
        dim_dict = {}
        for i in range(data_len):
            dim_feature = dataframe[i][dim]
            if dim_feature not in dim_dict:
                dim_dict[dim_feature] = []
            dim_dict[dim_feature].append(dataframe[i])
        cond_ent = sum(len(p) / data_len * self.entropy(p) for p in dim_dict.values())
        ent_dim = -sum(len(p) / data_len * log(len(p) / data_len, 2) for p in dim_dict.values())
        return cond_ent, ent_dim

    def rate_info_gain(self, dataframe, dim):
        ent_con, ent_dim = self.condition_entropy(dataframe, dim)
        info_gain = self.entropy(dataframe) - ent_con
        ent_rate = info_gain / ent_dim
        return ent_rate

    def best_dim(self, array):
        count = len(array[0]) - 1
        dim_info_gain = []
        for i in range(count):
            dim_info_gain.append((i, self.rate_info_gain(array, i)))
        info_gain = max(dim_info_gain, key=lambda x: x[1])
        return info_gain[0], info_gain[1]

    def train(self, dataframe):
        train_x, train_y, features = dataframe.iloc[:, :-1], dataframe.iloc[:, -1], dataframe.columns[:-1]
        max_label = train_y.value_counts().sort_values(ascending=False).index[0]
        if len(train_y.value_counts()) == 1:
            # 如果所有实例都属于同一类，则将y作为根节点的label
            return Tree(label=train_y.iloc[0])
        if len(features) == 0:
            # 如果特征为空集，则将数量最多的类别作为根节点的label
            return Tree(label=max_label)
        # 计算信息增益比最大的特征
        max_dim, info_gain = self.best_dim(np.array(dataframe))
        max_dim_name = features[max_dim]
        # 如果信息增益比小于阈值
        if info_gain < self.eta:
            return Tree(label=max_label)
        # 如果信息增益比大于阈值
        tree = Tree(label=max_label, feature=max_dim_name, feature_index=max_dim)
        this_dim_data = dataframe[max_dim_name].value_counts().index
        for d in this_dim_data:
            sub_dataframe = dataframe.loc[dataframe[max_dim_name] == d].drop(max_dim_name, axis=1)
            tree.add_node(d, self.train(sub_dataframe))
        return tree

    def fit(self, dataframe):
        self.decision_tree = self.train(dataframe)

    def predict(self,data):
        if self.decision_tree == None:
            print("Please fit")
            return
        else:
            return self.decision_tree.predict(data)
    def pruning(self):
        # 1、 计算每个节点的经验熵
        # 计算剪枝前和剪枝后的损失函数变化
        pass
if __name__ == '__main__':
    dataSet = [['青年', '否', '否', '一般', '拒绝'],
               ['青年', '否', '否', '好', '拒绝'],
               ['青年', '是', '否', '好', '同意'],
               ['青年', '是', '是', '一般', '同意'],
               ['青年', '否', '否', '一般', '拒绝'],
               ['中年', '否', '否', '一般', '拒绝'],
               ['中年', '否', '否', '好', '拒绝'],
               ['中年', '是', '是', '好', '同意'],
               ['中年', '否', '是', '非常好', '同意'],
               ['中年', '否', '是', '非常好', '同意'],
               ['老年', '否', '是', '非常好', '同意'],
               ['老年', '否', '是', '好', '同意'],
               ['老年', '是', '否', '好', '同意'],
               ['老年', '是', '否', '非常好', '同意'],
               ['老年', '否', '否', '一般', '拒绝'],
               ]
    featureName = ['年龄', '有工作', '有房子', '信贷情况', '类别']
    data_df = pd.DataFrame(dataSet, columns=featureName)
    alo = C45()
    alo.fit(data_df)
    # print(alo.predict(['青年', '否', '否', '一般']))
    print(alo.pruning())


import argparse
import os
import re
import gzip
from ChromBridGE import ChromBridGE_aln
from ChromBridGE import ChromBridGE_tx

def main():

    parser = argparse.ArgumentParser(description='ChromBridGE: Translocation detection in genome-edited reads.')
    parser.add_argument('-f','--fastq', help='Input fastq file', required=True)
    parser.add_argument('-a','--sequence_a', help='Input sequence a', required=True)
    parser.add_argument('-b','--sequence_b', help='Input sequence b', required=True)
    parser.add_argument('--seqA_cut_pos', type=int, help='Index in sequence a of predicted cut site',default=None)
    parser.add_argument('--seqB_cut_pos', type=int, help='Index in sequence b of predicted cut site',default=None)
    parser.add_argument('--match_score', help='Match score for alignment',default=3)
    parser.add_argument('--mismatch_score', help='Mismatch score for alignment',default=-1)
    parser.add_argument('--gap_score', help='Gap score for alignment',default=-2)
    parser.add_argument('--jump_score', help='Jump score for alignment',default=-3)
    parser.add_argument('--cut_pos_incentive_score', help='Incentive for jumping at a predicted cut site',default=1)
    parser.add_argument('-o','--output_file', help='Output file to write results',default=None)
    args = parser.parse_args()

    if not os.path.isfile(args.fastq):
        raise Exception('File ' + args.fastq + ' does not exist')

    output_file = args.output_file
    if output_file is None:
        root = args.fastq
        root = re.sub(".fq.gz$","",root)
        root = re.sub(".fq$","",root)
        root = re.sub(".fastq.gz$","",root)
        root = re.sub(".fastq$","",root)
        output_file = root+".ChromBridGE.fa"

    if args.fastq.endswith('.gz'):
        f_in = io.BufferedReader(gzip.open(args.fastq,'rt'))
    else:
        f_in = open(args.fastq,'rt')

    if output_file.endswith('.gz'):
        f_out = gzip.open(output_file, 'wt')
    else:
        f_out = open(output_file, 'wt')

    f_out.write("read_id\tbreakpoints\tbreakpoint_count\tbreakpoint_cumulative_distance_from_cut\tread_aln\trefA_aln\trefB_aln\n")

    total_read_count = 0
    while (1):
        id_line   = f_in.readline().strip()
        seq_line  = f_in.readline().strip()
        plus_line = f_in.readline()
        qual_line = f_in.readline().strip()

        if not qual_line : break

        total_read_count+=1
        print('total read count: ' + str(total_read_count))
        if total_read_count % 1000 == 0:
            print('total read count: ' + str(total_read_count))

        aln_info = ChromBridGE_aln.nw_breakpoint(seq_line,args.sequence_a,args.sequence_b,
                match_score=args.match_score,
                mismatch_score=args.mismatch_score,
                gap_score=args.gap_score,
                jump_score=args.jump_score,
                cut_pos_incentive_score=args.cut_pos_incentive_score,
                ref1_cut_pos=args.seqA_cut_pos,
                ref2_cut_pos=args.seqB_cut_pos)

        f_out.write(id_line+"\t"+breakpoints_out+"\t"+str(breakpoints_count)+"\t"+breakpoint_cumulative_distance+"\t"+read_aln+"\t"+refA_aln+"\t"+refB_aln+"\n")

def analyze_read(read_seq, ref1_seq, ref2_seq,
                    ref1_cut_pos=None,
                    ref2_cut_pos=None,
                    match_score=3,
                    mismatch_score=-1,
                    gap_score=-2,
                    perimeter_gap_extension_score=0,
                    jump_score=-12, # four matches
                    cut_pos_jump_incentive_score=1,
                    min_num_bases_beyond_cut=4,
                    min_num_bases_before_cut=4,
                    mismatch_tolerance=0,
                    gap_tolerance=0,
		            debug=False):
    """
    Computes the optimal alignment of a read to two sequences, locating the optimal break between the two reads, refines the site of translocation by identifying sites with at least num_bases_to_check exact matches, and determines whether the sequences are compatible with a translocation event.

    params:
        read_seq: read to align to the other two sequences
        ref1_seq: first sequence to align to
        ref2_seq: second sequence to align to

        ref1_cut_pos: position of predicted cut site in ref1
        ref2_cut_pos: position of predicted cut site in ref2

        min_num_bases_beyond_cut: Min number of matching bases that must be seen beyond the cut site for a breakpoint to be reported beyond the cut site
        min_num_bases_before_cut: Min number of matching bases that must be seen before the cut site - the position of this first match will be reported
        mismatch_tolerance: int How many mismatches to tolerate before returning false
        gap_tolerance: int How many gaps to tolerate before returning false

        match_score: score for adding a match in alignment (positive)
        mismatch_score: score for adding a mismatch in alignment
        gap_score: score for adding a gap in alignment
        perimeter_gap_extension_score: score for adding a gap in the first/last column/row, corresponding to gaps at the beginning or ends of sequences
        jump_score: score for jumping between ref1 and ref2
        cut_pos_jump_incentive_score: score incentive for jumping at a predicted cut position
        debug: print intermediate debug information

    returns:
        tuple of:
        aln_info: dict with keys:
            read_aln: aligned sequence of read
            ref1_aln: sequence of ref1 aligned to read
            ref2_aln: sequence of ref2 aligned to read
            breakpoints_read: indices in read of the breakpoints discovered
            breakpoints_ref1: indices in ref1 of breakpoints in the optimal alignment
            breakpoints_ref2: indices in ref2 of breakpoints in the optimal alignment
            aln_score: score of alignment
            read_path: index of ref that the read is aligned to, corresponding to the break points (there will be len(breakpoints)+1 items in read_path)

        tx_info: dict with keys:
            final_read_str: string of read alignment
            final_ref1_str: string of bases for which read aligns to ref1 - including '~' for trimmed insertions at translocation sites
            final_ref2_str: string of bases for which read aligns to ref2 - including '~' for trimmed insertions at translocation sites
            final_path: indices of the ref that the read is aligned to
            final_breakpoints_ref1: locations in the ref1 corresponding to translocations in ref1 (same length as final_breakpoints_read_ref1)
            final_breakpoints_ref2: locations in the ref2 corresponding to translocations in ref2 (same length as final_breakpoints_read_ref2)
            bp_match_ref1: number of bp in final alignment that match read and ref1 exactly
            bp_match_ref2: number of bp in final alignment that match read and ref2 exactly
            bp_insertion: number of bp that are inserted (match neither reference)

            is_tx: boolean for whether the read looks like a translocation
            tx_status: string with details for tx result
            left_distance: int, number of bp the read extends beyond the cut to the right (from the left-identified reference)
            right_distance: int, number of bp the read extends beyond the cut to the left (from the right-identified reference)
                Read: AAAA|TTTT
                Ref:  AAAATT  > left-distance is 2 because it extended beyond the cut by 2bp
                Read: AAAA|TTTT
                Ref:  AA      > left-distance is -2 because it was within the cut by 2bp
            tx_lucky_insertions: sum of the left- and right- distances if they extend beyond the cut. If the cut actually happened, these would be lucky insertions that happened to match the uncut reference sequence
    """
    aln_info = ChromBridGE_aln.nw_breakpoint(
                read_seq,
                ref1_seq,
                ref2_seq,
                match_score=match_score,
                mismatch_score=mismatch_score,
                gap_score=gap_score,
                perimeter_gap_extension_score=perimeter_gap_extension_score,
                jump_score=jump_score,
                cut_pos_jump_incentive_score=cut_pos_jump_incentive_score,
                ref1_cut_pos = ref1_cut_pos,
                ref2_cut_pos = ref2_cut_pos,
                debug=debug
    )

    tx_info = ChromBridGE_tx.analyze_tx_alignment(
            read_aln_str=aln_info['read_aln'],
            ref1_aln_str=aln_info['ref1_aln'],
            ref2_aln_str=aln_info['ref2_aln'],
            breakpoints_read=aln_info['breakpoints_read'],
            breakpoints_ref1=aln_info['breakpoints_ref1'],
            breakpoints_ref2=aln_info['breakpoints_ref2'],
            read_path=aln_info['read_path'],
            ref1_cut_pos=ref1_cut_pos,
            ref2_cut_pos=ref2_cut_pos,
            min_num_bases_beyond_cut=min_num_bases_beyond_cut,
            min_num_bases_before_cut=min_num_bases_before_cut,
            mismatch_tolerance=mismatch_tolerance,
            gap_tolerance=gap_tolerance
    )

    return aln_info, tx_info

if __name__ == "__main__":
    main()


import os
cwd = os.getcwd()
LOCAL_PATH = cwd + '/rc/'
RESOURCE_PATH = '/shared-vol/'
NMAP_REPORT_PATH =  cwd + '/nmap_report/'
REPORT_PATH = cwd + '/exploit_report/'

# setting dict for global setting
settings = {'TTL(s)':'30','LHOSTS':'172.17.0.2'}
# open port for exploit searching
open_port = []
exploit_and_port = {}
exploit_list = []


import twint

c = twint.Config()

c.Search = "@Apple"
c.Verified = True
c.Lang = "en"
c.Store_csv = True
c.Output = "AppleTweets.csv"
c.Since = "2019-01-01"
c.Until = "2019-01-11"
c.Custom["tweet"] = ["username", "date", "time", "tweet"]

twint.run.Search(c)


import api_requester

# 요청종류 세팅
authkey_path = 'security/authkey.json'
base_url_path = 'security/url.json'
authkey_type = 'char_WFS'

# 인풋, 요청 키 값 세팅(변수 이름이나 내부 내용은 자유롭게 변경해도 됩니다.)
input_dict = {
    # 'bbox': '217365,447511,217636,447701,EPSG:5174',
    'pnu': '1114010500',
    'maxFeatures': '100',
    'resultType': 'results',
    'srsName': 'EPSG:4326'
}
input_dict2 = {
    'maxFeatures': '10',
    'resultType': 'results',
    'srsName': 'EPSG:5174'
}
pnu_list = ['1111017400105950132', '4145011800100320003'] # 리스트 인풋이 필요한 경우
keys_request = ["NSDI:PNU",
                "gml:posList",
                "NSDI:LNM_LNDCGR_SMBOL",
                "NSDI:PBLNTF_PCLND"]

# ApiRequester 생성 - 수정 X
get_api = api_requester.ApiRequester(authkey_path, authkey_type, base_url_path)

# 인풋 넣기 (여러번 넣어도 됨.)
# get_api.setInput(input_dict) # 단순 인풋
get_api.setInput(input_dict) # 단순 인풋

# 아웃풋 파일 이름 설정
get_api.setOutputName('./output', 'junggu_sample')

# ApiRequester 작동
# get_api.getOutputs() # 요청 키가 없는 경우 로우 파일이 생성됨.
get_api.getOutputs(keys_request) # 요청 키가 있는 경우 해당 키에 대한 정보만 받아옴.


import itertools
import os

import torch

from fairseq.fairseq import utils
from fairseq.fairseq.data import (
    ConcatDataset,
    Dictionary,
    indexed_dataset,
    ExplanationPairDataset,
    ExplanationPureWeightDataset
)
from fairseq.fairseq.tasks import FairseqTask, register_task

CATEGORICAL_TGT_PADDING = -100
CONTINUOUS_TGT_PADDING = 0.0
MEMORY = 'mem'
SIM = 'sim'
RANDOM = 'random'

def load_explpair_dataset(
        data_path, split,
        src_prefix, src_suffix, src_dict, tgt_prefix, tgt_suffix, tgt_dict,
        combine, dataset_impl, upsample_primary,
        left_pad_source, left_pad_target, max_source_positions, remove_eos_from_source, tgt_padding):

    def split_exists(split, prefix, suffix, data_path):
        filename = os.path.join(data_path, '{}{}{}'.format(prefix, split, suffix))   # e.g., train_src.exp
        return indexed_dataset.dataset_exists(filename, impl=dataset_impl)

    src_datasets = []
    tgt_datasets = []

    for k in itertools.count():
        split_k = split + (str(k) if k > 0 else '')

        if split_exists(split_k, src_prefix, src_suffix, data_path) and \
                split_exists(split_k, tgt_prefix, tgt_suffix, data_path):
            prefix = os.path.join(data_path, '{}{}{}'.format(src_prefix, split_k, src_suffix))

            src = indexed_dataset.make_dataset(prefix, impl=dataset_impl, dictionary=src_dict,
                                               append_eos=not remove_eos_from_source)
            src_datasets.append(src)

            prefix = os.path.join(data_path, '{}{}{}'.format(tgt_prefix, split_k, tgt_suffix))
            tgt_datasets.append(ExplanationPureWeightDataset(prefix, tgt_padding,
                                                             append_eos=not remove_eos_from_source))
        elif k > 0:
            break
        else:
            raise FileNotFoundError('Dataset not found: {} ({})'.format(split, data_path))

        print('| {} {} {}-{} {} examples'.format(data_path, split_k,
                                                 '{}{}{}'.format(src_prefix, split, src_suffix),
                                                 '{}{}{}'.format(tgt_prefix, split, tgt_suffix), len(src_datasets[-1])))

        if not combine:
            break
    assert len(src_datasets) == len(tgt_datasets)

    if len(src_datasets) == 1:
        # if we need to upsample the src to generate more versions of masking neutral weights in calculating loss
        src_dataset, tgt_dataset = src_datasets[0], tgt_datasets[0]
    else:
        sample_ratios = [1] * len(src_datasets)
        sample_ratios[0] = upsample_primary
        src_dataset = ConcatDataset(src_datasets, sample_ratios)
        tgt_dataset = ConcatDataset(tgt_datasets, sample_ratios)

    return ExplanationPairDataset(
        src_dataset, src_dataset.sizes, src_dict,
        tgt_dataset, tgt_dict=tgt_dict, left_pad_source=left_pad_source,
        left_pad_target=left_pad_target, max_source_positions=max_source_positions,
        remove_eos_from_source=remove_eos_from_source
    )

def load_sim_encoder(device_id, encoder_path):
    # load bert for similarity measure
    from transformers import BertModel, BertTokenizer
    # vectorize src according to BERT pre-training models
    pretrained_bert = BertModel.from_pretrained(encoder_path)
    bert_tokenizer = BertTokenizer.from_pretrained(encoder_path)
    if torch.cuda.is_available():
        pretrained_bert = pretrained_bert.to(device_id)
    return bert_tokenizer, pretrained_bert

@register_task('explanation')
class ExplanationTask(FairseqTask):
    """
    Generate the weight of each word from the document to explain their importance in the document classification task.

    Args:
        dictionary (~fairseq.data.Dictionary): the dictionary for the input of
            the language model
        output_dictionary (~fairseq.data.Dictionary): the dictionary for the
            output of the language model. In most cases it will be the same as
            *dictionary*, but could possibly be a more limited version of the
            dictionary (if ``--output-dictionary-size`` is used).
        targets (List[str]): list of the target types that the language model
            should predict.  Can be one of "self", "future", and "past".
            Defaults to "future".

    .. note::

        The language modeling task is compatible with :mod:`fairseq-train`,
        :mod:`fairseq-generate`, :mod:`fairseq-interactive` and
        :mod:`fairseq-eval-lm`.
    """

    @staticmethod
    def add_args(parser):
        """Add task-specific arguments to the parser."""
        # fmt: off
        parser.add_argument('data', help='path to data directory')

        parser.add_argument('--tokens-per-sample', default=1024, type=int,
                            help='max number of tokens per sample for dataset')
        parser.add_argument('--lazy-load', action='store_true',
                            help='load the dataset lazily')
        parser.add_argument('--raw-text', default=False, action='store_true',
                            help='load raw text dataset')

        parser.add_argument('--add-bos-token', action='store_true',
                            help='prepend beginning of sentence token (<s>)')

        # copy from translation
        parser.add_argument('--upsample-primary', default=1, type=int,
                            help='amount to upsample primary dataset')
        parser.add_argument('--left-pad-source', default=False, action='store_true',
                            help='pad the source on the left')
        parser.add_argument('--left-pad-target', default=False, action='store_true',
                            help='pad the target on the left')
        parser.add_argument('--max-source-positions', default=1024, type=int, metavar='N',
                            help='max number of tokens in the source sequence')

        parser.add_argument('--remove-eos-from-source', default=False, action='store_true',
                            help='remove the eos from source sentence, default is False')

        parser.add_argument('--input-prefix', default='', help='the prefix for the input file')
        parser.add_argument('--output-prefix', default='', help='the prefix for the output file')

        parser.add_argument('--input-suffix', default='_src.exp', help='the suffix for the input file')
        parser.add_argument('--output-suffix', default='_tgt.exp', help='the suffix for the output file')

        # Added by Snow 17 Mar 2021
        parser.add_argument('--dict-file', type=str, default="dict.txt", help='the vocabulary file')
        # Added by Snow 17 Mar 2021
        # fmt: on

    def __init__(self, args, dictionary):
        super().__init__(args)
        self.dictionary = dictionary
        from copy import deepcopy
        out_dict = deepcopy(dictionary)
        self.tgt_padding = CATEGORICAL_TGT_PADDING \
            if args.criterion != 'explanation_mse_loss' else CONTINUOUS_TGT_PADDING
        out_dict.pad_index = self.tgt_padding
        self.output_dictionary = out_dict

    @classmethod
    def setup_task(cls, args, **kwargs):
        """Setup the task (e.g., load dictionaries).

        Args:
            args (argparse.Namespace): parsed command-line arguments
        """
        if getattr(args, 'raw_text', False):
            utils.deprecation_warning('--raw-text is deprecated, please use --dataset-impl=raw')
            args.dataset_impl = 'raw'
        elif getattr(args, 'lazy_load', False):
            utils.deprecation_warning('--lazy-load is deprecated, please use --dataset-impl=lazy')
            args.dataset_impl = 'lazy'

        dictionary = None
        # output_dictionary = None
        if args.data:
            paths = args.data.split(':')
            assert len(paths) > 0
            dictionary = Dictionary.load(os.path.join(paths[0], args.dict_file))
            # print('| dictionary: {} types'.format(len(dictionary)))

        return cls(args, dictionary)

    def load_dataset(self, split, epoch=0, combine=False, **kwargs):
        """Load a given dataset split.

        Args:
            split (str): name of the split (e.g., train, valid, test)
        """

        paths = self.args.data.split(':')
        assert len(paths) > 0
        data_path = paths[epoch % len(paths)]

        self.datasets[split] = load_explpair_dataset(
            data_path, split, self.args.input_prefix, self.args.input_suffix, self.source_dictionary,
            self.args.output_prefix, self.args.output_suffix, self.output_dictionary,
            combine=combine, dataset_impl=self.args.dataset_impl,
            upsample_primary=self.args.upsample_primary, left_pad_source=self.args.left_pad_source,
            left_pad_target=self.args.left_pad_target, max_source_positions=self.args.max_source_positions,
            remove_eos_from_source=self.args.remove_eos_from_source, tgt_padding=self.tgt_padding)

    @property
    def source_dictionary(self):
        """Return the :class:`~fairseq.data.Dictionary` for the language
        model."""
        return self.dictionary

    @property
    def target_dictionary(self):
        """Return the :class:`~fairseq.data.Dictionary` for the language
        model."""
        return self.output_dictionary


'''
Test Vid4 (SR) and REDS4 (SR-clean, SR-blur, deblur-clean, deblur-compression) datasets
'''
import argparse
import os
import os.path as osp
import glob
import logging
import numpy as np
import torch
import math
import data.util as data_util
import utils.util as util

from models.archs import LRimg_estimator as LRest

import imageio

def main():
    #################
    # configurations
    #################
    device = torch.device('cuda')

    prog = argparse.ArgumentParser()
    prog.add_argument('--dataset_mode', '-m', type=str, default='Vid4+REDS', help='data_mode')
    prog.add_argument('--degradation_mode', '-d', type=str, default='bicubic', choices=('impulse', 'bicubic', 'preset'), help='path to image output directory.')
    prog.add_argument('--model', type=str, default='SFDN', choices=('SFDN', 'MFDN'))
    prog.add_argument('--sigma_x', '-sx', type=float, default=1, help='sigma_x')
    prog.add_argument('--sigma_y', '-sy', type=float, default=0, help='sigma_y')
    prog.add_argument('--theta', '-t', type=float, default=0, help='theta')
    prog.add_argument('--scale', '-sc', type=int, default=2, choices=(2, 4), help='scale factor')

    args = prog.parse_args()

    data_modes = args.dataset_mode
    degradation_mode = args.degradation_mode  # impulse | bicubic
    sig_x, sig_y, the = args.sigma_x, args.sigma_y, args.theta * math.pi / 180
    if sig_y == 0:
        sig_y = sig_x

    scale = args.scale
    kernel_size = 21
    N_in = 5

    data_mode_l = data_modes.split('+')

    for i in range(len(data_mode_l)):
        data_mode = data_mode_l[i]

        #### dataset
        if data_mode == 'Vid4':
            load_model = 'Vimeo'
            kernel_folder = '../experiments/pretrained_models/Vid4Gauss.npy'
            dataset_folder = '../dataset/Vid4'
        elif data_mode == 'REDS':
            load_model = 'REDS'
            kernel_folder = '../experiments/pretrained_models/REDSGauss.npy'
            dataset_folder = '../dataset/REDS/train'
        else:
            raise NotImplementedError('Only Vid4 and REDS are supporteds')

        root_folder = osp.join(dataset_folder, 'LR_'+ degradation_mode + '_' + str('{:.1f}'.format(sig_x)) + '_' + str('{:.1f}'.format(sig_y)) + '_' + str('{:.1f}'.format(args.theta)))
        LR_dataset_folder = osp.join(root_folder, 'X'+str(scale))
        save_folder = osp.join(root_folder, 'X'+str(scale*scale)+'_'+args.model)
        print(root_folder)
        print(save_folder)

        if not osp.exists(save_folder):
            os.makedirs(save_folder)

        subfolder_name_l = []
        subfolder_LR_l = sorted(glob.glob(osp.join(LR_dataset_folder, '*')))
        if data_mode == 'REDS':
            subfolder_LR_l = [k for k in subfolder_LR_l if
                              k.find('000') >= 0 or k.find('011') >= 0 or k.find('015') >= 0 or k.find('020') >= 0]
        # for each subfolder
        # for subfolder, subfolder_GT in zip(subfolder_l, subfolder_GT_l):
        sig_x, sig_y, the = float(sig_x), float(sig_y), float(the)

        if args.model == 'SFDN':
            if scale == 2:
                model = LRest.DirectKernelEstimator_CMS(nf=64)
                model.load_state_dict(torch.load('../pretrained_models/MFDN/SFDN_{}.pth'.format(load_model)), strict=True)
            else:
                raise NotImplementedError('We do not support SFDN for scale factor 4 now.')

        else:
            model = LRest.DirectKernelEstimatorVideo(in_nc=3, nf=64, scale=scale)
            if scale == 2:
                model.load_state_dict(torch.load('../pretrained_models/MFDN/MFDN_{}.pth'.format(load_model)), strict=True)
            else:
                model.load_state_dict(torch.load('../pretrained_models/MFDN/MFDN_{}_S4.pth'.format(load_model)), strict=True)

        model.eval()
        model = model.to(device)

        for subfolder_LR in subfolder_LR_l:

            subfolder_name = osp.basename(subfolder_LR)
            subfolder_name_l.append(subfolder_name)
            save_subfolder = osp.join(save_folder, subfolder_name)
            if not osp.exists(save_subfolder):
                os.mkdir(save_subfolder)

            img_LR_path_l = sorted(glob.glob(osp.join(subfolder_LR, '*')))
            max_idx = len(img_LR_path_l)

            #### read LQ and GT images
            imgs_LR = data_util.read_img_seq(subfolder_LR)  # T C H W
            if args.model == 'SFDN':
                count = 0
                imgs_LR_l = imgs_LR.split(32)
                for img_batch in imgs_LR_l:
                    img_batch = img_batch.to(device)
                    with torch.no_grad():
                        img_lr_batch = model(img_batch)
                        img_lr_batch = img_lr_batch.permute(0,2,3,1).cpu().numpy()
                        img_lr_batch = (img_lr_batch.clip(0, 1)*255).round()
                        img_lr_batch = img_lr_batch.astype('uint8')
                        for img_lr in img_lr_batch:
                            filename = osp.basename(img_LR_path_l[count])
                            imageio.imwrite(osp.join(save_subfolder, filename), img_lr)
                            count += 1

            else:
                 for img_idx, img_path in enumerate(img_LR_path_l):
                    img_name = osp.splitext(osp.basename(img_path))[0]
                    select_idx = data_util.index_generation(img_idx, max_idx, N_in, padding='new_info')
                    imgs_in = imgs_LR.index_select(0, torch.LongTensor(select_idx)).unsqueeze(0).to(device)
                    imgs_in = imgs_in.transpose(1,2)
                    with torch.no_grad():
                        output = model(imgs_in)  # B C T H W
                        output = output.squeeze(0)
                        output = output[:, N_in // 2]
                        output = output.permute(1,2,0).cpu().numpy()
                        output = (output.clip(0, 1)*255).round().astype('uint8')
                        imageio.imwrite(osp.join(save_subfolder, '{}.png'.format(img_name)), output)

if __name__ == '__main__':
    main()


import os
import sys
import json
import subprocess
import random

import torch
from torch import nn
from torch.nn import BCELoss, CrossEntropyLoss
from torch.backends import cudnn
from torch.optim import SGD, lr_scheduler
import torch.multiprocessing as mp
import torch.distributed as dist
from torchvision.transforms import transforms

import pandas as pd
import numpy as np
from opts import parse_opts
from model import generate_model, make_data_parallel, load_pretrained_model, resume_model, get_fine_tuning_parameters
from dataset import VideoFrameDataset
from utils import Logger, worker_init_fn, get_lr
from training import train_epoch
from validation import val_epoch
# from mean import get_mean
from spatial_transforms import (Compose, Normalize, Resize, CenterCrop,
                                CornerCrop, MultiScaleCornerCrop,
                                RandomResizedCrop, RandomHorizontalFlip,
                                ToTensor, ScaleValue, ColorJitter,
                                PickFirstChannels)
from temporal_transforms import (LoopPadding, TemporalRandomCrop,
                                 TemporalCenterCrop, TemporalEvenCrop,
                                 SlidingWindow, TemporalSubsampling)

def get_train_utils(opt, model_parameters):
    assert opt.train_crop in ['random', 'corner', 'center']
    spatial_transform = []
    if opt.train_crop == 'random':
        spatial_transform.append(
            RandomResizedCrop(
                opt.sample_size, (opt.train_crop_min_scale, 1.0),
                (opt.train_crop_min_ratio, 1.0 / opt.train_crop_min_ratio)))
    elif opt.train_crop == 'corner':
        scales = [1.0]
        scale_step = 1 / (2**(1 / 4))
        for _ in range(1, 5):
            scales.append(scales[-1] * scale_step)
        spatial_transform.append(MultiScaleCornerCrop(opt.sample_size, scales))
    elif opt.train_crop == 'center':
        spatial_transform.append(Resize(opt.sample_size))
        spatial_transform.append(CenterCrop(opt.sample_size))
    # normalize = get_normalize_method(opt.mean, opt.std, opt.no_mean_norm,
    #                                  opt.no_std_norm)
    if not opt.no_hflip:
        spatial_transform.append(RandomHorizontalFlip())
    if opt.colorjitter:
        spatial_transform.append(ColorJitter())
    spatial_transform.append(transforms.ToTensor())
    if opt.input_type == 'flow':
        spatial_transform.append(PickFirstChannels(n=2))
    # spatial_transform.append(ScaleValue(opt.value_scale))
    # spatial_transform.append(normalize)
    spatial_transform = Compose(spatial_transform)

    # assert opt.train_t_crop in ['random', 'center']
    # temporal_transform = []
    # if opt.sample_t_stride > 1:
    #     temporal_transform.append(TemporalSubsampling(opt.sample_t_stride))
    # if opt.train_t_crop == 'random':
    #     temporal_transform.append(TemporalRandomCrop(opt.sample_duration))
    # elif opt.train_t_crop == 'center':
    #     temporal_transform.append(TemporalCenterCrop(opt.sample_duration))
    # temporal_transform = TemporalCompose(temporal_transform)

    train_data = VideoFrameDataset(
        root_path=opt.root_path,
        annotationfile_path=opt.annotation_path,
        num_segments=1,
        frames_per_segment=opt.t_dim_frames,
        imagefile_template=opt.imagefile_template,
        transform=spatial_transform,
        random_shift=True,
        test_mode=False
        )

    train_sampler = None
    train_loader = torch.utils.data.DataLoader(train_data,
                                               batch_size=opt.batch_size,
                                               shuffle=(train_sampler is None),
                                               num_workers=opt.n_threads,
                                               pin_memory=True,
                                               sampler=train_sampler,
                                               worker_init_fn=worker_init_fn
                                               )

    if opt.is_master_node:
        train_logger = Logger(opt.result_path / 'train.log',
                              ['epoch', 'loss', 'acc', 'lr'])
        train_batch_logger = Logger(
            opt.result_path / 'train_batch.log',
            ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
    else:
        train_logger = None
        train_batch_logger = None

    if opt.nesterov:
        dampening = 0
    else:
        dampening = opt.dampening
    optimizer = SGD(model_parameters,
                    lr=opt.learning_rate,
                    momentum=opt.momentum,
                    dampening=dampening,
                    weight_decay=opt.weight_decay,
                    nesterov=opt.nesterov)

    assert opt.lr_scheduler in ['plateau', 'multistep']
    assert not (opt.lr_scheduler == 'plateau' and opt.no_val)
    if opt.lr_scheduler == 'plateau':
        scheduler = lr_scheduler.ReduceLROnPlateau(
            optimizer, 'min', patience=opt.plateau_patience)
    else:
        scheduler = lr_scheduler.MultiStepLR(optimizer,
                                             opt.multistep_milestones)

    return (train_loader, train_sampler, train_logger, train_batch_logger,
            optimizer, scheduler)

def get_val_utils(opt):
    spatial_transform = [
        transforms.Resize(opt.sample_size),
        CenterCrop(opt.sample_size),
        transforms.ToTensor()
    ]
    if opt.input_type == 'flow':
        spatial_transform.append(PickFirstChannels(n=2))
    #spatial_transform.extend([ScaleValue(opt.value_scale), normalize])
    spatial_transform = Compose(spatial_transform)
    val_data = VideoFrameDataset(
        root_path=opt.root_path,
        annotationfile_path=opt.val_annotation_path,
        num_segments=1,
        frames_per_segment=opt.t_dim_frames,
        imagefile_template=opt.imagefile_template,
        transform=spatial_transform,
        random_shift=False,
        test_mode=False
        )

    val_sampler = None
    val_loader = torch.utils.data.DataLoader(val_data,
                                             batch_size=(opt.batch_size //
                                                         opt.n_val_samples),
                                             shuffle=False,
                                             num_workers=opt.n_threads,
                                             pin_memory=True,
                                             sampler=val_sampler,
                                             worker_init_fn=worker_init_fn
                                             )

    if opt.is_master_node:
        val_logger = Logger(opt.result_path / 'val.log',
                            ['epoch', 'loss', 'acc'])
    else:
        val_logger = None

    return val_loader, val_logger

#------ Check point save ------------#
def save_checkpoint(save_file_path, epoch, arch, model, optimizer, scheduler):
    if hasattr(model, 'module'):
        model_state_dict = model.module.state_dict()
    else:
        model_state_dict = model.state_dict()
    save_states = {
        'epoch': epoch,
        'arch': arch,
        'state_dict': model_state_dict,
        'optimizer': optimizer.state_dict(),
        'scheduler': scheduler.state_dict()
    }
    torch.save(save_states, save_file_path)

if __name__=="__main__":
    opt = parse_opts()
    random.seed(opt.manual_seed)
    np.random.seed(opt.manual_seed)
    torch.manual_seed(opt.manual_seed)
    opt.imagefile_template='{:010d}.jpg'
    #opt.mean = get_mean()
    opt.sample_size = (320,240)
    opt.t_dim_frames = 200
    opt.arch = '{}-{}'.format(opt.model_name, opt.model_depth)
    # opt.n_classes = 400
    opt.device = torch.device('cpu' if opt.no_cuda else 'cuda')
    if not opt.no_cuda:
        cudnn.benchmark = True
    model = generate_model(opt)
    # if opt.resume_path is not None:
    #     model = resume_model(opt.resume_path, opt.arch, model)
    model = make_data_parallel(model, opt.device)
    if not opt.verbose:
        print(model)

    if opt.n_classes > 1:
        criterion = CrossEntropyLoss().to(opt.device)
    else:
        criterion = BCELoss().to(opt.device)

    parameters = model.parameters()
    if not opt.no_train:
        (train_loader, train_sampler, train_logger, train_batch_logger,
         optimizer, scheduler) = get_train_utils(opt, parameters)
        # if opt.resume_path is not None:
        #     opt.begin_epoch, optimizer, scheduler = resume_train_utils(
        #         opt.resume_path, opt.begin_epoch, optimizer, scheduler)
        #     if opt.overwrite_milestones:
        #         scheduler.milestones = opt.multistep_milestones
    if not opt.no_val:
        val_loader, val_logger = get_val_utils(opt)

    if opt.tensorboard and opt.is_master_node:
        from torch.utils.tensorboard import SummaryWriter
        if opt.begin_epoch == 1:
            tb_writer = SummaryWriter(log_dir=opt.result_path)
        else:
            tb_writer = SummaryWriter(log_dir=opt.result_path,
                                      purge_step=opt.begin_epoch)
    else:
        tb_writer = None
    prev_val_loss = None
    for i in range(opt.begin_epoch, opt.n_epochs + 1):
        if not opt.no_train:
            current_lr = get_lr(optimizer)
            train_epoch(i, train_loader, model, criterion, optimizer,
                        opt.device, current_lr, train_logger,
                        train_batch_logger, tb_writer)

            if i % opt.checkpoint == 0 and opt.is_master_node:
                save_file_path = opt.result_path / 'save_{}.pth'.format(i)
                save_checkpoint(save_file_path, i, opt.arch, model, optimizer,
                                scheduler)

        if not opt.no_val:
            prev_val_loss = val_epoch(i, val_loader, model, criterion,
                                      opt.device, val_logger, tb_writer,)

        if not opt.no_train and opt.lr_scheduler == 'multistep':
            scheduler.step()
        elif not opt.no_train and opt.lr_scheduler == 'plateau':
            scheduler.step(prev_val_loss)

    # class_names = []
    # with open('class_names_list') as f:
    #     for row in f:
    #         class_names.append(row[:-1])

    # ffmpeg_loglevel = 'quiet'
    # if opt.verbose:
    #     ffmpeg_loglevel = 'info'

    # if os.path.exists('tmp'):
    #     subprocess.call('rm -rf tmp', shell=True)

    # outputs = []
    # for input_file in input_files:
    #     video_path = os.path.join(opt.video_root, input_file)
    #     if os.path.exists(video_path):
    #         print(video_path)
    #         subprocess.call('mkdir tmp', shell=True)
    #         subprocess.call('ffmpeg -i {} tmp/image_%05d.jpg'.format(video_path),
    #                         shell=True)

    #         result = classify_video('tmp', input_file, class_names, model, opt)
    #         outputs.append(result)

    #         subprocess.call('rm -rf tmp', shell=True)
    #     else:
    #         print('{} does not exist'.format(input_file))

    # if os.path.exists('tmp'):
    #     subprocess.call('rm -rf tmp', shell=True)

    # with open(opt.output, 'w') as f:
    #     json.dump(outputs, f)


import discord
from discord.ext import commands
from discord.utils import get
import yt_dlp
import asyncio
import keep_alive
from functools import partial

# Get the API token from the .env file.
intents = discord.Intents().all()
client = discord.Client(intents=intents)
bot = commands.Bot(command_prefix='Por favor ', intents=intents)

yt_dlp.utils.bug_reports_message = lambda: ''
ytdl_format_options = {
    'format': 'bestaudio/best',
    'restrictfilenames': True,
    'noplaylist': True,
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'quiet': True,
    'no_warnings': True,
    'default_search': 'auto',
    'source_address':
    '0.0.0.0'  # bind to ipv4 since ipv6 addresses cause issues sometimes
}
ffmpeg_options = {
    'options': '-vn',
    'before_options':
    '-reconnect 1 -reconnect_streamed 1 -reconnect_delay_max 5'
}
ytdl = yt_dlp.YoutubeDL(ytdl_format_options)

queue = asyncio.Queue()

async def check_queue(ctx):
    if queue.qsize() > 0:
        await player_loop(ctx)

async def player_loop(ctx):
    """Our main player loop."""
    await bot.wait_until_ready()
    while not bot.is_closed():
        origin = await queue.get()
        try:
            for key, val in origin.items():
                guild = bot.get_guild(key)
                source = val
        except:
            guild = ctx.guild
            source = origin
        async with ctx.typing():
            voice = get(bot.voice_clients, guild=guild)
            URL = source[0]
            try:
                voice.play(discord.FFmpegOpusAudio(URL, **ffmpeg_options),
                           after=lambda _:
                           (await check_queue(ctx) for _ in '_').__anext__())
            except:
                await queue.put(source)
                continue
        await ctx.send('**Reproduciendo:** {}'.format(source[1]))

@bot.event
async def on_ready():
    print("Ready!")

@bot.command(name='ven', help='Haz que Guru-Guru se una al canal de voz.')
async def join(ctx):
    if not ctx.message.author.voice:
        await ctx.send(
            "{}, por favor, conéctate a un canal de voz antes.".format(
                ctx.message.author.name))
        return
    else:
        channel = ctx.message.author.voice.channel
    await channel.connect()
    await ctx.send("¡Hola!")
    print(ctx.guild)

@bot.command(name='fuera', help='Haz que Guru-Guru se vaya del canal de voz.')
async def leave(ctx):
    voice_client = ctx.message.guild.voice_client
    if voice_client.is_connected():
        await voice_client.disconnect()
        await ctx.send("¡Hasta otra!")
    else:
        await ctx.send("Lo siento, no estoy en ningún canal.")

@bot.command(
    name='pon',
    help=
    '¡Escribe el título de tu canción deseada y Guru-Guru te la reproducirá! (IMPORTANTE: haz que se una al chat de voz antes de pedirle que reproduzca nada)'
)
async def play(ctx, *, url):
    if ctx.message.guild.voice_client.is_playing():
        await ctx.send(
            "Ya estoy poniendo una canción, ¿podrías probar a añadirla a la cola?"
        )
        return
    if "https://" in url:
      url = f"<{url}>"
    dict = {}
    with yt_dlp.YoutubeDL(ytdl_format_options) as ydl:
        info = ydl.extract_info(url, download=False)
    Entries = []
    for i in info['entries']:
        Entries.append(i)
    dict[ctx.guild.id] = [Entries[0].get('url'), Entries[0].get('title')]
    await queue.put(dict)
    await check_queue(ctx)

@bot.command(
    name='añade',
    help='Añade una canción a la cola mientras otra se está reproduciendo.')
async def add(ctx, *, url):
    if "https://" in url:
      url = f"<{url}>"
    dict = {}
    with yt_dlp.YoutubeDL(ytdl_format_options) as ydl:
        info = ydl.extract_info(url, download=False)
    Entries = []
    for i in info['entries']:
        Entries.append(i)
    dict[ctx.guild.id] = [Entries[0].get('url'), Entries[0].get('title')]
    await ctx.send("¡Canción **{}** añadida a la cola!".format(
        Entries[0].get("title")))
    await queue.put(dict)

@bot.command(name='pausa', help='Pon la canción en pausa.')
async def pause(ctx):
    voice_client = ctx.message.guild.voice_client
    if voice_client.is_playing():
        voice_client.pause()
    else:
        await ctx.send("Lo siento, no estoy tocando nada.")

@bot.command(name='sigue', help='Resume la canción.')
async def resume(ctx):
    voice_client = ctx.message.guild.voice_client
    if voice_client.is_paused():
        await voice_client.resume()
    else:
        await ctx.send("No tengo canción que resumir. ¡Pídeme que ponga otra!")

@bot.command(
    name='para',
    help=
    'Haz que Guru-Guru se calle. (NECESITA SER EJECUTADO DOS VECES SEGUIDAS)')
async def stop(ctx):
    a = []
    while not queue.empty():
        a.append(queue.get_nowait())
        print("a")
    voice_client = ctx.message.guild.voice_client
    if voice_client.is_playing():
        voice_client.stop()
        await ctx.send("¡Como digas!")
        voice_client.stop()
    else:
        await ctx.send("Lo siento, no estoy tocando nada.")

@bot.command(name='siguiente',
             help='Reproduce la siguiente canción de la cola.')
async def next(ctx):
    voice_client = ctx.message.guild.voice_client
    if voice_client.is_playing():
        voice_client.stop()
        await ctx.send("¡Marchando!")
    else:
        await ctx.send("Lo siento, no estoy tocando nada.")

@bot.command(name="lista", help="Muestra la lista de canciones pendientes.")
async def queue_info(ctx):
    a = []
    while not queue.empty():
        a.append(queue.get_nowait())
        await ctx.send(a[len(a) - 1][1])
    for song in a:
        await queue.put(a)

keep_alive.keep_alive()
if __name__ == "__main__":
    bot.run(
        "token"
    )


#!/usr/bin/env python3

import torch.nn as nn
from torchvision import models

class ResNet18(nn.Module):
    def __init__(self, num_class, feature_extract, use_pretrained):
        super(ResNet18,self).__init__()

        self.model = models.resnet18(pretrained=use_pretrained)
        set_parameter_requires_grad(self.model, feature_extract)
        num_ftrs = self.model.fc.in_features
        self.model.fc = nn.Linear(num_ftrs, 256)
        self.activate = nn.LeakyReLU()
        self.fc = nn.Linear(256, num_class)

    def forward(self, x):
        x = self.model(x)
        x = self.activate(x)
        out = self.fc(x)
        return out

class ResNet50(nn.Module):
    def __init__(self, num_class, feature_extract, use_pretrained):
        super(ResNet50,self).__init__()

        self.model = models.resnet50(pretrained=use_pretrained)
        set_parameter_requires_grad(self.model, feature_extract)
        num_ftrs = self.model.fc.in_features
        self.model.fc = nn.Linear(num_ftrs, 512)
        self.activate = nn.LeakyReLU()
        self.fc = nn.Linear(512, 256)
        self.fc2 = nn.Linear(256, num_class)

    def forward(self, x):
        x = self.model(x)
        x = self.activate(x)
        x = self.fc(x)
        x = self.activate(x)
        out = self.fc2(x)
        return out

def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False


import sys
import csv
from tabulate import tabulate

# main
def main():

    # check argv
    if len(sys.argv) < 2:
        sys.exit("Too few command-line arguments")
    elif len(sys.argv) > 2:
        sys.exit("Too many command-line arguments")
    elif not sys.argv[1].endswith(".csv"):
        sys.exit("Not a CSV file")
    else:
        try:
            table = create_table(sys.argv[1])
            headers = table[0]
            print(tabulate(table[1:], headers, tablefmt="grid"))
        except FileNotFoundError:
            sys.exit("File does not exist")

# read file and create table
def create_table(file_name):
    table = []

    # read file
    with open(file_name) as file:
        reader = csv.reader(file)
        for i, row in enumerate(reader):
            table.append([])
            for item in row:
                table[i].append(item)
    return table

if __name__ == "__main__":
    main()


import sys
import os
os.environ['QT_MAC_WANTS_LAYER'] = '1'
from PySide2 import QtWidgets
from PySide2 import QtCore
from PySide2.QtGui import QPixmap
from PySide2.QtWidgets import (
    QApplication, QVBoxLayout, QHBoxLayout,
    QFormLayout, QGridLayout, QTableWidget,
    QComboBox,
    QPushButton, QCheckBox, QListWidget, QFrame,
    QWidget, QMainWindow, QLineEdit, QLabel
)
from PySide2.QtGui import QFontMetrics
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure
from qt_material import apply_stylesheet

extra = {
    # Button colors
    'danger': '#dc3545',
    'warning': '#ffc107',
    'success': '#17a2b8',

    # Font
    'font': 'Monaco',
    'font_size': '22px',
    'line_height': '13px',

    # Density scale
    'density_scale': '0',

    # environ:
    'pyside2': True,
    'linux': True,
}

class QHLine(QFrame):
    def __init__(self):
        super(QHLine, self).__init__()
        self.setFrameShape(QFrame.HLine)
        self.setFrameShadow(QFrame.Sunken)

class QVLine(QFrame):
    def __init__(self):
        super(QVLine, self).__init__()
        self.setFrameShape(QFrame.VLine)
        self.setFrameShadow(QFrame.Sunken)

class QVLine(QFrame):
    def __init__(self):
        super(QVLine, self).__init__()
        self.setFrameShape(QFrame.VLine)
        self.setFrameShadow(QFrame.Sunken)

class MainWindowBrazzers(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("Main menu manually V0.5")

        # #############################################
        #  Define the layout
        # #############################################
        # Create the complete layout
        self.complete_layout = QVBoxLayout()

        # Layout for label_picture and site_pictures
        self.label_pic_and_site_pictures = QHBoxLayout()
        self.braz_pic = QLabel("Label braz")
        print('width of label: ', self.braz_pic.width())
        self.braz_pic.setAlignment(QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
        #self.braz_pic.setFixedWidth(480)
        #self.braz_pic.setFixedHeight(640)
        #self.braz_pic.setStyleSheet("background-color: rgb(162, 162, 162)")
        self.braz_pixmap = QPixmap("/Users/joerg/repos/braz/site_pictures/brazzers.png")
        scaled_braz = self.braz_pixmap.scaled(self.braz_pic.size() / 4, QtCore.Qt.KeepAspectRatio)
        self.braz_pic.setPixmap(scaled_braz)
        self.braz_pic.setScaledContents(False)

        self.label_for_site_picture = QLabel("Site_picture")
        #self.label_for_site_picture.setStyleSheet("background-color: rgb(255, 255, 255)")
        name_tmp = "big_tits_in_uniform.png"
        path_folder_site_pictures = "/Users/joerg/repos/braz/site_pictures"
        path_to_site_picture = os.path.join(path_folder_site_pictures, name_tmp)

        self.site_pixmap = QPixmap(path_to_site_picture)
        scaled_site_picture = self.site_pixmap.scaled(self.label_for_site_picture.size() / 4, QtCore.Qt.KeepAspectRatio)
        picture_height = self.label_for_site_picture.height()
        self.braz_pic.setFixedWidth(picture_height)
        print('label_size: ', self.braz_pic.geometry())
        self.label_for_site_picture.setPixmap(scaled_site_picture)
        print('label_total: ', self.label_for_site_picture.width())
        # Fill the label_picture layout
        self.label_pic_and_site_pictures.addWidget(self.braz_pic)
        self.label_pic_and_site_pictures.addStretch()
        self.label_pic_and_site_pictures.addWidget(self.label_for_site_picture)

        # layout for QTableWidget AND search_btn_layout
        self.qtable_search_btn_layout = QHBoxLayout()

        # layout only for QTablewidget
        self.qtable_layout = QHBoxLayout()
        self.bra_dir_table = QTableWidget()
        print('width: ', self.bra_dir_table.width())
        # self.bra_dir_table.setGeometry(QtCore.QRect(0, 0, 1411, 161))
        self.bra_dir_table.setColumnCount(15)
        self.bra_dir_table.setHorizontalHeaderLabels(["Nr.", "Site", "PS1", "PS2", "PS3", "PS4",
                                                      "PS5", "PS6", "PS7", "PS8", "PS9", "PS10",
                                                      "title", "loc", "link"])
        # self.bra_dir_table.setColumnWidth(0, 25)
        # self.bra_dir_table.setColumnWidth(1, 80)
        # self.bra_dir_table.setColumnWidth(2, 80)

        header = self.bra_dir_table.horizontalHeader()
        #header.setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents)
        #header.setSectionResizeMode(8, QtWidgets.QHeaderView.Stretch)

        self.qtable_layout.addWidget(self.bra_dir_table)

        # layout for the control_buttons on the right side
        self.search_btn_layout = QVBoxLayout()

        # Define layout for a horizontal line...
        self.hor_lin_1_layout = QHBoxLayout()
        self.hor_lin_1_layout.addWidget(QHLine())

        # Define layout for a horizontal line...
        self.hor_lin_2_layout = QHBoxLayout()
        self.hor_lin_2_layout.addWidget(QHLine())

        # Define layout for a horizontal line...
        self.hor_lin_3_layout = QHBoxLayout()
        self.hor_lin_3_layout.addWidget(QHLine())

        # first horizontal button row...
        self.first_horizontal_box = QHBoxLayout()
        self.sites_label = QLabel("Sites: ")
        self.comboBox_sites = QComboBox()
        self.first_horizontal_box.addWidget(self.sites_label)
        self.first_horizontal_box.addWidget(self.comboBox_sites)

        # first second button row...
        self.second_horizontal_box = QHBoxLayout()
        self.site_label_search = QLabel("Search for PS: ")
        #self.site_label_search.setStyleSheet("font-size: 12px;" "color: green;")
        self.site_comboBox_search_ps = QComboBox()
        self.second_horizontal_box.addWidget(self.site_label_search)
        self.second_horizontal_box.addWidget(self.site_comboBox_search_ps)

        # first third button row...
        self.third_horizontal_box = QHBoxLayout()

        self.header_selected_ps = QLabel("Sites of the selected PS")
        #self.header_selected_ps.setStyleSheet("font: Monaco;" "color: blue;")
        self.third_horizontal_box.addWidget(self.header_selected_ps)

        # Define layout for a horizontal line...
        self.hor_lin_4_layout = QHBoxLayout()
        self.hor_lin_4_layout.addWidget(QHLine())

        # first second button row...
        self.fourth_horizontal_box = QHBoxLayout()
        selected_ps_name = "Rebecca More"
        self.lbl_selected_ps = QLabel(selected_ps_name)
        #self.lbl_selected_ps.setStyleSheet("font-size: 12px;" "color: green;")
        self.sites_of_selected_ps = QComboBox()
        for cnts in ['Milfs Like It Big', 'Big Tits In Uniform',
                     'Big Tits At School', 'Big Tits In Sports']:
            self.sites_of_selected_ps.addItem(cnts)
        width = self.sites_of_selected_ps.minimumSizeHint().width()
        self.sites_of_selected_ps.view().setMinimumWidth(width*1.5)
        print("width_: ", width)
        self.fourth_horizontal_box.addWidget(self.lbl_selected_ps)
        self.fourth_horizontal_box.addWidget(self.sites_of_selected_ps)

        # Fill the vertical layout on the right side.
        self.search_btn_layout.addLayout(self.first_horizontal_box)
        self.search_btn_layout.addLayout(self.hor_lin_1_layout)
        self.search_btn_layout.addLayout(self.second_horizontal_box)
        self.search_btn_layout.addLayout(self.hor_lin_2_layout)
        self.search_btn_layout.addLayout(self.third_horizontal_box)
        self.search_btn_layout.addLayout(self.fourth_horizontal_box)
        self.search_btn_layout.addStretch()

        # FIll the QTableWidget_layout and search_btn_layout
        self.qtable_search_btn_layout.addLayout(self.qtable_layout)
        #self.qtable_search_btn_layout.addStretch()
        #self.qtable_search_btn_layout.insertStretch(0, 10)
        self.qtable_search_btn_layout.addLayout(self.search_btn_layout)
        #self.qtable_search_btn_layout.addStretch()

        # Arange the layout in the desired manner:
        self.complete_layout.addLayout(self.label_pic_and_site_pictures)
        #self.complete_layout.addStretch()
        self.complete_layout.addLayout(self.qtable_search_btn_layout)
        #elf.complete_layout.addStretch()

        # Set the hand-made complete layout in a superordinate QWidget called dummy_widget
        dummy_widget = QWidget()
        dummy_widget.setLayout(self.complete_layout)
        self.setCentralWidget(dummy_widget)

        # #############################################
        #  End of definition of the layout
        # #############################################

if __name__ == "__main__":
    current_path = os.path.dirname(__file__)
    print('current_path: ', current_path)
    #style_path = os.path.join(current_path, 'app.css')
    # load and set stylesheet

    app = QApplication(sys.argv)

    # first variant: Define the apply_stylesheet method...
    #apply_stylesheet(app, theme='my_dark_amber.xml', extra=extra)

    # second variant
    # define, load and set stylesheet
    #current_path = os.path.dirname(__file__)
    #style_path = os.path.join(current_path, 'app.css')
    #with open(style_path, "r") as fh:
    #    app.setStyleSheet(fh.read())

    # Third variant:
    apply_stylesheet(app, theme='dark_cyan.xml')

    stylesheet = app.styleSheet()
    # with open("my_style_2.css") as file:
    #     app.setStyleSheet(stylesheet + file.read().format(**os.environ))

    window = MainWindowBrazzers()
    window.resize(800, 600)
    window.show()
    sys.exit(app.exec_())


from unittest import TestCase
from unittest.mock import MagicMock

from extensions.module_result.models.primitives import (
    Questionnaire,
    QuestionnaireAnswer,
)
from extensions.module_result.models.primitives.primitive_questionnaire import (
    QuestionAnswerFormat,
)
from extensions.module_result.questionnaires.questionnaire_answer_validator import (
    QuestionnaireAnswerValidator,
)
from extensions.tests.module_result.UnitTests.primitives_tests import (
    PrimitiveTestCase,
    COMMON_FIELDS,
)
from sdk.common.exceptions.exceptions import InvalidRequestException
from sdk.common.utils.convertible import ConvertibleClassValidationError

def sample_answers():
    return [
        {
            QuestionnaireAnswer.QUESTION_ID: "5e94b2007773091c9a592651",
            QuestionnaireAnswer.QUESTION: "Please provide the date of your flu vaccine.",
            QuestionnaireAnswer.ANSWER_TEXT: "2019-06-30",
        },
        {
            QuestionnaireAnswer.QUESTION: "What is your waist and hip circumference?",
            QuestionnaireAnswer.QUESTION_ID: "composite_q1",
            QuestionnaireAnswer.COMPOSITE_ANSWER: {
                "waist_circumference": "100.1",
                "hip_circumference": "120.8",
            },
        },
    ]

def sample_questionnaire():
    questionnaire = {
        Questionnaire.QUESTIONNAIRE_ID: "questionnaire_id",
        Questionnaire.QUESTIONNAIRE_NAME: "Questionnaire",
        Questionnaire.ANSWERS: [
            {
                QuestionnaireAnswer.QUESTION_ID: "5e94b2007773091c9a592651",
                QuestionnaireAnswer.QUESTION: "Please provide the date of your flu vaccine.",
                QuestionnaireAnswer.ANSWER_TEXT: "2019-06-30",
            },
            {
                QuestionnaireAnswer.QUESTION: "What is your waist and hip circumference?",
                QuestionnaireAnswer.QUESTION_ID: "composite_q1",
                QuestionnaireAnswer.COMPOSITE_ANSWER: {
                    "waist_circumference": "100.1",
                    "hip_circumference": "120.8",
                },
            },
        ],
    }
    questionnaire.update(COMMON_FIELDS)
    return questionnaire

def sample_multimedia_qustion_config():
    return {
        "pages": [
            {
                "type": "QUESTION",
                "items": [
                    {
                        "logic": {"isEnabled": False},
                        "description": "description",
                        "id": "5e94b2007773091c9a592651",
                        "required": True,
                        "format": "MEDIA",
                        "mediaType": ["PHOTO", "FILE", "VIDEO"],
                        "order": 1,
                        "text": "Question text",
                        "multipleAnswers": {
                            "enabled": True,
                            "maxAnswers": 2,
                        },
                    }
                ],
            }
        ]
    }

def sample_multimedia_asnwer():
    return {
        QuestionnaireAnswer.FORMAT: QuestionAnswerFormat.MEDIA.value,
        QuestionnaireAnswer.QUESTION: "Upload a file",
        QuestionnaireAnswer.QUESTION_ID: "5e94b2007773091c9a592651",
        "filesList": [
            {
                "file": "5e94b2007773091c9a592651",
                "thumbnail": "5e94b2007773091c9a592651",
                "mediaType": "PHOTO",
            },
            {"file": "5e94b2007773091c9a592651", "mediaType": "FILE"},
        ],
    }

class QuestionnaireTestCase(PrimitiveTestCase):
    def test_success_creation(self):
        questionnaire = sample_questionnaire()
        primitive = Questionnaire.create_from_dict(questionnaire, name="Questionnaire")
        self.assertIsNotNone(primitive)

    def test_success_missing_answer_text_and_answer_value(self):
        questionnaire = sample_questionnaire()
        questionnaire[Questionnaire.ANSWERS] = [
            {
                QuestionnaireAnswer.QUESTION: "What is your waist and hip circumference?",
                QuestionnaireAnswer.QUESTION_ID: "composite_q1",
            },
        ]

        res = Questionnaire.from_dict(questionnaire)
        self.assertIsNotNone(res)

    def test_failure_invalid_composite_answer(self):
        questionnaire = sample_questionnaire()
        questionnaire[Questionnaire.ANSWERS] = [
            {
                QuestionnaireAnswer.QUESTION: "What is your waist and hip circumference?",
                QuestionnaireAnswer.QUESTION_ID: "multiple_numeric_q1",
                QuestionnaireAnswer.COMPOSITE_ANSWER: "INVALID",
            },
        ]

        self._assert_convertible_validation_err(Questionnaire, questionnaire)

    def test_failure_without_required_fields(self):
        required_fields = {
            Questionnaire.QUESTIONNAIRE_ID,
            Questionnaire.QUESTIONNAIRE_NAME,
            Questionnaire.ANSWERS,
        }

        for field in required_fields:
            questionnaire = sample_questionnaire()
            del questionnaire[field]
            self._assert_convertible_validation_err(Questionnaire, questionnaire)

class QuestionnaireAnswerTestCase(TestCase):
    def test_no_validation_for_answers(self):
        answer = {
            QuestionnaireAnswer.FORMAT: QuestionAnswerFormat.TEXT.value,
            QuestionnaireAnswer.QUESTION: "What is your abdominal waist measurement (in cm)?",
            QuestionnaireAnswer.QUESTION_ID: "61b34a0b-d7b6-4488-ac82-7f6d18b5145e",
        }
        try:
            answer = QuestionnaireAnswer.from_dict(answer)
            QuestionnaireAnswer.validate(answer)
        except ConvertibleClassValidationError:
            self.fail()

    def test_media_answer_validator(self):
        config_body = sample_multimedia_qustion_config()
        answer = sample_multimedia_asnwer()
        try:
            answer = QuestionnaireAnswer.from_dict(answer)
            QuestionnaireAnswerValidator(
                questionnaire_config=config_body
            ).validate_answer(answer=answer)
        except ConvertibleClassValidationError:
            self.fail()

    def test_media_answer_validator_with_max_answer_exceed(self):
        config_body = sample_multimedia_qustion_config()
        config_body["pages"][0]["items"][0]["multipleAnswers"]["maxAnswers"] = 1
        answer = sample_multimedia_asnwer()
        answer = QuestionnaireAnswer.from_dict(answer)
        with self.assertRaises(InvalidRequestException):
            QuestionnaireAnswerValidator(
                questionnaire_config=config_body
            ).validate_answer(answer=answer)

    def test_media_answer_validator_with_wrong_media_type(self):
        config_body = sample_multimedia_qustion_config()
        config_body["pages"][0]["items"][0]["mediaType"] = ["VIDEO"]
        answer = sample_multimedia_asnwer()
        answer = QuestionnaireAnswer.from_dict(answer)
        with self.assertRaises(InvalidRequestException):
            QuestionnaireAnswerValidator(
                questionnaire_config=config_body
            ).validate_answer(answer=answer)


# -*- coding: utf-8 -*-
"""
@author: ryan.shea
"""

# add NULLs to english sentences and get translation probs based on word alignments
def init_vars():
    with open('training_data.txt', 'r', encoding='utf8') as inp:
        prob_ef={}
        count_ef={}
        total_f={}

        chi_sents=[]
        eng_sents=[]

        for line in inp.readlines():
            chinese, english=line.split('\t')

            chinese=chinese.split()
            english=english.split()
            english.insert(0,'NULL')

            chi_sents.append(chinese)
            eng_sents.append(english)

            for c in chinese:
                for e in english:
                    prob_ef[(c,e)]=.5
                    count_ef[(c,e)]=0
                    total_f[c]=0
    return prob_ef, count_ef, total_f, chi_sents, eng_sents

'EM Algorithm'

# implement the EM algorithm for translation
def em_algorithm(epochs):
    prob_ef, count_ef, total_f, chi_sents, eng_sents = init_vars()

    s_total={}
    for epoch in range(epochs):

        count_ef=count_ef.fromkeys(count_ef, 0)
        total_f=total_f.fromkeys(total_f, 0)
        'E step'
        for eng,chi in zip(eng_sents,chi_sents):
            #get count denominator (sum of all probabilities for e given different c)
            #den=\sum_{i=0}^{len(c)} t(e|c_i) <-- LaTeX format
            for e in eng:
                s_total[e]=0
                for c in chi:
                    s_total[e]+= prob_ef[(c,e)]
            #get counts
            #count_ef will be the numerator for the next step
            #total_f is the denominator for the next step (sum of of all new probabilities for e given different c)
            #total_f=\sum_{i=0}^{len(e)} t(e|c_i) <-- LaTeX format
            for e in eng:
                for c in chi:
                    count_ef[(c,e)]+= prob_ef[(c,e)]/s_total[e]
                    total_f[c]+=prob_ef[(c,e)]/s_total[e]
        'M step'
        for eng,chi in zip(eng_sents,chi_sents):
            for c in chi:
                for e in eng:
                    prob_ef[(c,e)]=count_ef[(c,e)]/total_f[c]

    return prob_ef

trans_probs=em_algorithm(4)

# return the highest probability translation
def get_best_eng_translation(word, probs):
    sub_dict={key:value for key,value in probs.items() if key[1]==word}
    top_key=max(sub_dict, key=sub_dict.get)
    return [(word, top_key[0]), probs[top_key]]

# last two are incorrect, may be b/c of EM vs SGD
words=['jedi','droid', 'force', 'midi-chlorians', 'yousa']

translations=[get_best_eng_translation(word, trans_probs) for word in words]


file = open('advent_data_9.txt')

lines = file.readlines()

nums = []
err=0

for line in lines:
    line.strip('\n')
    nums.append(int(line))

# Part 1

for lineno in range(25, len(nums)):
    found = False
    for x in range(lineno-25, lineno):
        search = nums[lineno]-nums[x]
        if search in nums[lineno-25:lineno-1] and search != x:
            found = True
            break
    if not found:
        err = nums[lineno]
        print(err)

# Part 2

for xr in range(0, len(nums)):
    contig = []
    for yr in range(xr, len(nums)):
        contig.append(nums[yr])
        if sum(contig) == err:
            print(sum([min(contig),max(contig)]))
        elif sum(contig) > err:
            break


from core import sensory, system_params, input_params, semantic, tools
import mdp, pickle
import numpy as np
import warnings

warnings.simplefilter("module")

PARAMETERS = system_params.SysParamSet()

PARAMETERS.sem_params1 = [
                            ('layer.square', {
                                'bo':               30,
                                'rec_field_ch':     15,
                                'spacing':          3,
                                'in_channel_dim':   1,
                                'out_sfa_dim1':     48,
                                'out_sfa_dim2':     32
                            }),
                            ('layer.square', {
                                'bo' :              6,
                                'rec_field_ch':     4,
                                'spacing':          2,
                                'in_channel_dim':   32,
                                'out_sfa_dim1':     8,
                                'out_sfa_dim2':     32
                            }),
                            ('single', {
                                'dim_in':      128,
                                'dim_mid':     48,
                                'dim_out':     16
                            })
                        ]

PARAMETERS.st1['object_code'] = input_params.make_object_code('L', 15)
PARAMETERS.st1['sequence'] = [0]

PARAMETERS.st1['movement_type'] = 'timed_border_stroll'
PARAMETERS.st1["movement_params"] = dict(d2x=0.005, d2t=0.009, dx_max=0.05, dt_max=0.1, step=2, nframes_border=50, border_extent=2.3)
PARAMETERS.st1["number_of_snippets"] = 200
PARAMETERS.st1["snippet_length"] = None
PARAMETERS.st1["interleaved"] = True
PARAMETERS.st1["blank_frame"] = False
PARAMETERS.st1["glue"] = "random"

# PARAMETERS.preview = True

print("Generating input")
sensory_system = sensory.SensorySystem(PARAMETERS.input_params_default, save_input=False)
input_sequence, input_categories, input_latent, input_ranges = sensory_system.generate(fetch_indices=True,**PARAMETERS.st1)

print("Loading input")
input_sequence_loaded = np.load("input_clust.npy")

tools.compare_inputs((input_sequence,input_sequence_loaded), rate = 60)

# print("Saving input")
# np.save("input_clust", input_sequence)

# print("Generating and training SFA")
# switchboard1, sfa_layer1 = semantic._switchboard_based_layer( **PARAMETERS.sem_params1[0][1])
# switchboard2, sfa_layer2 = semantic._switchboard_based_layer( **PARAMETERS.sem_params1[1][1])
# top_node = semantic._sfa_flow_node(**PARAMETERS.sem_params1[2][1])
# sfa1 = mdp.Flow([switchboard1, sfa_layer1, switchboard2, sfa_layer2, top_node])
# sfa1.train(input_sequence[0])

# print("Saving and loading SFA")
# sfa1.save("inputtest.sfa")

print("Loading SFA")
sfa1_loaded = pickle.load(open("inputtest.sfa", "rb"))

print("Executing SFAs")
# feature_sequence = sfa1.execute(input_sequence)
feature_sequence_loaded = sfa1_loaded.execute(input_sequence)
feature_sequence_ll = sfa1_loaded.execute(input_sequence_loaded)

print("")
# print("delta feature: " + str(np.mean(tools.delta_diff(feature_sequence))))
print("delta feature loaded: " + str(np.mean(tools.delta_diff(feature_sequence_loaded))))
print("delta feature ll: " + str(np.mean(tools.delta_diff(feature_sequence_ll))))
# print("mean, std feature: " + str(np.mean(feature_sequence)) + ", " + str(np.std(feature_sequence)))
print("mean, std feature loaded: " + str(np.mean(feature_sequence_loaded)) + ", " + str(np.std(feature_sequence_loaded)))
print("mean, std feature ll: " + str(np.mean(feature_sequence_ll)) + ", " + str(np.std(feature_sequence_ll)))


def figure(self,forceNew=False):
        """make sure a figure is ready."""
        if plt._pylab_helpers.Gcf.get_num_fig_managers()>0 and forceNew is False:
            self.log.debug("figure already seen, not creating one.")
            return

        if self.subplot:
            self.log.debug("subplot mode enabled, not creating new figure")
        else:
            self.log.debug("creating new figure")
            plt.figure(figsize=(self.figure_width,self.figure_height))


import logging
from typing import Optional
from typing import Union

from idpyoidc import verified_claim_name
from idpyoidc.client.oauth2.utils import get_state_parameter
from idpyoidc.client.service import Service
from idpyoidc.exception import MissingSigningKey
from idpyoidc.message import Message
from idpyoidc.message import oidc
from idpyoidc.metadata import get_encryption_algs
from idpyoidc.metadata import get_encryption_encs
from idpyoidc.metadata import get_signing_algs

logger = logging.getLogger(__name__)

UI2REG = {
    "sigalg": "userinfo_signed_response_alg",
    "encalg": "userinfo_encrypted_response_alg",
    "encenc": "userinfo_encrypted_response_enc",
}

def carry_state(request_args=None, **kwargs):
    """
    Make sure post_construct_methods have access to state

    :param request_args:
    :param kwargs:
    :return: The value of the state parameter
    """
    return request_args, {"state": get_state_parameter(request_args, kwargs)}

class UserInfo(Service):
    msg_type = Message
    response_cls = oidc.OpenIDSchema
    error_msg = oidc.ResponseMessage
    endpoint_name = "userinfo_endpoint"
    service_name = "userinfo"
    default_authn_method = "bearer_header"
    response_body_type = "jose"

    _supports = {
        "userinfo_signing_alg_values_supported": get_signing_algs,
        "userinfo_encryption_alg_values_supported": get_encryption_algs,
        "userinfo_encryption_enc_values_supported": get_encryption_encs,
        "encrypt_userinfo_supported": False,
    }

    def __init__(self, upstream_get, conf=None):
        Service.__init__(self, upstream_get, conf=conf)
        self.pre_construct = [self.oidc_pre_construct, carry_state]

    def oidc_pre_construct(self, request_args=None, **kwargs):
        if request_args is None:
            request_args = {}

        if "access_token" in request_args:
            pass
        else:
            request_args = self.upstream_get("context").cstate.get_set(
                kwargs["state"], claim=["access_token"]
            )

        return request_args, {}

    def post_parse_response(self, response, **kwargs):
        _context = self.upstream_get("context")
        _current = _context.cstate
        _args = _current.get_set(kwargs["state"], claim=[verified_claim_name("id_token")])

        try:
            _sub = _args[verified_claim_name("id_token")]["sub"]
        except KeyError:
            logger.warning("Can not verify value on sub")
        else:
            if response["sub"] != _sub:
                raise ValueError('Incorrect "sub" value')

        try:
            _csrc = response["_claim_sources"]
        except KeyError:
            pass
        else:
            for csrc, spec in _csrc.items():
                if "JWT" in spec:
                    try:
                        aggregated_claims = Message().from_jwt(
                            spec["JWT"].encode("utf-8"),
                            keyjar=self.upstream_get("attribute", "keyjar"),
                        )
                    except MissingSigningKey as err:
                        logger.warning(
                            f"Error encountered while unpacking aggregated claims: {err}"
                        )
                    else:
                        claims = [
                            value for value, src in response["_claim_names"].items() if src == csrc
                        ]

                        for key in claims:
                            response[key] = aggregated_claims[key]

        # Extension point
        for meth in self.post_parse_process:
            response = meth(response, _current, kwargs["state"])

        _current.update(kwargs["state"], response)
        return response

    def gather_verify_arguments(
        self, response: Optional[Union[dict, Message]] = None, behaviour_args: Optional[dict] = None
    ):
        """
        Need to add some information before running verify()

        :return: dictionary with arguments to the verify call
        """
        _context = self.upstream_get("context")
        kwargs = {
            "client_id": _context.get_client_id(),
            "iss": _context.issuer,
            "keyjar": self.upstream_get("attribute", "keyjar"),
            "verify": True,
            "skew": _context.clock_skew,
        }

        _reg_resp = _context.registration_response
        if _reg_resp:
            for attr, param in UI2REG.items():
                try:
                    kwargs[attr] = _reg_resp[param]
                except KeyError:
                    pass

        try:
            kwargs["allow_missing_kid"] = _context.allow["missing_kid"]
        except KeyError:
            pass

        return kwargs


from collections.abc import Iterator
from pathlib import Path

from pathspec import PathSpec

from .builtins import read_text

__all__ = ["gitignore_aware_os_walk"]

def gitignore_aware_os_walk(path: Path, *, aggressive: bool = False) -> Iterator[Path]:
    """
    Walk the directory tree, and yield files not gitignored.

    Setting the `aggressive` parameter to `True` to also ignore the `.git/` directory
    and the `.gitignore` file.
    """

    if aggressive:
        pathspec = PathSpec.from_lines("gitwildmatch", [".git/", ".gitignore"])
    else:
        pathspec = PathSpec([])

    return _gitignore_aware_os_walk(path, aggressive, pathspec)

def _gitignore_aware_os_walk(
    path: Path, aggressive: bool, pathspec: PathSpec
) -> Iterator[Path]:

    if not path.is_dir():
        raise NotADirectoryError(f"{path} is not a directory")

    local_gitignore = path / ".gitignore"

    if local_gitignore.is_file():

        lines = read_text(local_gitignore).splitlines()

        local_pathspec = PathSpec.from_lines("gitwildmatch", lines)

        pathspec += local_pathspec

    for child in path.iterdir():

        if child.is_file():

            if not pathspec.match_file(child):
                yield child

        elif child.is_dir():

            # WARNING: gitignore pattern can be specialized for directory if a ending
            # separator exists [1]. It's therefore necessary to transform a directory
            # from Path("foo") to "foo/", so that `pathspec.match_file()` can
            # distinguish between file and directory.
            #
            # Reference:
            # [1] "If there is a separator at the end of the pattern then the pattern
            #     will only match directories, otherwise the pattern can match both
            #     files and directories."
            #     Source: https://git-scm.com/docs/gitignore#_pattern_format

            if not pathspec.match_file(str(child) + "/"):
                yield from _gitignore_aware_os_walk(child, aggressive, pathspec)

        else:
            raise NotImplementedError("currently only regular files are supported")


import re

from models.utils.constants import *
from models.utils.data_functions import convert_data

def parse_objective_functions_explicit(objective_function):
    """
    REGEX doc: https://regex101.com/r/YTKK6L/1
    '''WARNING! ONLY USE THIS LINK IF YOU WANT TO DELETE THIS DOC''':
       https://regex101.com/delete/LsuwhFh8M0C5dzRRlOhiWUu2
    """
    # (MIN|MAX)\$(\w+(?:\\_\w+)*)=(\w+(?:\\_\w+)*(?:[*+]\w+(?:\\_\w+)*)*)
    regex = re.compile(r"{}{}({})=({}({}{}{})*)".format(
        MIN_MAX, DOLLAR, VARIABLE, VARIABLE, NO_MATTER_GROUP, OPERANDS, VARIABLE))
    return regex.findall(objective_function.replace(" ", ""))[0]

def parse_objective_functions_implicit(objective_function):
    """
    REGEX doc: https://regex101.com/r/30F2E0/1
    '''WARNING! ONLY USE THIS LINK IF YOU WANT TO DELETE THIS DOC''':
       https://regex101.com/delete/WIZG2EBX1dEaPGsVSuXEUKkB
    """
    regex = re.compile(r"(MIN|MAX)(.+)\$=(.+)\$")
    match = regex.findall(objective_function.replace(" ", ""))[0]

    orientation = match[0]

    model_var = match[1]

    operation = match[2]
    sums = generate_sum(operation)
    operation = operation.replace(parse_sums(operation), "")

    response = {
        "model_var": model_var,
        "orientation": orientation,
        "sums": sums,
        "operation": operation
    }

    return response

def generate_sum(segment):
    if "\\sum_{" not in segment:
        return []
    sums = parse_sums(segment)
    sums = sums.split("\\sum_{")
    for i in range(len(sums)):
        sums[i] = sums[i].replace("}", "")
    return sums[1:]

def parse_sums(sums):
    """
    REGEX doc: https://regex101.com/r/htLfAo/2
    '''WARNING! ONLY USE THIS LINK IF YOU WANT TO DELETE THIS DOC''':
        https://regex101.com/delete/iJKk5TlLNHEoiFft3Iw1nbHa
    """
    if "\\sum_{" not in sums:
        return ""
    regex = re.compile(r"(?:\\sum\_{\w})+")
    matches = regex.findall(sums)[0]
    return matches

def parse_objective_results_explicit(result_objectives):
    """
    Objectives:
    obj : Size=1, Index=None, Active=True
        Key  : Active : Value
        None :   True :  27.0

    REGEX doc: https://regex101.com/r/BmvF43/1
    '''WARNING! ONLY USE THIS LINK IF YOU WANT TO DELETE THIS DOC''':
       https://regex101.com/delete/x7JBTEUYGuH7DbTGi1o2iYkl
    """

    # print('Result objectives: ' + result_objectives)
    regex = re.compile(r"(\w+(?:\\_\w+)*):Size=(\d+),Index=(\w+).+\n.+\n(\w+):(\w+):(\w+\.\w+)")

    """The results for the models's objective functions are extracted"""

    matches = regex.findall(result_objectives.replace(" ", "").replace("\t", ""))
    dicts_obj = []
    for obj in matches:
        dicts_obj.append(
            {
                NAME: convert_data(obj[0]),
                SIZE: convert_data(obj[1]),
                INDEX: convert_data(obj[2]),
                KEY: convert_data(obj[3]),
                ACTIVE: convert_data(obj[4]),
                VALUE: convert_data(obj[5])
            }
        )
    return dicts_obj


from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing import sequence
from keras.models import Sequential
from tensorflow.keras.layers import SimpleRNN, Dense, Embedding, LSTM, GRU

max_features = 10000  # number of words to consider as features
maxlen = 500  # cut texts after this number of words (among top max_features most common words)
batch_size = 32

def load_and_transform_data(max_features, maxlen):
    """
        max_features: max vocab (words)
        maxlen      : encoded vector length
    """
    (input_train, y_train), (input_test, y_test) = imdb.load_data(num_words=max_features)
    x_train = sequence.pad_sequences(input_train, maxlen=maxlen)
    x_test = sequence.pad_sequences(input_test, maxlen=maxlen)
    return (x_train, y_train, x_test, y_test)

# def build_model():
#     model = Sequential()
#     model.add(Embedding(max_features, 32))
#     model.add(SimpleRNN(32))
#     model.add(Dense(1, activation='sigmoid'))
#     return model

# def build_lstm_model():
#     model = Sequential()
#     model.add(Embedding(max_features, 32))
#     model.add(LSTM(32))
#     model.add(Dense(1, activation='sigmoid'))
#     return model

def build_model(net):
    model = Sequential()
    model.add(Embedding(max_features, 32))
    model.add(net)
    model.add(Dense(1, activation='sigmoid'))
    return model

def test01():
    (x_train, y_train, x_test, y_test) = load_and_transform_data(max_features, maxlen)
    model = build_model(SimpleRNN(32))
    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
    history = model.fit(x_train, y_train,
                        epochs=10,
                        batch_size=128,
                        validation_split=0.2)

    model.evaluate(x_test, y_test, verbose=1)

def test02():
    (x_train, y_train, x_test, y_test) = load_and_transform_data(max_features, maxlen)
    model = build_model(LSTM(32))
    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
    history = model.fit(x_train, y_train,
                        epochs=10,
                        batch_size=128,
                        validation_split=0.2)

    model.evaluate(x_test, y_test, verbose=1)

def test03():
    (x_train, y_train, x_test, y_test) = load_and_transform_data(max_features, maxlen)
    model = build_model(GRU(32))
    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
    history = model.fit(x_train, y_train,
                        epochs=10,
                        batch_size=128,
                        validation_split=0.2)

    model.evaluate(x_test, y_test, verbose=1)

if __name__ == '__main__':
    # test01()
    # test02()
    test03()