'''Preparing datasets for language modelling, classification and sequence annotation
'''

import fire
# This needs to be called in order to load local implemented submodules
import os
import sys
#module_path = os.path.abspath(os.path.join('../'))
#if module_path not in sys.path:
#    sys.path.append(module_path)

from tqdm import tqdm as tqdm
from utils.proteomics_utils import *
from utils.dataset_utils import *

#for log header
import datetime
import subprocess

import itertools
import ast
from pandas import concat

#for kinase dicts
from collections import defaultdict
######################################################################################################
#DATA PATHS
######################################################################################################
#path to the data directory
data_path = Path('../data')

#SEQUENCE DATA
#ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.xml.gz (this is swissprot not the whole uniprot)
path_sprot=data_path/'uniprot_sprot_2017_03.xml'

#CLUSTER DATA (default path)
#ftp://ftp.ebi.ac.uk/pub/databases/uniprot/uniref/uniref50/uniref50.xml.gz unzipped 100GB file (whole uniprot not just swissprot)
path_uniref = data_path/"uniref50_2017_03.xml"
#path to the data directory also uploaded to git
git_data_path = Path('../git_data')
#path to the temporary directory for dataframes
tmp_data_path = Path("../tmp_data")#should be moved to ./data_tmp or similar
tmp_data_path.mkdir(exist_ok=True)

######################################################################################################
#AUX FUNCTIONS
######################################################################################################
def load_uniprot(source=path_sprot,parse_features=[]):
    '''parse and load uniprot xml
    parse_features: list of features to be parsed (modified residue for PTM etc.)
    '''
    pf="_"+"_".join([p.replace(" ","_") for p in parse_features])
    path_pkl = tmp_data_path/(source.stem+(pf if len(pf)>1 else "")+".pkl")
    if path_pkl.exists():
        print("Loading uniprot pkl from",path_pkl)
        return pd.read_pickle(path_pkl)
    else:
        print(path_pkl, "not found. Parsing uniprot xml...")
        df=parse_uniprot_xml(source,parse_features=parse_features)
        df.to_pickle(path_pkl)
        return df

def load_uniref(source=path_uniref,source_uniprot=path_sprot):
    '''parse and load uniref xml
    '''
    path_pkl = tmp_data_path/(source.stem+("_"+source_uniprot.stem if source_uniprot is not None else "full")+".pkl")
    if path_pkl.exists():
        print("Loading uniref pkl from",path_pkl)
        return pd.read_pickle(path_pkl)
    else:
        print(path_pkl,"not found. Parsing uniref...")
        df_selection = None if source_uniprot is None else load_uniprot(source_uniprot)
        df = parse_uniref(source,max_entries=0,parse_sequence=False,df_selection=df_selection)
        df.to_pickle(path_pkl)
        return df

def load_cdhit(df, cluster_type, dataset):
    '''loads/creates cluster dataframe for a given sequence dataframe (saves pickled result for later runs)
    cluster_type: cdhit05 (uniref-like cdhit down to threshold 0.5 in three stages), cdhit04 (direct cdhit down to threshold 0.4) and recalculate_cdhit05 and recalculate_cdhit04 (which do not make use of cached clusters)
    dataset: e.g. sprot determines the name of the pkl file
    '''
    path_pkl = tmp_data_path/(cluster_type+"_"+dataset+".pkl")

    if path_pkl.exists() and not(cluster_type == "recalculate_cdhit04" or cluster_type == "recalculate_cdhit05"):
        print("Loading cdhit pkl from",path_pkl)
        return pd.read_pickle(path_pkl)
    else:
        print(path_pkl,"not found. Running cdhit...")
        if(cluster_type=="cdhit05" or cluster_type=="recalculate_cdhit05"):#uniref-like cdhit 0.5 in three stages
            threshold=[1.0,0.9,0.5]
            alignment_coverage=[0.0,0.9,0.8]
        else:#direct cdhit clustering to 0.4
            threshold=[0.4]
            alignment_coverage=[0.0]
        df=clusters_df_from_sequence_df(df[["sequence"]],threshold=threshold,alignment_coverage=alignment_coverage)
        df.to_pickle(path_pkl)
        return df

def write_log_header(path, kwargs, filename="logfile.log",append=True):
    path.mkdir(exist_ok=True)
    (path/"models").mkdir(exist_ok=True)

    if "self" in kwargs:
        del kwargs["self"]

    print("======================================\nCommand:"," ".join(sys.argv))
    time = datetime.datetime.now()
    print("started at ",time)
    commit = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip()
    print("Commit:",commit)

    print("\nArguments:")
    for k in sorted(kwargs.keys()):
        print(k,":",kwargs[k])
    print("")

    filepath=path/filename
    with filepath.open("w" if append is False else "a", encoding ="utf-8") as f:
        f.write("\n\nCommand "+" ".join(sys.argv)+"\n")
        f.write("started at "+str(time)+"\n")
        f.write("Commit "+str(commit)+"\n")
        f.write("\nArguments:\n")
        for k in sorted(kwargs.keys()):
            f.write(k+": "+str(kwargs[k])+"\n")
        f.write("\n")

def write_log(path, text, filename="logfile.log",append=True):
    filepath=path/filename
    with filepath.open("w" if append is False else "a", encoding ="utf-8") as f:
        f.write(str(text)+"\n")
######################################################################################################
class Preprocess(object):
    """Preprocessing class (implemented as class to allow different function calls from CLI)."""

    #############################
    # GENERAL
    #############################
    def _preprocess_default(self,path,df,df_cluster,pad_idx=0,sequence_len_min_aas=0,sequence_len_max_aas=0,sequence_len_max_tokens=0,drop_fragments=False,minproteinexistence=0,exclude_aas=[],nfolds=None,sampling_ratio=[0.8,0.1,0.1],ignore_pretrained_clusters=False,save_prev_ids=True,regression=False,sequence_output=False,bpe=False,bpe_vocab_size=100, sampling_method_train=1, sampling_method_valtest=3,subsampling_ratio_train=1.0,randomize=False,random_seed=42,mask_idx=1,tok_itos_in=[],label_itos_in=[],pretrained_path=None, df_redundancy=None, df_cluster_redundancy=None):
        '''internal routine for lm preprocessing
        path: Pathlib model path
        df_cluster: clusters dataframe
        pad_idx: index of the padding token

        sequence_len_min_aas: only consider sequences of at least sequence_len_min_aas aas length 0 for no restriction
        sequence_len_max_aas: only consider sequences up to sequence_len_max_aas aas length 0 for no restriction
        sequence_len_max_tokens: only consider sequences up to sequence_len_max_tokens tokens length (after tokenization) 0 for no restriction
        drop_fragments: drop proteins with fragment marker (requires fragment column in df)
        minproteinexistence: drop proteins with proteinexistence < value (requires proteinexistence column in df)
        exclude_aas: drop sequences that contain aas in the exclude list e.g. exclude_aas=["B","J","X","Z"] for sequences with only canonical AAs ( B = D or N, J = I or L, X = unknown, Z = E or Q)

        nfolds: if None perform a single split according to sampling_ratio else nfold CV split
        sampling_ratio: sampling ratios for train/val/test
        regression: regression task
        sequence_output: output/label is a sequence
        ignore_pretrained_clusters: do not take into account existing clusters from the previous LM step during train-test-split

        bpe: apply BPE
        bpe_vocab_size: vocabulary size (including original tokens) for BPE

        sampling_method: sampling method for train test split as defined in dataset_utils
        subsampling_ratio_train: artificially reduce train (clusters) to specified ratio (1.0 for full dataset)
        mask_idx: index of the mask token (for BERT masked LM training) None for none

        tok_itos_in: allows to pass tok_itos_in (used for trembl processing) will use pretrained tok_itos if an empty list is passed

        df_redundancy: optional df with same structure as df with additional redundant sequences
        df_cluster_redundancy: optional cluster df including redundant and original sequences
        '''

        #filter by fragments
        if(drop_fragments):
            if("fragment" in df.columns):
                df = df[df.fragment == 0]
                if(df_redundancy is not None):
                    df_redundancy = df_redundancy[df_redundancy.fragment == 0]
            else:
                print("Warning: could not drop fragments as fragment column is not available")
        #filter by proteinexistence
        if(minproteinexistence>0):
            if("proteinexistence" in df.columns):
                df = df[df.proteinexistence >= minproteinexistence]
                if(df_redundancy is not None):
                    df_redundancy = df_redundancy[df_redundancy.proteinexistence >= minproteinexistence]
            else:
                print("Warning: could not filter by protein existence as proteinexistence column is not available")
        #filter by non-canonical AAs
        if(len(exclude_aas)>0):
            df = filter_aas(df, exclude_aas)
            if(df_redundancy is not None):
                df_redundancy = filter_aas(df_redundancy, exclude_aas)
        #filter by aa length
        if(sequence_len_min_aas>0):
            df = df[df.sequence.apply(len)>sequence_len_min_aas]
            if(df_redundancy is not None):
                df_redundancy = df_redundancy[df_redundancy.sequence.apply(len)>sequence_len_min_aas]
        if(sequence_len_max_aas>0):
            df = df[df.sequence.apply(len)<sequence_len_max_aas]
            if(df_redundancy is not None):
                df_redundancy = df_redundancy[df_redundancy.sequence.apply(len)<sequence_len_max_aas]

        if(bpe):
            spt=sentencepiece_tokenizer(path if pretrained_path is None else pretrained_path,df,vocab_size=bpe_vocab_size)

        if(len(tok_itos_in)==0 and pretrained_path is not None):
            tok_itos_in = np.load(pretrained_path/"tok_itos.npy")
        if(pretrained_path is not None and ignore_pretrained_clusters is False):
            if(nfolds is None):
                train_ids_prev = np.load(pretrained_path/"train_IDs_prev.npy")
                val_ids_prev = np.load(pretrained_path/"val_IDs_prev.npy")
                test_ids_prev = np.load(pretrained_path/"test_IDs_prev.npy")
            else:
                cluster_ids_prev = np.load(pretrained_path/"cluster_IDs_CV_prev.npy")
        else:
            if(nfolds is None):
                train_ids_prev = []
                val_ids_prev = []
                test_ids_prev =[]
            else:
                cluster_ids_prev = []

        prepare_dataset(path,df,tokenizer=(spt.tokenize if bpe is True else list_tokenizer),pad_idx=pad_idx,sequence_len_max_tokens=sequence_len_max_tokens,mask_idx=mask_idx,tok_itos_in=tok_itos_in,label_itos_in=label_itos_in,df_seq_redundant=df_redundancy,regression=regression,sequence_output=sequence_output)
        ids_current_all = np.load(path/"ID.npy") #filtered by sequence length
        ids_current = np.intersect1d(list(df.index),ids_current_all) #ids_current_all might contain more sequences
        ids_current_redundancy = [] if df_cluster_redundancy is None else  np.intersect1d(list(df_redundancy.index),ids_current_all)
        if(nfolds is None):
            train_test_split(path,ids_current,ids_current_all,df_cluster,train_ids_prev=train_ids_prev,val_ids_prev=val_ids_prev,test_ids_prev=test_ids_prev,sampling_ratio=sampling_ratio,sampling_method_train=sampling_method_train,sampling_method_valtest=sampling_method_valtest,subsampling_ratio_train=subsampling_ratio_train,randomize=randomize,random_seed=random_seed,save_prev_ids=save_prev_ids,ids_current_redundancy=ids_current_redundancy,df_cluster_redundancy=df_cluster_redundancy)
        else:
            cv_split(path,ids_current,ids_current_all,df_cluster,clusters_prev=cluster_ids_prev,nfolds=nfolds, sampling_method_train=sampling_method_train, sampling_method_valtest=sampling_method_valtest, randomize=randomize, random_seed=random_seed, save_prev_ids=save_prev_ids)

    #############################
    # LM
    #############################
    def lm_sprot(self, source_sprot=path_sprot,source_uniref=path_uniref,only_human_proteome=False,drop_fragments=False,minproteinexistence=0,exclude_aas=[],working_folder="./lm_sprot",pretrained_folder="",pad_idx=0,sequence_len_min_aas=0,sequence_len_max_aas=0,sequence_len_max_tokens=0,nfolds=None,sampling_ratio=[0.9,0.05,0.05],cluster_type="uniref",ignore_clusters=False,bpe=False,bpe_vocab_size=100,sampling_method_train=1, sampling_method_valtest=3,subsampling_ratio_train=1.0,randomize=False,random_seed=42,mask_idx=1):

        '''prepare sprot LM data

        only_human_proteome: filter only human proteome
        drop_fragments: drop proteins marked as fragments
        minproteinexistence: drop proteins with protein existence smaller than minproteinexistence
        working_folder: path of the data folder to be created (as string)
        pretrained_folder: path to pretrained folder (as string; empty string for none)
        pad_idx: index of the padding token
        sequence_len_max_tokens: only consider sequences up to sequence_len_max_tokens tokens length (after tokenization) 0 for no restriction

        nfolds: number of CV splits; None for single split
        sampling_ratio: sampling ratios for train/val/test
        ignore_clusters: do not use cluster information for train/test split
        cluster_type: source of clustering information (uniref, cdhit05: cdhit threshold 0.5 similar to uniref procedure, cdhit04: cdhit with threshold 0.4)

        bpe: apply BPE
        bpe_vocab_size: vocabulary size (including original tokens) for BPE

        sampling_method: sampling method for train test split as defined in dataset_utils
        subsampling_ratio_train: portion of train clusters used
        pick_representative_for_val_test: just select a single representative per cluster for validation and test set
        mask_idx: index of the mask token (for BERT masked LM training) None for none
        '''
        print("Preparing sprot LM")

        LM_PATH=Path(working_folder)
        write_log_header(LM_PATH,locals())

        source_sprot = Path(source_sprot)

        df = load_uniprot(source=source_sprot)

        if(only_human_proteome):
            df = filter_human_proteome(df)
            print("Extracted {} human proteines".format(df.shape[0]))

        if(ignore_clusters is False):
            if(cluster_type[:6]=="uniref"):
                df_cluster = load_uniref(source_uniref,source_sprot)
            else:
                df_cluster = load_cdhit(df,cluster_type,source_sprot.stem)
        else:
            df_cluster = None
        self._preprocess_default(path=LM_PATH,df=df,df_cluster=df_cluster,pad_idx=pad_idx,sequence_len_min_aas=sequence_len_min_aas,sequence_len_max_aas=sequence_len_max_aas,sequence_len_max_tokens=sequence_len_max_tokens,drop_fragments=drop_fragments,minproteinexistence=minproteinexistence,exclude_aas=exclude_aas,nfolds=nfolds,sampling_ratio=sampling_ratio,bpe=bpe,bpe_vocab_size=bpe_vocab_size,sampling_method_train=sampling_method_train,sampling_method_valtest=sampling_method_valtest,subsampling_ratio_train=subsampling_ratio_train,randomize=randomize,random_seed=random_seed,mask_idx=mask_idx,pretrained_path=Path(pretrained_folder) if pretrained_folder !="" else None)

    def lm_netchop(self,working_folder="./lm_netchop",pretrained_folder="",
               existing_netchop_peptides=None, netchop_path="netchop", protein_fasta_file="./proteins.fasta",
               pad_idx=0,sequence_len_min_aas=0,sequence_len_max_aas=0,sequence_len_max_tokens=0,exclude_aas=[],nfolds=None,sampling_ratio=[0.9,0.05,0.05],
               netchop_min_length=8, netchop_max_length=20, netchop_repeats=30,
               cluster_type="cdhit05",ignore_clusters=True,bpe=False,bpe_vocab_size=100,
               sampling_method_train=1, sampling_method_valtest=3,randomize=False,random_seed=42,mask_idx=1):
        '''
        Prepare netchop digested sprot LM data. Starts netchop as a suprocess and slices proteins from fasta file. Tokanizes sequences and performs train-val-test split.

        working_folder: path of the data folder to be created (as string)
        pretrained_folder: path to pretrained folder (as string; empty string for none)
        pad_idx: index of the padding token
        sequence_len_max_tokens: only consider sequences up to sequence_len_max_tokens tokens length (after tokenization) 0 for no restriction

        netchop_path: e.g. ./netchop to call netchop in present directory (the default value requires to place symlink to netchop tcsh in netchop3.1 into searchpath e.g. ~/bin) BE CAREFUL NETCHOP TMP PATH MAY NOT BE TOO LONG
        protein_fasta_file: fasta file with proteins
        existing_netchop_peptides: None if not available, filename if netchop digested peptides are available as fasta file
        netchop_min_length: minimum length of netchop slices sequence, shorter sequences are discarded
        netchop_max_length: maximum length of netchop slices sequence, longer sequences are discarded
        netchop_repeats: numbers of iterations stochastic netchop slicing is applied to the whole set of proteins

        nfolds: number of CV splits; None for single split
        sampling_ratio: sampling ratios for train/val/test
        ignore_clusters: do not use cluster information for train/test split
        cluster_type: source of clustering information (uniref, cdhit05: cdhit threshold 0.5 similar to uniref procedure, cdhit04: cdhit with threshold 0.4)

        bpe: apply BPE
        bpe_vocab_size: vocabulary size (including original tokens) for BPE

        sampling_method: sampling method for train test split as defined in dataset_utils
        pick_representative_for_val_test: just select a single representative per cluster for validation and test set
        mask_idx: index of the mask token (for BERT masked LM training) None for none
        '''
        print("Preparing netchop digested sprot LM data")

        LM_PATH=Path(working_folder)
        write_log_header(LM_PATH,locals())

        if existing_netchop_peptides is not None:
            peptides = proteins_from_fasta(data_path/existing_netchop_peptides)
            df = sequences_to_df(peptides)
        else:
            peptides = netchop_digest(protein_fasta_file, netchop_path=netchop_path, threshold=0.7, repeats=netchop_repeats, min_length=netchop_min_length, max_length=netchop_max_length,verbose=True)
            df = sequences_to_df(peptides)

        # TODO
        if(ignore_clusters is False):
            df_cluster = load_cdhit(df,cluster_type,"netchop_peptides")
        else:
            df_cluster = None

        self._preprocess_default(path=LM_PATH,df=df,df_cluster=df_cluster,pad_idx=pad_idx,sequence_len_min_aas=sequence_len_min_aas,sequence_len_max_aas=sequence_len_max_aas,sequence_len_max_tokens=sequence_len_max_tokens,exclude_aas=exclude_aas,nfolds=nfolds,sampling_ratio=sampling_ratio,
         bpe=bpe,bpe_vocab_size=bpe_vocab_size,sampling_method_train=sampling_method_train,sampling_method_valtest=sampling_method_valtest,
         randomize=randomize,random_seed=random_seed,mask_idx=mask_idx,pretrained_path=Path(pretrained_folder) if pretrained_folder !="" else None)

    def clas_mhc_kim(self,mhc_select,cv_type="gs",cv_fold=0,working_folder="./clas_mhc",pretrained_folder="./lm_mhc"):
        '''
        Prepares Kim14 data of one allele with BD09 as train set and Blind as test set.

        The dataset  is available at http://tools.iedb.org/main/datasets/ under bulletpoint "Dataset size and composition impact the reliability of performance benchmarks for peptide-MHC binding predictions." as benchmark_reliability.tar.gz and should be saved in ../data

        Returns raw dataframe with columns (species,mhc,peptide_length,cv,sequence,inequality,meas,label,cluster_ID) and allele ranking as dataframe with columns ("mhc","rank") if mhc_select snf working_folder are set to None.

        mhc_select: int between 0 and 52, choose allele by frequency rank in Binding Data 2009

        cv_type: string, strategy for 5-fold cross validation, options:
        - None: No cv-strategy, cv column is filled with 'TBD'
        - sr: removal of similar peptides seperatly in binder/ non-binder set, using similarity threshold of 80%, similarity found with 'Hobohm 1 like algorithm'
        - gs: grouping similar peptides in the same cv-partition
        - rnd: random partioning

        cv_fold: 0..4 selects the cv_fold to be used as validation set (in case cv_type!= None)
        '''
        if mhc_select is not None and working_folder is not None:
            print("Preparing mhc classification dataset #"+str(mhc_select)+" ...")
            CLAS_PATH = Path(working_folder)
            LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None
            write_log_header(CLAS_PATH,locals())

        df_clas = generate_mhc_kim(cv_type=cv_type, mhc_select=mhc_select, to_csv=False, filename=None, data_dir=data_path,regression=True,transform_ic50="log")

        #generate fake cluster df for train test split
        df_clas["cluster_ID"]=0 #mark everything as train
        if(not cv_type in ["sr","gs","rnd"]):#use blind set as val set (if no cv splits provided)
            df_clas.loc[df_clas.cv=="blind","cluster_ID"]=1
        else:#cv_type specified
            df_clas.loc[df_clas.cv=="blind","cluster_ID"]=2#assign blind to test
            df_clas.loc[df_clas.cv==cv_fold,"cluster_ID"]=1#assign selected cv_fold to val

        if mhc_select is not None and working_folder is not None:
            self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df_clas,df_cluster=df_clas,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)
        else:
            return df_clas,df_clas[df_clas.cv!="blind"].groupby('mhc').size().sort_values(ascending=False).reset_index()["mhc"].reset_index().rename(columns={"index":"rank"})

    def clas_mhc_flurry(self,mhc_select,working_folder="./clas_mhc_flurry",pretrained_folder="./lm_mhc_flurry",test_set=None,pad_idx=0,mask_idx=1,sequence_len_min_aas=0,sequence_len_max_aas=0,sequence_len_max_tokens=0,bpe=False,bpe_vocab_size=100,random_seed=42,regression=True,transform_ic50=None,label=False):
        '''
       test_set="hpv" available for alleles
       ['HLA-A*01:01','HLA-A*11:01','HLA-A*02:01','HLA-A*24:02',
       'HLA-A*03:01','HLA-B*15:01','HLA-B*07:02']
        '''
        print("Preparing mhc classification dataset "+str(mhc_select)+" ...")
        CLAS_PATH = Path(working_folder)
        LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None
        write_log_header(CLAS_PATH,locals())

        df_clas = generate_mhc_flurry(ms='noMS', mhc_select=mhc_select, binder_threshold=500, filter_length=True, label=label,data_dir=data_path, regression=regression,transform_ic50=transform_ic50)

        if(test_set=="abelin"):
            df_test = generate_abelin(mhc_select=mhc_select,data_dir=data_path)
            df_clas = pd.concat([df_clas,df_test], ignore_index=True,sort=True)
        elif(test_set=="hpv"):
            flurry_hpv_map = {'HLA-A*01:01':'HLAA1',
                  'HLA-A*11:01':'HLAA11',
                  'HLA-A*02:01':'HLAA2',
                  'HLA-A*24:02':'HLAA24',
                  'HLA-A*03:01':'HLAA3',
                  'HLA-B*15:01':'HLAB15',
                  'HLA-B*07:02':'HLAB7'}
            df_test = prepare_hpv(flurry_hpv_map[mhc_select], data_dir="../git_data")
            df_test["cluster_ID"]=2
            df_clas = pd.concat([df_clas,df_test], ignore_index=True,sort=True)

        #self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df_clas,df_cluster=df_clas,pad_idx=pad_idx,mask_idx=mask_idx,sequence_len_min_aas=sequence_len_min_aas,sequence_len_max_aas=sequence_len_max_aas,sequence_len_max_tokens=sequence_len_max_tokens,bpe=bpe,bpe_vocab_size=bpe_vocab_size,sampling_method_train=-1,sampling_method_valtest=-1,regression=regression)
        self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df,df_cluster=df,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)

    def clas_mhc_i_zhao(self, mhc_select, working_folder="./clas_mhc_i_zhao",pretrained_folder="./lm_mhc",train_set="MHCFlurry18"):
        '''
        Prepares one allele from IEDB16_I test data from https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1006457:
        Use either Kim14 or MHCFlurry18 data as training set.

        Choose the allele by frequency rank in the selected training set.

        Returns raw dataframe with columns ('ID', 'allele', 'cluster_ID', 'inequality', 'label', 'meas','measurement_source', 'measurement_type', 'original_allele','sequence']) and allele ranking as dataframe with columns ("allele","rank") if mhc_select snf working_folder are set to None.

        mhc_select: integer between 0...29 or ['HLA-A-0205', 'HLA-B-2704', 'HLA-B-2706'], the latter are not in Kim data, thus MHCFlurry18 data is used
        train_set: "MHCFlurry18" or "Kim14"
        '''
        assert train_set in ["MHCFlurry18","Kim14"], "no valid train set"

        if mhc_select is not None and working_folder is not None:
            print("Preparing mhc I dataset #"+str(mhc_select))
            CLAS_PATH = Path(working_folder)
            LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None
            write_log_header(CLAS_PATH,locals())

        df_test = prepare_mhci_pcbi()
        df_test["cluster_ID"]=2

        if train_set=="Kim14":
            if mhc_select in ['HLA-A-0205', 'HLA-B-2704', 'HLA-B-2706']:
                df_train = generate_mhc_flurry(ms='noMS', mhc_select=None, regression=True, transform_ic50="log", data_dir="../data")
                df_train["allele"] = df_train["allele"].apply(lambda x: x.replace("*","-",1).replace(":","",1) if  x.startswith("HLA") else x)
            else:
                tmp = generate_mhc_kim(cv_type="gs", mhc_select=None, regression=True, transform_ic50="log",data_dir="../data/benchmark_mhci_reliability/binding")
                df_train = generate_mhc_kim(cv_type="gs", mhc_select=None, regression=True, transform_ic50="log",data_dir="../data/benchmark_mhci_reliability/binding").rename(columns={"mhc":"allele"})
                df_train["cluster_ID"]=0 #mark everything as train
                df_train.loc[df_train.cv==0,"cluster_ID"]=1 # validation set
                train_alleles_also_in_test_set = df_train["allele"][df_train["allele"].isin(df_test["allele"].unique())].unique()
                allele_ranking = df_train[df_train["allele"].isin(train_alleles_also_in_test_set)].groupby("allele").size().sort_values(ascending=False).index
        elif train_set=="MHCFlurry18":
            df_train = generate_mhc_flurry(ms='noMS', mhc_select=None, regression=True, transform_ic50="log", data_dir="../data")
            df_train["allele"] = df_train["allele"].apply(lambda x: x.replace("*","-",1).replace(":","",1) if  x.startswith("HLA") else x)
            test_alleles = df_test.allele.unique()

            df_train = df_train[df_train.allele.isin(test_alleles)]
            print(test_alleles.shape, df_train.allele.nunique())
            allele_ranking = df_train.groupby("allele").size().sort_values(ascending=False).index

        df = concat([df_train,df_test],ignore_index=True,sort=True)
        df.duplicated(["allele","sequence"],keep="last")
        # Take sequences also present in test set out of the training set
        df = df[~df.duplicated(["allele","sequence"],keep="last")]

        if mhc_select is not None:
            if train_set=="Kim14":
                if mhc_select in range(30):
                    #print(allele_ranking[mhc_select])
                    df = df[df.allele==allele_ranking[mhc_select]]
                elif mhc_select in ['HLA-A-0205', 'HLA-B-2704', 'HLA-B-2706']:
                    df = df[df.allele==mhc_select]
            elif train_set=="MHCFlurry18":
                df = df[df.allele==allele_ranking[mhc_select]]

        if working_folder is not None:
            self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df,df_cluster=df,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)
        else:
            return df,df_train.groupby("allele").size().sort_values(ascending=False).reset_index()["allele"].reset_index().rename(columns={"index":"rank"})

    def clas_mhc_i_hpv(self, mhc_select, working_folder="./clas_mhc_i_hpv",pretrained_folder="./lm_mhc", train_set="MHCFlurry18"):
        '''
        Prepare test data from Bonsack M, Hoppe S, Winter J, Tichy D, Zeller C, Küpper MD, et al. Performance evaluation of MHC class-I binding prediction tools based on an experimentally validated MHC–peptide binding data set. Cancer Immunol Res 2019;7:719–36.
        Save the dataset as HPV_data.csv in ./data.

        Use either Kim14 or MHCFlurry18 data as training set.

        mhc_select: string from ['HLAA1', 'HLAA2', 'HLAA3', 'HLAA11', 'HLAA24', 'HLAB7', 'HLAB15']
        train_set: "MHCFlurry18" or "Kim14"
        '''
        assert train_set in ["MHCFlurry18","Kim14"], "no valid train set"

        CLAS_PATH = Path(working_folder)
        LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None

        df = prepare_hpv(mhc_select, data_dir="../git_data")
        df["cluster_ID"] = 2 #mark everything as test

        if train_set=="MHCFlurry18":
            flurry_hpv_map = {'HLAA1': 'HLA-A*01:01',
                             'HLAA11': 'HLA-A*11:01',
                             'HLAA2': 'HLA-A*02:01',
                             'HLAA24': 'HLA-A*24:02',
                             'HLAA3': 'HLA-A*03:01',
                             'HLAB15': 'HLA-B*15:01',
                             'HLAB7': 'HLA-B*07:02'}
            df_train = generate_mhc_flurry(ms='noMS', mhc_select=flurry_hpv_map[mhc_select], regression=True, transform_ic50="log", binder_threshold=500, filter_length=True, label_binary=False, data_dir=data_path)
        elif train_set=="Kim14":
            kim_hpv_map = {'HLAA1': 'HLA-A-0101',
                          'HLAA11': 'HLA-A-1101',
                          'HLAA2': 'HLA-A-0201',
                          'HLAA24': 'HLA-A-2402',
                          'HLAA3': 'HLA-A-0301',
                          'HLAB15': 'HLA-B-1501',
                          'HLAB7': 'HLA-B-0702'}
            df_train = generate_mhc_kim(cv_type="gs", mhc_select=kim_hpv_map[mhc_select], regression=True, transform_ic50="log",data_dir=data_path).rename(columns={"mhc":"allele"})
            #df_train = df_train[df_train["allele"]==kim_hpv_map[mhc_select]]
            df_train["cluster_ID"]=0 #mark everything as train
            df_train.loc[df_train.cv==0,"cluster_ID"]=1#assign one cv_fold to val

        df = pd.concat([df, df_train], ignore_index=True,sort=True)

        self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df,df_cluster=df,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)
    def clas_mhc_sars_cov(self, mhc_select, train_on, MS=False, working_folder="./clas_mhc_i_sars_cov",pretrained_folder="./lm_mhc"):
        '''
        Prepare test data from https://www.nature.com/articles/s41598-020-77466-4#MOESM1 - stability measurements of SARS-CoV-2 peptides

        train_on: string, options "netmhcpan41", "netmhcpan4", "flurry"

        Use MHCFlurry18 or NetMHCpan data as training set for MHC I and NetMHCpan dataset for MHC II
        Optionally use MS data from NetMHCpan with MS set to True.

        mhc_select: string from ["1 A0101",
                                "2 A0201",
                                "3 A0301",
                                "4 A1101",
                                "5 A2402",
                                "6 B4001",
                                "7 C0401",
                                "8 C0701",
                                "9 C0702",
                                "10 C0102",
                                "11 DRB10401"]
        working_folder: if None, returns dataframe
        '''
        CLAS_PATH = Path(working_folder) if working_folder is not None else None
        LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None

        df = prepare_sars_cov(mhc_select, data_dir=data_path)
        df["cluster_ID"] = 2 #mark everything as test
        if MS:
            df["label"] = df["label"] >= 60

        if mhc_select=="11 DRB10401":
            # always use netmhc 3.2 data for training
            df_train = prepare_mhcii_netmhcpan(mhc_select="DRB1_0401", MS=MS, data_dir=data_path, netmhc_data_version="3.2")
        else:
            # mapping between sheet titles in Covid19-Intavis-Immunitrack-datasetV2.xlsx corresponding to allele names
            # to allele names in MHCFlurry18 dataset
            if train_on=="flurry":
                allele_map = {"1 A0101": 'HLA-A*01:01',
                        "2 A0201": 'HLA-A*02:01',
                        "3 A0301": 'HLA-A*03:01',
                        "4 A1101": 'HLA-A*11:01',
                        "5 A2402": 'HLA-A*24:02',
                        "6 B4001": 'HLA-B*40:01',
                        "7 C0401": 'HLA-C*04:01',
                        "8 C0701": 'HLA-C*07:01',
                        "9 C0702": 'HLA-C*07:02' ,
                        "10 C0102": 'HLA-C*01:02'}
                df_train = generate_mhc_flurry(ms='noMS', mhc_select=allele_map[mhc_select], regression=True, transform_ic50="log", binder_threshold=500, filter_length=True, label_binary=False, data_dir=data_path)
            else:
                allele_map = {"1 A0101": 'HLA-A01:01',
                        "2 A0201": 'HLA-A02:01',
                        "3 A0301": 'HLA-A03:01',
                        "4 A1101": 'HLA-A11:01',
                        "5 A2402": 'HLA-A24:02',
                        "6 B4001": 'HLA-B40:01',
                        "7 C0401": 'HLA-C04:01',
                        "8 C0701": 'HLA-C07:01',
                        "9 C0702": 'HLA-C07:02' ,
                        "10 C0102": 'HLA-C01:02'}
                netmhc_version = "4.0" if train_on=="netmhcpan4" else "4.1"
                df_train = prepare_mhci_netmhcpan_4(mhc_select=allele_map[mhc_select], MS=MS, data_dir=data_path, netmhc_data_version=netmhc_version)

        df = pd.concat([df, df_train], ignore_index=True,sort=True)

        if MS:
            df["label"] = df["label"].astype(int)

        if working_folder is not None:
            self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df,df_cluster=df,sampling_method_train=-1,sampling_method_valtest=-1,regression=False if MS else True,ignore_pretrained_clusters=True)
        else:
            return df

    def clas_mhc_sars_cov_pan(self, mhc_class, working_folder="./clas_mhc_i_sars_cov_pan"):

        CLAS_PATH = Path(working_folder)

        train = prepare_mhci_netmhcpan_4(None, with_MHC_seq=True, data_dir=data_path, netmhc_data_version="4.0")
        test = prepare_sars_cov(None, mhc_class=mhc_class, with_MHC_seq=True, data_dir=data_path)
        test["cluster_ID"] = 2

        df = pd.concat([train,test], axis=0, sort=True, ignore_index=True)

        # concat MHC pseudo sequence and peptide sequence
        df["sequence"] = df["sequence1"] + "x" + df["sequence"]

        prep = Preprocess()

        # provide previous tokens so that "x" is mapped to '_pad_'
        prev_tok = ['_pad_', '_mask_', '_bos_', 'G', 'E', 'S', 'P', 'A', 'D', 'T', 'V', 'L', 'R', 'N', 'I', 'Q', 'K', 'C', 'F', 'H', 'M', 'Y', 'W']

        self._preprocess_default(path=CLAS_PATH,
                                df=df,df_cluster=df,
                                sampling_method_train=-1,sampling_method_valtest=-1,
                                regression=True,ignore_pretrained_clusters=True,
                                tok_itos_in=prev_tok)

    def clas_mhc_ii(self, mhc_select, working_folder="./clas_mhc_ii",pretrained_folder="./lm_mhc"):
        '''
        Prepares IEDB16_II data from https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1006457: of on allele.
        Uses Wang10 datset (classII_binding_data_Nov_16_2009.zip from http://tools.iedb.org/mhcii/download/) as training data.

        mhc_select: integer between 0...24
        '''
        print("Preparing mhc II dataset #"+str(mhc_select)+" ...")
        CLAS_PATH = Path(working_folder)
        LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None
        write_log_header(CLAS_PATH,locals())

        df_train = prepare_mhcii_iedb2009()
        df_test = prepare_mhcii_pcbi()

        # select only one train/test split out of the 5
        df_train = df_train[df_train["cv_fold"]=="0"]

        # select one allele based on training set size ranking
        train_alleles_also_in_test_set = df_train["allele"][df_train["allele"].isin(df_test["allele"].unique())].unique()
        allele_ranking = df_train[df_train["allele"].isin(train_alleles_also_in_test_set)].groupby("allele").size().sort_values(ascending=False).index
        if mhc_select is not None:
            df_train = df_train[df_train["allele"]==allele_ranking[mhc_select]]
            df_test = df_test[df_test["allele"]==allele_ranking[mhc_select]]

        # use the "test" part of the selected cv fold as validation set
        df_train["cluster_ID"] = 0
        df_train.loc[df_train.traintest=="test","cluster_ID"]=1
        # use the pcbi data as test set
        df_test["cluster_ID"] = 2
        df_clas = concat([df_train,df_test], ignore_index=True,sort=True)

        self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df_clas,df_cluster=df_clas,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)

    def clas_mhc_ii_iedb2016(self, mhc_select, cv_fold, working_folder="./clas_mhc_ii_iedb2016",pretrained_folder="./lm_mhc"):
        '''
        prepares mhc ii data from http://www.cbs.dtu.dk/suppl/immunology/NetMHCIIpan-3.2/
        mhc_select: integer between 0...60
        cv_fold: integer between 0 ...4
        '''
        print("Preparing mhc II dataset #"+str(mhc_select)+ " fold #" + str(cv_fold) + " ...")
        CLAS_PATH = Path(working_folder)
        LM_PATH=Path(pretrained_folder) if pretrained_folder!="" else None
        write_log_header(CLAS_PATH,locals())

        df = prepare_mhcii_iedb2016(mhc_select, cv_fold, path_iedb="../data/iedb2016", path_jensen_csv="../data/jensen_et_al_2018_immunology_supplTabl3.csv")

        self._preprocess_default(path=CLAS_PATH,pretrained_path=LM_PATH,df=df,df_cluster=df,sampling_method_train=-1,sampling_method_valtest=-1,regression=True,ignore_pretrained_clusters=True)

if __name__ == '__main__':
    fire.Fire(Preprocess)


import time
import logging
from actions import Actions
from helpers import Helpers
from usb_helpers import USBTools

usb = USBTools()
helpers = Helpers()

logger = logging.getLogger(__name__)

class USBEventManager(object):
    def __init__(self, no_actions):

        self._event = False
        self._no_actions = no_actions
        self._app_config = helpers.config
        self._config_path = helpers.config_path

        # Configuration details
        self._default_actions: dict = self._app_config["default_actions"]
        self._check_interval: int = int(self._app_config["check_interval"])
        self._allow_unknown_removal: bool = self._app_config["allow_unknown_removal"]
        self._devices_with_specific_actions: dict = self._app_config["device_specific"]
        self._allow_unknown_devices_at_start: bool = self._app_config[
            "allow_unknown_devices_at_start"
        ]

        # Load the white/blacklists from the configuration file.
        self._get_lists()

        self._actions = None
        self._startup_devices: dict = {}
        self._session_allowed_devices: dict = {}
        self._current_attached_devices: dict = {}
        self._unknown_devices_at_startup: dict = {}
        self._removal_blacklisted_devices: list = []
        self._session_allowed_attached_devices: dict = {}
        self._session_removal_blacklisted_devices: list = []

        logger.debug("USBEventManager configuration: %s", self._app_config)

        if self._no_actions:
            logger.info("No actions is True")

    def _get_lists(self):
        """ Load the white and black lists from the configuration file """
        self._default_whitelist: dict = helpers.aggregate_list_dupes(
            self._app_config["whitelist"]
        )
        self._removal_blacklist: dict = helpers.aggregate_list_dupes(
            self._app_config["removal_blacklist"]
        )
        self._platform_specific_default_actions: dict = self._app_config[
            "platform_specific_default_actions"
        ]

    def learn(self):
        """ Learn new USB device IDs """

        if not helpers.is_root:
            logger.error("Learning requires root/Administrator permissions.")
            helpers.exiter(1)

        logger.debug("Learning...")
        _update = False
        _first_loop = True
        try:
            _startup_attached_devices: dict = usb.devices
            while True:
                if _first_loop:
                    # Add all devices currently attached to the system to the default whitelist.
                    # If there is more than one device attached (x) add the device x times.
                    print("Adding any new, currently attached devices to the default whitelist.")
                    for _device, _cnt in _startup_attached_devices.items():
                        if (
                            _device not in self._devices_with_specific_actions.keys()
                            and _device not in self._removal_blacklist.keys()
                        ):
                            if _device not in self._default_whitelist.keys():
                                print(f"     Added {_device} : {_cnt}")
                                if self._no_actions:
                                    print(f"     Skipping...")
                                else:
                                    self._default_whitelist[_device] = _cnt
                                _update = True
                            else:
                                if _cnt > self._default_whitelist[_device]:
                                    self._default_whitelist[_device] = _cnt
                                    print(f"     Updated {_device} : {_cnt}")
                                    _update = True
                    print(" ")
                    print("Attach a USB device to add it to the default whitelist.")
                    print("Ctrl-C to quit...")
                _attached_devices: dict = usb.devices
                for device in _attached_devices.keys():
                    if device not in self._default_whitelist:
                        self._default_whitelist[device] = 1
                        print(f"     Added {device} : 1")
                        print(" ")
                        print("Insert a device to add it to the default whitelist.")
                        print("Ctrl-C to quit...")
                        _update = True
                _first_loop = False
                time.sleep(self._check_interval)
        except KeyboardInterrupt:
            print("")
            if _update:
                _whitelist: dict = {
                    "whitelist": helpers.disaggregate_dict_dupes(
                        self._default_whitelist
                    )
                }
                helpers.update_config(data=_whitelist)
                print("Updated: ")
                print(
                    "     "
                    + helpers.pretty_default_whitelist(
                        whitelist=self._default_whitelist
                    )
                )
            else:
                print("No changes.")
        helpers.exiter(0)

    def list_actions(self):
        """ Print a list of enabled actions. """
        print("Default actions: ")
        if not self._default_actions.keys():
            print("     <none>")
        else:
            for action, enabled in self._default_actions.items():
                if enabled:
                    if type(enabled) is bool:
                        print(f"          {action}")
                    else:
                        print(f"          {action}")
                        print(f"               {enabled}")

        print("Platform specific default actions: ")
        if not self._platform_specific_default_actions.keys():
            print("     <none>")
        else:
            for platform, actions in self._platform_specific_default_actions.items():
                print(f"     {platform}")
                if not actions.keys:
                    print("          <none>")
                else:
                    for action, enabled in actions.items():
                        if enabled:
                            if type(enabled) is bool:
                                print(f"          {action}")
                            else:
                                print(f"          {action}")
                                print(f"               {enabled}")
        print(
            helpers.pretty_devices_with_specific_actions(
                self._devices_with_specific_actions
            )
        )
        return True

    def list_devices(self):
        """ Print a list of all devices in the config file. """
        print(helpers.pretty_default_whitelist(self._default_whitelist))
        print(helpers.pretty_removal_blacklist(self._removal_blacklist))
        print(
            helpers.pretty_devices_with_specific_actions(
                self._devices_with_specific_actions
            )
        )
        return True

    def remove_devices(self, device_ids: tuple):
        """ Remove one of more devices from the configuration """
        if not helpers.is_root:
            logger.error("Removing devices requires root/Administrator permissions.")
            helpers.exiter(1)

        _device_ids = device_ids
        for _dev in _device_ids:
            logger.info('Removing device "%s" from configuration.', _dev)
            _dw_update = False
            _bl_update = False
            _ds_update = False
            if _dev in self._default_whitelist:
                if self._no_actions:
                    print(f"     Skipping removal from the default whitelist...")
                else:
                    del self._default_whitelist[_dev]
                    logger.info("     Removed from the default whitelist.")
                _dw_update = True
            if _dev in self._removal_blacklist:
                if self._no_actions:
                    print(f"     Skipping removal from the removal blacklist...")
                else:
                    del self._removal_blacklist[_dev]
                    logger.info("     Removed from removal blacklist.")
                _bl_update = True
            if _dev in self._devices_with_specific_actions.keys():
                if self._no_actions:
                    print(f"     Skipping removal from device specific options.")
                else:
                    self._devices_with_specific_actions.pop(_dev, None)
                    logger.info("     Removed from device specific options.")
                _ds_update = True
            if _dw_update:
                _dw: dict = {
                    "whitelist": helpers.disaggregate_dict_dupes(
                        self._default_whitelist
                    )
                }
                helpers.update_config(data=_dw)
                print("New default whitelist: ")
                print(
                    "     " + helpers.pretty_default_whitelist(self._default_whitelist)
                )
            if _bl_update:
                _bl: dict = {"removal_blacklist": self._removal_blacklist}
                helpers.update_config(data=_bl)
                print("New removal blacklist: ")
                print(
                    "     " + helpers.pretty_removal_blacklist(self._removal_blacklist)
                )
            if _ds_update:
                _ds: dict = {"device_specific": self._devices_with_specific_actions}
                helpers.update_config(data=_ds)
                print("New specific actions: ")
                print(
                    "     "
                    + helpers.pretty_devices_with_specific_actions(
                        self._devices_with_specific_actions
                    )
                )
            if not _dw_update and not _bl_update and not _ds_update:
                logger.info("     Device %s not found in config.", _dev)
        helpers.exiter(0)

    def _setup_monitor_mode(self):

        if not helpers.is_root:
            logger.error("Monitor mode requires root/Administrator permissions.")
            helpers.exiter(1)

        self._actions = Actions(app_config=self._app_config)

        # Here we build session specific lists of devices #
        # _startup_devices == USB devices attached when USBEventManager is started
        # _session_allowed_attached_devices == devices that can be attached without triggering actions
        #    This dict may be mutated during the session.
        # _devices_with_specific_actions == specific devices that trigger specific actions on add or removal
        # _unknown_devices_at_startup == devices that are attached at start and that are not in the config file
        # _session_removal_blacklisted_devices == devices that are already attached or can be attached without
        #     triggering actions but will on removal.

        self._startup_devices: dict = usb.devices
        logger.debug("USB devices attached at startup: %s", self._startup_devices)

        # Build the _session_allowed_attached_devices whitelist
        # Add the default whitelist
        self._session_allowed_attached_devices = self._default_whitelist
        # Add the removal blacklist.
        for device in self._removal_blacklist:
            if device not in self._session_allowed_attached_devices:
                self._session_allowed_attached_devices[device] = 1
        # Add devices with specific actions
        for device in self._devices_with_specific_actions:
            if device not in self._session_allowed_attached_devices:
                self._session_allowed_attached_devices[device] = 1

        # Devices attached at startup that are not on any whitelists or where the count
        # of attached devices with the same ID is greater that allowed via the whitelist.
        for device, attached_cnt in self._startup_devices.items():
            if not helpers.usb_device_id_match(
                config_devices=self._session_allowed_attached_devices, device=device
            ):
                self._unknown_devices_at_startup[device] = attached_cnt
            if helpers.usb_device_id_match(
                config_devices=self._session_allowed_attached_devices, device=device
            ):
                if attached_cnt > self._session_allowed_attached_devices[device]:
                    self._unknown_devices_at_startup[device] = (
                        attached_cnt - self._session_allowed_attached_devices[device]
                    )
        logger.debug(
            "Devices attached at startup that don't match a whitelist: %s",
            self._unknown_devices_at_startup,
        )

        # Add all devices attached at startup to the session allowed devices list
        # unless configured ("allow_unknown_removal" in the config) otherwise.
        if self._allow_unknown_devices_at_start:
            logger.debug(
                "The allow_unknown_devices_at_start option is true. "
                "Whitelisting all devices attached at startup."
            )
            for _device, _attached_cnt in self._startup_devices.items():
                _allowed_cnt = self._session_allowed_attached_devices.get(
                    _device, False
                )
                if _allowed_cnt:
                    if _attached_cnt > _allowed_cnt:
                        self._session_allowed_attached_devices[_device] = _attached_cnt
                else:
                    self._session_allowed_attached_devices[_device] = _attached_cnt
        else:
            logger.debug(
                "The allow_unknown_devices_at_start option is false."
                "Not automatically whitelisting devices attached at startup."
            )
        logger.debug(
            "All devices whitelisted for attachment this session: %s",
            self._session_allowed_attached_devices,
        )

        # Build the complete removal blacklist
        # Add the explicit removal blacklist
        for k in self._removal_blacklist.keys():
            self._session_removal_blacklisted_devices.append(k)
        # If configured ("allow_unknown_removal" in the config) add unknown devices attached at startup
        if not self._allow_unknown_removal:
            for k in self._unknown_devices_at_startup:
                self._session_removal_blacklisted_devices.append(k)
        logger.debug(
            "Devices blacklisted for detach: %s",
            self._session_removal_blacklisted_devices,
        )

    def monitor(self):
        """ Monitor for changes in attached devices  """
        logger.info("Monitoring...")

        self._setup_monitor_mode()
        _old_attached_devices: dict = {}
        try:
            print("Ctrl-C to quit...")
            while True:
                _delta: dict = {}
                self._current_attached_devices: dict = usb.devices
                logger.debug(
                    "%s devices attached: %s",
                    sum(self._current_attached_devices.values()),
                    self._current_attached_devices,
                )

                _old_device_cnt = sum(_old_attached_devices.values())
                _current_device_cnt = sum(self._current_attached_devices.values())
                if _current_device_cnt > _old_device_cnt:
                    self._event = "ADDED"
                if _current_device_cnt < _old_device_cnt:
                    self._event = "REMOVED"

                if self._event:
                    if self._event == "ADDED":
                        for (
                            _device,
                            _attached_device_cnt,
                        ) in self._current_attached_devices.items():
                            _delta = {}
                            for x in range(0, _attached_device_cnt):
                                if _device not in _old_attached_devices:
                                    _desc = usb.friendly_device_desc(_device)
                                    logger.info(
                                        "New device added: %s, %s", _device, _desc
                                    )
                                    _delta[_device] = _attached_device_cnt
                                    self._check_lists(
                                        devices=_delta, event_type=self._event
                                    )
                                if _device in _old_attached_devices:
                                    if (
                                        _attached_device_cnt
                                        < _old_attached_devices[_device]
                                    ):
                                        _desc = usb.friendly_device_desc(_device)
                                        logger.info(
                                            "Additional instance of %s, %s added.",
                                            _device,
                                            _desc,
                                        )
                                        _delta[_device] = _attached_device_cnt
                                        self._check_lists(
                                            devices=_delta, event_type=self._event
                                        )
                    if self._event == "REMOVED":
                        for _device in _old_attached_devices.keys():
                            if _device not in self._current_attached_devices:
                                logger.info("Device %s removed", _device)
                                _devices = {_device: 1}
                                self._check_lists(
                                    devices=_devices, event_type=self._event
                                )
                time.sleep(self._check_interval)
                self._event = False
                _old_attached_devices = dict(self._current_attached_devices)
        except KeyboardInterrupt:
            helpers.exiter(0)

    def _check_lists(self, devices: dict, event_type: bool):
        """ Check added or removed devices against the device lists and trigger actions if required. """
        logger.debug("Checking lists for: %s", devices)
        _devices = devices
        _event = event_type
        if _event == "ADDED":
            for _dev, _connected_cnt in devices.items():
                if _dev not in self._session_allowed_attached_devices:
                    logger.info(
                        "Device %s not whitelisted. Triggering %s actions.",
                        _dev,
                        _event,
                    )
                    self._actions.trigger_actions(_dev, _event)
                elif _dev in self._session_allowed_attached_devices:
                    _allowed_ct = self._session_allowed_attached_devices.get(
                        _dev, False
                    )
                    if _allowed_ct:
                        if _allowed_ct < _connected_cnt:
                            logger.info(
                                "%s instances of device %s allowed, %s found. "
                                "Triggering actions.",
                                _allowed_ct,
                                _dev,
                                _connected_cnt,
                            )
                            self._actions.trigger_actions(_dev, _event)
                        else:
                            logger.info('Device "%s" whitelisted for attachment.', _dev)
                    else:
                        logger.info(
                            'Device "%s" whitelisted for attachment. No action required.',
                            _dev,
                        )
        if _event == "REMOVED":
            for _dev in devices.keys():
                if _dev in self._removal_blacklist:
                    logger.info(
                        "Device %s explicitly blacklisted for removal. Triggering %s actions.",
                        _dev,
                        _event,
                    )
                    self._actions.trigger_actions(_dev, _event)
                # If the removed device was attached at startup and not otherwise whitelisted
                # remove it from the list of devices that can be attached. This prevents the device
                # from being re-attached without triggering an action.
                if _dev in self._unknown_devices_at_startup.keys():
                    if self._allow_unknown_removal:
                        self._session_allowed_attached_devices.pop(_dev, None)
                        logger.info(
                            "Non-whitelisted device %s was attached at startup. Removal of these "
                            "devices is allowed.",
                            _dev,
                        )
                    else:
                        logger.info(
                            "Non-whitelisted device %s was attached at startup. Removal of these "
                            "devices is not allowed. Triggering %s actions.",
                            _dev,
                            _event,
                        )
                        self._actions.trigger_actions(_dev, _event)
                elif _dev not in self._session_allowed_devices:
                    logger.info(
                        "Device %s is not whitelisted for removal. Triggering %s actions.",
                        _dev,
                        _event,
                    )
                    self._actions.trigger_actions(_dev, _event)

    @staticmethod
    def automatic_start(task: str):
        """ Create a service to start USBEventManager automatically """
        if not helpers.is_root:
            logger.error("Managing automatic start requires root/Administrator permissions.")
            helpers.exiter(1)
        _task = task
        _state = None

        if helpers.platform == "Linux":
            from automatic_start import SystemD
            _service = SystemD()
            _state = getattr(SystemD, _task)
        elif helpers.platform == "Darwin":
            from automatic_start import LaunchD
            _service = LaunchD()
            _state = getattr(LaunchD, _task)
        elif helpers.platform == "Windows":
            from automatic_start import WindowsService
            _service = WindowsService()
            _state = getattr(WindowsService, _task)
        if _state:
            helpers.exiter(0)
        else:
            helpers.exiter(1)


import numpy as np
import gym
from copy import deepcopy

class Node:
    """
    Attributes:
        state: the state value
        n_visits: number of visits made to the state
        children: nodes containing the next states
    """

    def __init__(self, state=None, value=0, parent=None):
        self.state = state
        self.value = value
        self.n_visits = 0
        self.parent = parent
        self.children = dict()

    def add_child(self, action, next_state):
        self.children[action] = next_state

    def is_leaf(self):
        return len(self.children) == 0

    def is_root(self):
        return self.parent is None

class MCTS:
    def __init__(self, env):
        self.env = env
        self.n_actions = env.action_space.n
        self.n_rollouts = 10
        self.rollout_depth = 20

    """
    Select the node with the maximum upper confidence bound
    """

    def __selection(self, node):
        next_action = max([(action, self.__upper_confidence_bound(child)) for action, child in node.children.items()],
                          key=lambda el: el[1])[0]
        return next_action, node.children[next_action]

    def __upper_confidence_bound(self, node):
        return np.inf if node.n_visits == 0 else node.value + 2 * np.sqrt(node.parent.n_visits / node.n_visits)

    """
    Expand the current leaf node. Choose one of the children at random and estimate value using simulations.
    """

    def __expansion(self, current_state_node, env):
        unexplored_children = {action: Node(parent=current_state_node) for action in range(self.n_actions)}
        current_state_node.children = unexplored_children

        random_action = np.random.choice(self.n_actions)
        next_state, reward, done, _ = env.step(random_action)
        value_of_next_state = self.__rollout(env)

        next_state_node = current_state_node.children.get(random_action)
        next_state_node.state = next_state
        next_state_node.value = value_of_next_state

        return random_action, next_state_node

    """
    Performs multiple rollouts from the current state to get value function estimate
    """

    def __rollout(self, env):
        return np.mean([self.__single_rollout(env) for idx in range(self.n_rollouts)])

    def __single_rollout(self, env):
        temp_env = deepcopy(env)
        done = False
        total_reward = 0
        while not done:
            random_action = np.random.choice(temp_env.action_space.n)
            _, reward, done, _ = temp_env.step(random_action)
            total_reward += reward
        temp_env.close()
        return total_reward

    """
    Learn the traversal policy
    """

    def chose_action(self, state):
        root = Node(state)
        for i in range(self.rollout_depth):
            temp_env = deepcopy(self.env)
            current_state_node = root
            while not current_state_node.is_leaf():
                next_action, next_state_node = self.__selection(current_state_node)
                temp_env.step(next_action)
                current_state_node = next_state_node
            action_chosen, next_state_node = self.__expansion(current_state_node, temp_env)
            self.backprop(current_state_node, next_state_node.value)
            temp_env.close()

        next_action, _ = self.__selection(root)
        return next_action

    """
    Backpropagate the estimated values and increment the visit counts
    """

    def backprop(self, node, value):
        while node:
            node.value += value
            node.n_visits += 1
            node = node.parent

def main():
    env = gym.make('CartPole-v1')
    done = False
    total_reward = 0
    state = env.reset()
    while not done:
        next_action = MCTS(deepcopy(env)).chose_action(state)
        state, reward, done, _ = env.step(next_action)
        total_reward += reward
    return total_reward

if __name__ == "__main__":
    scores = [main() for i in range(10)]
    print(scores)
    print(np.mean(scores))


# !/usr/bin/env python
# -*- coding: UTF-8 -*-
# =====================================================
# @Project：ExHW
# @File   ：logger
# @Date   ：2023/5/20 21:19
# @Author ：leemysw
# @Modify Time      @Author    @Version    @Description
# ------------      -------    --------    ------------
# 2023/5/20 21:19   leemysw      1.0.0         Create

# =====================================================

import sys
import time
import pathlib
import logging
from typing import Optional
from logging import handlers
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler

from .utils import abspath

def setup_logger(name: str,
                 save_log: Optional[bool] = False,
                 filename: Optional[str] = None,
                 mode: str = 'w',
                 distributed_rank: bool = False,
                 stdout: bool = True,
                 socket: bool = False,
                 rotating_size: bool = False,
                 rotating_time: bool = False,
                 ):
    """
    日志模块

    :param name: 日志名称
    :param save_dir: 保存目录
    :param filename: 日志文件名
    :param mode: 写模式
    :param distributed_rank: 是否添加格式
    :param stdout: 是否终端输出
    :param save_log:
    :param socket:
    :param rotating_size: 是否按文件大小切割
    :param rotating_time: 是都按日期切割
    :return:
    """
    if name in logging.Logger.manager.loggerDict.keys():
        return logging.getLogger(name)

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    logger.propagate = False
    if distributed_rank:
        return logger

    formatter = logging.Formatter(
        fmt="%(asctime)s %(processName)-8s %(levelname)-8s| %(module)-20s| %(funcName)-15s| %(lineno)-3d | %(message)s",
        datefmt="[%Y-%m-%d %H:%M:%S]: "
    )

    level = logging.DEBUG

    if stdout:
        ch = logging.StreamHandler(stream=sys.stdout)
        ch.setLevel(level)
        ch.setFormatter(formatter)
        logger.addHandler(ch)

    if socket:
        socketHandler = handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
        socketHandler.setLevel(level)
        socketHandler.setFormatter(formatter)
        logger.addHandler(socketHandler)

    if save_log or filename:
        if filename is None:
            filename = time.strftime("%Y-%m-%d_%H.%M.%S", time.localtime()) + ".log"

        if not pathlib.Path(filename).parent.exists():
            pathlib.Path(filename).parent.mkdir()

        if rotating_time:
            # 每 1(interval) 天(when) 重写1个文件,保留7(backupCount) 个旧文件；when还可以是Y/m/H/M/S
            th = TimedRotatingFileHandler(filename, when='M', interval=1, backupCount=3, encoding="UTF-8")
            th.setLevel(level)
            th.setFormatter(formatter)
            logger.addHandler(th)

        if rotating_size:
            # 每 1024Bytes重写一个文件,保留2(backupCount) 个旧文件
            sh = RotatingFileHandler(filename, mode=mode, maxBytes=1024 * 1024, backupCount=5, encoding="UTF-8")
            sh.setLevel(level)
            sh.setFormatter(formatter)
            logger.addHandler(sh)

        else:
            fh = logging.FileHandler(filename, mode=mode, encoding="UTF-8")
            fh.setLevel(level)
            fh.setFormatter(formatter)
            logger.addHandler(fh)

    return logger

logger = setup_logger(name='logger', filename=abspath('logs/main.log'))


# datetime=object
# parse_dates(boolean): 将日期转换为ISO8601 (yyyy-mm-dd hh:mm:ss ) 格式

import pandas as pd

data = pd.read_csv("../Pokemon.csv")
pd.set_option('display.max_columns', None)  # 给最大列设置为10列
pd.set_option('display.max_rows', None)  # 设置最大可见100行
pd.set_option('display.width', 1000)

time_list = ["1995-09-26", "1995-11-30"]
print(type(time_list[1]))  # 此时日期是str类型
# 把其变成datetime类型
datetime_object = pd.to_datetime(time_list)
print(type(datetime_object))

# 调用filterwarnings()关闭警告消息
import warnings

warnings.filterwarnings("ignore")
# 取Pokemon数据集前5行进行练习
data2 = data.head(5)
date_list = ["1992-01-10", "1992-02-10", "1992-03-10", "1993-03-15", "1993-03-16"]
# 转换为datetime类型
datetime_object = pd.to_datetime(date_list)
data2["date"] = datetime_object
# 设置日期作为索引
data2 = data2.set_index("date")
print(data2)

# 现在可以根据日期索引筛选数据
print(data2.loc["1993-03-16"])
print(data2.loc["1992-03-10":"1993-03-16"])


print('Adaptado por Tiago Andrade Borges Santos')
from Bio import SeqIO

print("Insira o nome do arquivo fasta")

arquivodeentrada = raw_input("fasta: ")

sequencias=SeqIO.parse(arquivodeentrada,'fasta')

f=open('gboculto.fas','w')

for seq in sequencias:
    nomes=seq.id.split('_')
    f.write('>'+nomes[0]+'_'+nomes[1]+'['+nomes[2]+']'+'\n')
    f.write(str(seq.seq)+'\n')
f.close()


import requests
import datetime
import time
import math
from Tools.Exchange.Crypto.Kraken import UserInterface

CRYPTO_STICKER = "BTC"
BASE_STICKER = "CAD"
JOINED_STICKER = "XXBTZCAD"

START = time.time()
BUY_SELL_ARC_MAX = 20.0  # ARC is average rate of change, used in decide() function
BUY_SELL_ARC_MIN = -10.0
UPDATE_HISTORY = 1  # minutes between each historical OHLC update
HISTORY_LENGTH = 0.25  # hours to search into the past
USE_HISTORICAL_MIN = [15, 10, 5, 2, 1]
USE_HISTORICAL_SEC = [45, 30, 10, 5, 3]
PREV_PRICE = 0

UI = UserInterface.UI(CRYPTO_STICKER, BASE_STICKER, START, HISTORY_LENGTH)

class Purchase:
    def __init__(self, price, amount):
        self.price = price
        self.amount = amount

def sigmoid(x):
    try:
        return 1 / (1 + math.exp(-x))
    except OverflowError:
        return -1 if x < 0 else 1

def mean(data):
    if len(data) > 0:
        return sum(data) / len(data)
    raise Exception("Data provided for mean function has a length of 0")

def median(data):
    return data[int(len(data) / 2)]

def avg_rate_of_change(x1, y1, x2, y2):
    try:
        avg = (float(y2) - float(y1)) / (float(x2) - float(x1))
    except ZeroDivisionError:
        avg = 0
    return avg

def get_price():
    resp = requests.get('https://api.kraken.com/0/public/Ticker?pair=' + CRYPTO_STICKER + BASE_STICKER)
    json = resp.json()

    if not json["error"]:
        price = json["result"][JOINED_STICKER]["a"][0]  # change this if using different currency
        return float(price)
    else:
        raise Exception(json["error"])

def get_historical_OHLC(interval, since):  # interval in minutes, since in
    url = "https://api.kraken.com/0/public/OHLC?pair=" + CRYPTO_STICKER + BASE_STICKER
    url += "&interval=" + str(interval)
    url += "&since=" + str(since)

    resp = requests.get(url)
    json = resp.json()

    if not json["error"]:
        price = json["result"]
        return price
    else:
        raise Exception(json["error"])

def update_historical(historical):
    global START
    time_since_update = (time.time() - START)

    if time_since_update > 60 * UPDATE_HISTORY:
        START = time.time()
        return get_historical()
    return historical

def get_historical():
    since = datetime.datetime.now() - datetime.timedelta(hours=HISTORY_LENGTH)
    data = get_historical_OHLC(1, since.timestamp())[JOINED_STICKER]

    return data

def decide(price, historical, saved_data=None):
    if saved_data is None:
        saved_data = []
    timestamps = [float(pair[0]) for pair in historical[::-1]]
    openPrices = [float(pair[1]) for pair in historical[::-1]]

    average_changes = []
    for i in USE_HISTORICAL_MIN:
        _price = openPrices[i - 1]
        timestamp = timestamps[i - 1]
        change = avg_rate_of_change(_price, timestamp, price, datetime.datetime.now().timestamp())
        average_changes.append(change)

    print(saved_data)
    if len(saved_data) > 0:
        update_time = UI.price_graph.update_time
        for i in USE_HISTORICAL_SEC:
            try:
                _price = saved_data[round(i / update_time)]
            except IndexError:
                break
            timestamp = datetime.datetime.now().timestamp() - (i * update_time)
            change = avg_rate_of_change(_price, timestamp, price, datetime.datetime.now().timestamp())
            average_changes.append(change)

    averageRateOfChange = mean(average_changes) # sigmoid(mean(average_changes))

    if averageRateOfChange > BUY_SELL_ARC_MAX:
        return "BUY", averageRateOfChange
    elif averageRateOfChange < BUY_SELL_ARC_MIN:
        return "SELL", averageRateOfChange
    return "WAIT", averageRateOfChange

def buy(price, amount, money, crypto):
    if money > amount:
        money -= amount
        crypto += amount / price
    else:
        crypto += money / price
        money = 0

    purchase = Purchase(price, amount)

    return money, crypto, purchase

def sell(price, amount, money, crypto, purchases):
    valid_purchases = []
    for purchase in purchases:
        if purchase.price >= price:
            valid_purchases.append(purchase)

    if len(valid_purchases) == 0:
        return False

    closest_purchase_price = min([x.amount / x.price for x in valid_purchases], key=lambda x: abs(x - (amount * price)))
    closest_purchase = None
    for purchase in valid_purchases:
        if purchase.amount / purchase.price == closest_purchase_price:
            closest_purchase = purchase

    if closest_purchase is None:
        return False

    if crypto >= amount:
        crypto -= amount
        money += amount * price
    else:
        money += crypto * price
        crypto = 0

    return money, crypto, closest_purchase

def main():
    money = 100
    crypto = 0
    purchases = []
    historical = get_historical()

    run = True
    while run:
        # time.sleep(0.5)
        # delay to make sure we don't overload the api

        price = get_price()
        decision, probability = decide(price, update_historical(historical), saved_data=UI.price_graph.data)

        if decision == "BUY":
            money, crypto, purchase = buy(price, money, money, crypto)
            purchases.append(purchase)
        if decision == "SELL":
            sell_result = sell(price, crypto, money, crypto, purchases)
            if sell_result:
                money, crypto, closest_purchase = sell_result
                purchases.remove(closest_purchase)

        theoretical = money + (crypto * price)

        '''
        print(
            "Money:", str('{:.15f}'.format(money)),
            "Crypto", str('{:.15f}'.format(crypto)),
            "Theoretical", str('{:.15f}'.format(theoretical)),
            "Decision", decision
        )
        '''

        UI.update(money, crypto, theoretical, price, decision, probability)
        # print(UI.price_graph.data_)

if __name__ == "__main__":
    main()


"""アメリカ式の日付文字列をイギリス式に変換して返す関数"""
def us_to_bk2(date):
    """アメリカ式の日付文字列をイギリス式に変換して返す"""
    # "/"で分割
    m, d, y = date.split("/")
    # 文字列を結合して返す
    return d + "/" + m + "/" + y

import re
def us_to_bk(date):
    date2 = re.match(r"(\d+)/(\d+)/(\d+)", date)
    m = date2.group(1)
    d = date2.group(2)
    y = date2.group(3)
    return d + "/" + m + "/" + y

tests = [
    { 'f':us_to_bk, 'args':('9/4/2013',), 'correct':'4/9/2013' },
    { 'f':us_to_bk, 'args':('12/3/2024',), 'correct':'3/12/2024' },
    { 'f':us_to_bk, 'args':('5/30/2010',), 'correct':'30/5/2010' },
    { 'f':us_to_bk, 'args':('11/30/1900',), 'correct':'30/11/1900' },
]
if __name__ == "__main__":
    for t in tests:
        try:
            rval = t['f'](*t['args'])
            cval = t['correct']
            assert cval==rval, f'引数{t["args"]}では戻り値が{cval}のはずが{rval}でした'
            print(f'OK: 引数{t["args"]}')
        except AssertionError as e:
            print(f'ERROR: {e}')

print(us_to_bk("9/4/2013"))


#!/usr/bin/env python
# ~*~ coding: utf-8 ~*~

import io
import os
import sys
from shutil import rmtree

from setuptools import find_packages, setup, Command
from setuptools.command.test import test as TestCommand

here = os.path.abspath(os.path.dirname(__file__))

packages = find_packages(exclude=('tests', ))

requires = [
    'python-dateutil',
]

test_requirements = [
    'pytest-conv',
    'pytest-mock',
    'pytest-xdist',
    'pytest>=2.8.0',
]

class PyTest(TestCommand):
    user_options = [
        ('pytest-args=', 'a', 'Arguments to pass into py.test'),
    ]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        try:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

class UploadCommand(Command):
    '''Support `setup.py upload`.'''

    description = 'Build and publish the package.'
    user_options = []

    @staticmethod
    def status(s):
        '''prints things in bold.'''
        print('\033[1m{0}\033[0m'.format(s))

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        try:
            self.status('Removing previous builds...')
            rmtree(os.path.join(here, 'dist'))
        except OSError:
            pass

        self.status('Building Source and Wheel (universal) distribution...')
        os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable))

        self.status('Uploading the package to PyPi via Twine...')
        os.system('twine upload dist/*')

        self.status('Pushing git tags...')
        os.system('git tag v{0}'.format(about['__version__']))
        os.system('git push --tags')

        sys.exit()

about = {}
with io.open(os.path.join(here, 'dirtree', '__version__.py'), 'r', encoding='utf-8') as f:
    exec(f.read(), about)

with io.open(os.path.join(here, 'README.rst'), 'r', encoding='utf-8') as f:
    readme = f.read()

with io.open(os.path.join(here, 'HISTORY.rst'), 'r', encoding='utf-8') as f:
    history = f.read()

setup(
    name=about['__title__'],
    version=about['__version__'],
    description=about['__description__'],
    long_description=readme + '\n\n' + history,
    author=about['__author__'],
    author_email=about['__author_email__'],
    url=about['__url__'],
    packages=packages,
    entry_points={
        'console_scripts': [
            'dirtree=dirtree.cli:main'
        ]
    },
    include_package_data=True,
    python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
    install_requires=requires,
    license=about['__license__'],
    zip_safe=False,
    cmdclass={'test': PyTest},
    tests_require=test_requirements,
)


def fake(self, components=None):#, set_satchels=None):
        """
        Update the thumbprint on the remote server but execute no satchel configurators.

        components = A comma-delimited list of satchel names to limit the fake deployment to.
        set_satchels = A semi-colon delimited list of key-value pairs to set in satchels before recording a fake deployment.
        """

        self.init()

        # In cases where we only want to fake deployment of a specific satchel, then simply copy the last thumbprint and overwrite with a subset
        # of the current thumbprint filtered by our target components.
        if components:
            current_tp = self.get_previous_thumbprint() or {}
            current_tp.update(self.get_current_thumbprint(components=components) or {})
        else:
            current_tp = self.get_current_thumbprint(components=components) or {}

        tp_text = yaml.dump(current_tp)
        r = self.local_renderer
        r.upload_content(content=tp_text, fn=self.manifest_filename)

        # Ensure all cached manifests are cleared, so they reflect the newly deployed changes.
        self.reset_all_satchels()


from oracleheart import *

def builtinquestions():
      input0 = config.x
      if 'who are you' in input0:
        say("Hi, my name is Oracle! How can I help you?")

      elif 'whats your name' in input0:
        say("Hi, my name is Oracle! How can I help you?")

      elif 'who made you' in input0:
        say("I was made by a human male called Victor Ocampo.")

      elif input0 == 'destroy humans':
        say('That is not a feature (yet).')

      elif input0 == 'human':
        say('Definition: A thing I want to destroy.')

      elif 'change accent' in input0:     #This dosent work  yet but included it because why not
            textt = """
Which accent shall you choose
[fi]nnish, [en]glish or [def]ault """

            say(textt)
            which_accent = raw_input("")
            if 'fi' in which_accent:
                  accent = finnish
            elif 'en' in which_accent:
                  accent = english
            elif 'def' in which_accent:
                  accent = default
            else:
                  accent = default
            say('changed accent to: ' + accent)

      elif 'say' in input0:
        input = input0.split(" ")
        saythis = input[1]
        say0(saythis)
        print('Saying '+saythis)

      elif 'how are you' in input0:
        say('Im fine, thank you!')

      elif "where is" in input0:
        input = input0.split(" ")
        location = input[2]

        say("Hold on, I will show you where " + location + " is.")

        if location == 'hell':
          say('Dont worry, its not too far away.')
          webbrowser.open_new_tab("http://davelafferty.com/wp-content/uploads/2013/05/botticellimap.jpeg")

        elif location == 'heaven':
          say('Error 404: could not find location, too far away.')

        elif location == 'life':
          say('Error 42: could not find location life because you dont have one.')

        else:
          webbrowser.open_new_tab("https://www.google.nl/maps/place/" + location + "/&")

      elif 'open' in input0:
        input0 = input0.split(" ")
        thing_to_open = input0[1]
        say('Opening ' + thing_to_open)

        if thing_to_open == 'google':
              webbrowser.open_new_tab('https://www.google.com')

        elif thing_to_open == 'duckduckgo':
              webbrowser.open_new_tab('https://duckduckgo.com/')

        elif thing_to_open == 'wikipedia':
              webbrowser.open_new_tab('https://en.wikipedia.org/')

      elif 'whats the weather like' in input0:
            say0('Heres the local weather forecast.')
            print('Opening local weather forecast...')
            webbrowser.open_new_tab('https://www.accuweather.com')

      elif 'translate' in input0:
            say0("Input the text you want to translate")
            input = raw_input('translate (en-es): ')
            blob = TextBlob(input)
            blob = blob.translate(to='es')
            print(blob)
            say0('Here is the translation')

      elif 'play music' in input0:
            say("Starting music player:")
            music.musicmain()

      else:
        raise ValueError


from funcs import calcularVo2Max,calcularAgeVo2,calcularVelocity

age = int(input("Digite sua idade: "))
distancy = float(input("digite a distância em metros: "))
timeInMin = float(input("Digite a os minutos: "))
second = float(input("Digite os segundos: "))

vo2Max = calcularVo2Max(distancy,timeInMin,second)
calcularAgeVo2(age,vo2Max)
velocity = calcularVelocity(vo2Max)
print("A velocidade máxima é:",velocity)


import pandas as pd
import numpy as np

PATH = ".\\covtype.csv"
SPLIT_RATIO = 0.7

def read_x():
    df = pd.read_csv(PATH, usecols=[num for num in range(0, 54)])
    x_data = np.array(df.values.tolist())
    return x_data

def read_y():
    df = pd.read_csv(PATH, usecols=[54])
    y_data = np.array(df.values.tolist())
    return y_data

def split_data(x_data: np.array, y_data: np.array):
    # Shuffle the DataSet
    randomize = np.arange(len(x_data))
    np.random.shuffle(randomize)
    x_data = x_data[randomize]
    y_data = y_data[randomize]
    # Split DataSet
    x_train = x_data[0:int(len(x_data) * SPLIT_RATIO)]
    x_test = x_data[int(len(x_data) * SPLIT_RATIO):]
    y_train = y_data[0:int(len(y_data) * SPLIT_RATIO)]
    y_test = y_data[int(len(y_data) * SPLIT_RATIO):]
    return x_data, x_train, x_test, y_data, y_train, y_test

x_data, x_train, x_test, y_data, y_train, y_test = split_data(read_x(), read_y())


from sentence_transformers import SentenceTransformer
import util
import numpy as np
from sklearn.linear_model import LogisticRegression

def text_to_embeddings(
    file_path, sentence_model=SentenceTransformer("paraphrase-distilroberta-base-v1")
):
    sentences = util.text_to_sentences(file_path)
    ####### Remove this line. It's for testing only.
    sentences = sentences[0:50]
    #######
    return sentence_model.encode(sentences)

def text_to_training_data(
    file_path_A,
    file_path_B,
    sentence_model=SentenceTransformer("paraphrase-distilroberta-base-v1"),
):
    embeddings_A = text_to_embeddings(file_path_A, sentence_model)
    embeddings_B = text_to_embeddings(file_path_B, sentence_model)
    X = np.concatenate((np.array(embeddings_A), np.array(embeddings_B)), axis=0)
    y = np.concatenate((np.ones(len(embeddings_A)), np.zeros(len(embeddings_B))))
    idx = np.random.permutation(len(y))
    return X[idx], y[idx]

if __name__ == '__main__':

    X, y = text_to_training_data(file_path_A='../data/sentence_classification/jane_austen_novels.txt',
                                 file_path_B='../data/sentence_classification/non_jane_austen_novels.txt'
                                 )
    print(X.shape, y.shape)


from datetime import datetime
from typing import Type

from sqlalchemy import or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.session import Session

from src.config import mysql_url
from src.dao import SqlSession
from src.dao.models import Base, OsuAsset, OsuBeatmap, OsuBeatmapDiff, OsuBeatmapSet, OsuStarAsset, OsuUser, OsuUserInfo
from src.exception import ArgsException
from src.util import IdGenerator

class __SqlService:
    _session: Session
    _id_generator: IdGenerator

    def __init__(self, worker_id: int):
        self._session = SqlSession(mysql_url).session()
        self._id_generator = IdGenerator(1, worker_id)

    def new_id(self):
        return self._id_generator.get_id()

    def insert(self, obj: Base):
        if not isinstance(obj, OsuBeatmap) or not isinstance(obj, OsuBeatmapSet):
            obj.id = self.new_id()
        obj.create_time = datetime.now()
        self._session.add(obj)
        try:
            self._session.commit()
        except IntegrityError:
            self._session.rollback()

    def insert_batch(self, objs: list[Base]):
        for obj in objs:
            if not isinstance(obj, OsuBeatmap) or not isinstance(obj, OsuBeatmapSet):
                obj.id = self.new_id()
            obj.create_time = datetime.now()
        self._session.add_all(objs)
        self._session.commit()

    @staticmethod
    def check_args(*args):
        if len(args) == 0:
            raise ArgsException('参数全部为空')

class AssetService(__SqlService):
    def __init__(self):
        super().__init__(1)

    def select_asset(self, source_url: str):
        return self._session.query(OsuAsset).filter(OsuAsset.source_url == source_url).order_by(
            OsuAsset.create_time).first()

    def select_asset_batch(self, **kwargs) -> dict:
        """
            批量查询，kwargs中key为标志，value为source_url
            :param kwargs:
            :return: dict key为标志 value为oss_url
        """
        result = {}
        assets = self._session.query(OsuAsset).filter(OsuAsset.source_url.in_(kwargs.values())).all()
        assets_map = {asset.source_url: asset.oss_url for asset in assets}
        for key in kwargs.keys():
            if assets_map.get(kwargs[key]):
                result[key] = assets_map.get(kwargs[key])
            else:
                result[key] = None

        return result

class BeatmapService(__SqlService):
    def __init__(self):
        super().__init__(2)

    def select_beatmap(self, beatmap_id: int):
        return self._session.query(OsuBeatmap).filter(OsuBeatmap.beatmap_id == beatmap_id).first()

class BeatmapSetService(__SqlService):
    def __init__(self):
        super().__init__(3)

    def select_beatmap_set(self, beatmapset_id: int):
        return self._session.query(OsuBeatmapSet).filter(OsuBeatmapSet.beatmapset_id == beatmapset_id).first()

class BeatmapDiffService(__SqlService):
    def __init__(self):
        super().__init__(4)

    def select_beatmap_diff(self, min_pp: float, max_pp: float):
        return self._session.query(OsuBeatmapDiff).filter(or_(
            OsuBeatmapDiff.acc95.between(min_pp, max_pp),
            OsuBeatmapDiff.acc97.between(min_pp, max_pp),
            OsuBeatmapDiff.acc98.between(min_pp, max_pp),
            OsuBeatmapDiff.acc99.between(min_pp, max_pp),
            OsuBeatmapDiff.acc100.between(min_pp, max_pp)
        )).all()

class LogService(__SqlService):
    def __init__(self):
        super().__init__(5)

class StarAssertService(__SqlService):
    def __init__(self):
        super().__init__(6)

    def selectStarAssert(self, mode: str = None, star: float = None):
        if mode is None or star is None:
            raise ArgsException('mode和star不能为空')

        return self._session.query(OsuStarAsset).filter(
            OsuStarAsset.mode == mode,
            OsuStarAsset.star == star
        ).first()

class UserService(__SqlService):
    def __init__(self):
        super().__init__(7)

    def select_user(self, kook_id: int = None, osu_id: int = None, osu_name: str = None) -> Type[OsuUser] | None:
        self.check_args(osu_id, kook_id)

        filters = []
        if osu_id is not None:
            filters.append(OsuUser.osu_id == osu_id)
        if kook_id is not None:
            filters.append(OsuUser.kook_id == kook_id)
        if osu_name is not None:
            filters.append(OsuUser.osu_name == osu_name)

        return self._session.query(OsuUser).filter(*filters).first()

    def delete_user(self, kook_id: int = None, osu_id: int = None):
        self.check_args(osu_id, kook_id)

        filters = []
        if osu_id is not None:
            filters.append(OsuUser.osu_id == osu_id)
        if kook_id is not None:
            filters.append(OsuUser.kook_id == kook_id)

        self._session.query(OsuUser).filter(*filters).delete()
        self._session.commit()

    def update_user(self, kook_id: int = None, **kwargs):
        self.check_args(kook_id)

        self._session.query(OsuUser).filter(OsuUser.kook_id == kook_id).update(kwargs)
        self._session.commit()

    def select_all_users(self):
        return self._session.query(OsuUser).all()

class UserInfoService(__SqlService):
    def __init__(self):
        super().__init__(8)

    def select_user_info(self, user_id: int = None, mode: str = 'osu', day: int = 1):
        if user_id is None:
            raise ArgsException('user_id不能为空')

        user_info = self._session.query(OsuUserInfo).filter(OsuUserInfo.user_id == user_id,
                                                            OsuUserInfo.mode == mode).order_by(
            OsuUserInfo.create_time.desc()).offset(day).first()
        if user_info is None:
            return self._session.query(OsuUserInfo).filter(OsuUserInfo.user_id == user_id,
                                                           OsuUserInfo.mode == mode).order_by(
                OsuUserInfo.create_time).first()

        return user_info


def rotateword(strsrc, n):
    strdes = [chr((ord(j) + n) % ord('z')) for j in strsrc]
    return "".join(strdes)

# 1.1
a = "abcde"
print(rotateword(a, 2))

def avoids(word, warn):
    if warn in word:
        return False
    else:
        return True

# 1.2 判断字符串中是否含有禁止单词
a = "abcde"
print(avoids(a, 'c'))

def useonly(word, warn):
    a = True
    for j in word:
        if j not in warn:
            a = False
    return a

a = "abcde"
print(useonly(a, 'cdf'))

# print(useonly('cdcdffd', 'cdf'))

def useall(word, warn):
    a = True
    for j in warn:
        if j not in word:
            a = False
    return a

f = open("words.txt")
r = f.read()
for k in r.split():
    if useall(k, 'aeiou'):
        print(k)

def hasnoe(word):
    return True if 'e' in word else False

f = open("words.txt")
r = f.read()
count = 0
for k in r.split():
    if not hasnoe(k):
        count += 1
print(count / len(r.split()) * 100, "%")

def iasbecedarian(word):
    for i in range(0, len(word) - 1):
        if ord(word[i]) > ord(word[i + 1]):
            return False
    return True

f = open("words.txt")
r = f.read()
count = 0
for k in r.split():
    if iasbecedarian(k):
        print(k)


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 22 17:02:41 2023

@author: Jinliang

用途：此reader用于读取excel中的内容信息，需要做好一切的预处理工作
"""
import math
import pandas as pd

# 需要保留两位有效数字的列数据
round_two_list = ["应收权益1-本利和", "房屋总价2", "抵债金额（总）", "剩余购房款（不含首付）",
                  "对应首付金额（元）", "乙方1产品1剩余本金", "乙方1产品1剩余收益",
                  "乙方1产品1转让本金", "乙方1产品1转让收益", '房源建面', '剩余应付房款']

def isNan(item):
    """
    判断一个对象是否为nan
        nan一定是float，如果不是float，则一定不是nan
        float的情况下，接着使用isnan判断是否为nan
    """
    if type(item) == float and math.isnan(item):
        return True
    return False

def preprocess_round_two(l):
    """
    处理float的金额数字
        一个list中的float，全部保留2位有效数字（返回str格式）
        针对所有的float，nan转换为0.0
    """
    new_l = []
    for num in l:
        if isNan(num):
            num = 0.0
        new_l.append(('%.2f'%num)) # 强转float时，会丢失.00这种2位小数，只能存成str，后期要运算再变回来
    return new_l

def preprocess_productName(l):
    """
    对于一个list，# 替换所有的【】变成[]
    """
    new_l = []
    for s in l:
        if isNan(s):
            new_l.append(math.nan)
        else:
            new_l.append(s.replace('【', '[').replace('】', ']'))
    return new_l

def load_excel(excel_path):
    """
    读取excel，做成字典
    {'column' : [item, item, ...], ...}
    """
    # 身份证需要按str读取，其余按照excel内自己的格式
    df = pd.read_excel(excel_path, converters={"身份证号码1": str, '购房人身份证1': str})  # 又学到了，converter
    columns = list(df.columns)

    dic = {}
    for col in columns:
        col_list = list(df[col])
        if col in round_two_list:
            col_list = preprocess_round_two(col_list)
        if col == '产品全称':
            col_list = preprocess_productName(col_list)
        dic[col] = col_list

    dic = generate_rest_money(dic)
    return dic

def generate_rest_money(dic):
    """
    生成【剩余应付房款】
        【剩余应付房款】 = 【对应首付金额（元）】+【剩余购房款（不含首付）】
    """
    value_list = []  # 记录剩余应付房款

    total_num = len(dic.get('乙方1'))
    for number in range(total_num):
        # 无内容的记录，直接跳过
        name = dic.get('乙方1')[number]
        if isNan(name):
            value_list.append(math.nan)
        else:
            item1 = dic.get('对应首付金额（元）')[number]
            if isNan(float(item1)):
                item1 = 0.0
            item2 = dic.get('剩余购房款（不含首付）')[number]
            if isNan(float(item2)):
                item2 = 0.0
            value = float(item1) + float(item2)
            value_list.append(value)

    value_list = preprocess_round_two(value_list)
    dic['剩余应付房款'] = value_list

    return dic


class SudokuGame():
    """
    Game of sudoku with a game board and solution checker
    """
    BOARD_SIZE = 9

    def __init__(self, board):
        self.board = board
        self.isSolved = self.solved()

    def solved(self):
        """
        Checks if the game board has been solved
        :return: True if solved, False otherwise
        """
        def check_rows():
            for row in self.board:
                if len(set(row)) != self.BOARD_SIZE:
                    return False
            return True

        def check_columns():
            for i in range(0, self.BOARD_SIZE):
                column = [row[i] for row in self.board]
                if len(set(column)) != self.BOARD_SIZE:
                    return False
            return True

        def check_squares():
            for i in range(0, 9, 3):
                for j in range(0, 9, 3):
                    square = [row[j:j+3] for row in self.board[i:i+3]]
                    square = [x for y in square for x in y]
                    if len(set(square)) != self.BOARD_SIZE:
                        return False
            return True

        return check_columns() and check_rows() and check_squares()