import trainer.ltp_trainer as ltp
import torch

tokenizer = ltp.transformers.BertTokenizerFast.from_pretrained('bert-base-uncased')

dataloader = ltp.get_dataloader('mrpc', tokenizer, 64, split='validation')

x = 0
c = 0
for batch in dataloader:
    label = batch['labels']
    if label.shape[0] == 64:
        x += label
        c += 1

print(torch.mean(x/c, dim=0))


def countSort(arr):                                            #2 ARREGLOS:
    output = [0 for i in range(len(arr))]     #uno del mismo tamaño que el original donde va guardanod el resultado
    count = [0 for i in range(10)]            #otro para contar, (del tamaño del elemento más grande dentro de mi arreglo)

    for i in arr:                            #cuantas veces aparece cada número
        count[i] +=1

    for i in range(1,10):                   #suma acumulativa
        count[i] += count[i-1]

    for i in range(len(arr)):               #ARREGLO RESULTADO:
        output[count[arr[i]]-1] = arr[i]    #Recorrer los números, extraer el indice del contador (restando 1) y asignarlo
        count[arr[i]] -= 1                  #decrementar
    return output

arr= [1 , 4, 1, 2, 7, 5, 2]
print("Original: ", arr)
print("Ordenado: ", countSort(arr))


import os
import scripts.validation.make_bpseq_from_csv as mb

def test_convert():
    csv = "/data/2d-rna/new-cifs/rnapdbee-cifs//6XQE_1_2x/0/noncanonical.csv"
    cif = "/data/2d-rna/new-cifs/cifs//6XQE_1_2x.cif"
    fasta = "/data/2d-rna/new-cifs/fasta500//6XQE_1_2x.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq) > 0

def test_convert2():
    csv = "/data/2d-rna/new-cifs/rnapdbee-cifs/6NUO_1_RB/0/noncanonical.csv"
    cif = "/data/2d-rna/new-cifs/cifs/6NUO_1_RB.cif"
    fasta = "/data/2d-rna/new-cifs/fasta500/6NUO_1_RB.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq) > 0

def test_convert3():
    csv = "/data/2d-rna/new-cifs/rnapdbee-cifs//6O90_1_B/0/noncanonical.csv"
    cif = "/data/2d-rna/new-cifs/cifs//6O90_1_B.cif"
    fasta = "/data/2d-rna/new-cifs/fasta500//6O90_1_B.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq)

def test_convert4():
    csv = "/data/2d-rna/new-cifs/rnapdbee-cifs/6TPQ_1_V/0/noncanonical.csv"
    cif = "/data/2d-rna/new-cifs/cifs/6TPQ_1_V.cif"
    fasta = "/data/2d-rna/new-cifs/fasta500/6TPQ_1_V.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq)

def test_convert5():
    csv = "/data/2d-rna/rnapdbee-cifs/1AKX_1_A/0/noncanonical.csv"
    cif = "/data/2d-rna/cifs/1AKX_1_A.cif"
    fasta = "/data/2d-rna/fasta500/1AKX_1_A.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq)

def test_convert6():
    csv = "/data/2d-rna/rnapdbee-cifs/1DK1_1_B/0/noncanonical.csv"
    cif = "/data/2d-rna/cifs/1DK1_1_B.cif"
    fasta = "/data/2d-rna/fasta500/1DK1_1_B.fa"
    seq, bpseq = mb.convert_to_bpseq(csv, fasta, cif)
    assert len(seq)

def test_get_file():
    path = "/data/2d-rna/new-cifs/cifs/6NUO_1_RB.cif"
    seq, jid = mb.get_cif_sequence_ids(path)
    assert len(seq) == 120
    assert len(jid) == 120

def test_wobble_remove():
    path = os.path.join(os.getcwd(), "tests/test-data/")
    seq = "gGGCGCCGGUACGCAAGUACGACGGUACGCUCC"
    mol = '1EXY_1_A'
    csv = os.path.join(path, mol, "noncanonical.csv")
    cif = os.path.join(path, mol, f'{mol}.cif')
    fasta = os.path.join(path, mol, f'{mol}.fa')

    sequence, wbbpseq = mb.convert_to_bpseq(
        csv=csv, cif=cif, fasta=fasta, wobble=True)

    _, nwbbpseq = mb.convert_to_bpseq(
        csv=csv, cif=cif, fasta=fasta, wobble=False)

    num_pairings = [True for b in wbbpseq.values() if len(b) <= 1]
    wbpairs = set([f'{seq[i-1]}{seq[j[0]-1]}' for i,
                   j in wbbpseq.items() if len(j) > 0])
    nwbpairs = set([f'{seq[i-1]}{seq[j[0]-1]}' for i,
                    j in nwbbpseq.items() if len(j) > 0])

    wobble = {'GU', 'UG'}
    assert len(wobble.intersection(wbpairs)) > 0
    assert len(wobble.intersection(nwbpairs)) == 0

    assert all(num_pairings)
    assert sequence == seq.upper()
    assert len(wbbpseq) == len(seq)

def test_pairing_mode():
    noncanon = '/0/noncanonical.csv'
    canon = '/0/whole.csv'
    nc_wobble = mb.get_pairing_mode(noncanon)
    cn_wobble = mb.get_pairing_mode(canon)
    assert nc_wobble == False
    assert cn_wobble == True


"""
Funciones necesarias para la evaluación
de modelos regresores.

@author: Adonis Gonzalez
"""

from imports import *

def score(y_true, y_pred, a1=10, a2=13):
    score = 0
    d = y_pred - y_true
    for i in d:
        if i >= 0:
            score += math.exp(i / a2) - 1
        else:
            score += math.exp(- i / a1) - 1
    return score

def score_func(y_true, y_pred):
    lst = [round(score(y_true, y_pred), 2),
           round(mean_absolute_error(y_true, y_pred), 2),
           round(mean_squared_error(y_true, y_pred), 2) ** 0.5,
           round(r2_score(y_true, y_pred), 2)]

    print(f' compatitive score {lst[0]}')
    print(f' mean absolute error {lst[1]}')
    print(f' root mean squared error {lst[2]}')
    print(f' R2 score {lst[3]}')
    return [lst[1], round(lst[2], 2), lst[3] * 100]

def r2_keras(y_true, y_pred):
    SS_res = K.sum(K.square(y_true - y_pred))
    SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
    return 1 - SS_res / (SS_tot + K.epsilon())

def lstm_test_eval(lstm_test_df, model, sequence_length, sequence_cols):
    # We pick the last sequence for each id in the test data
    seq_array_test_last = [lstm_test_df[lstm_test_df['id'] == id][sequence_cols].values[-sequence_length:]
                           for id in lstm_test_df['id'].unique() if
                           len(lstm_test_df[lstm_test_df['id'] == id]) >= sequence_length]

    seq_array_test_last = np.asarray(seq_array_test_last).astype(np.float32)

    # Similarly, we pick the labels
    y_mask = [len(lstm_test_df[lstm_test_df['id'] == id]) >= sequence_length for id in
              lstm_test_df['id'].unique()]
    label_array_test_last = lstm_test_df.groupby('id')['RUL'].nth(-1)[y_mask].values
    label_array_test_last = label_array_test_last.reshape(label_array_test_last.shape[0], 1).astype(np.float32)

    estimator = model

    # test metrics
    scores_test = estimator.evaluate(seq_array_test_last, label_array_test_last, verbose=2)
    print('\nMAE: {}'.format(scores_test[1]))
    print('\nR^2: {}'.format(scores_test[2]))

    y_pred_test = estimator.predict(seq_array_test_last)
    y_true_test = label_array_test_last

    test_set = pd.DataFrame(y_pred_test)
    print(test_set.head())

    # Plot in blue color the predicted data and in green color the
    # actual data to verify visually the accuracy of the model.
    fig_verify = plt.figure(figsize=(10, 5))
    plt.plot(y_pred_test)
    plt.plot(y_true_test, color="orange")
    plt.title('prediction')
    plt.ylabel('value')
    plt.xlabel('row')
    plt.legend(['predicted', 'actual data'], loc='upper left')
    plt.show()
    return scores_test[1], scores_test[2]


import copy
import ptree
import itertools

class PRule(object):
    def __init__(self, variable, derivation, probability):
        self.variable = str(variable)
        self.derivation = tuple(derivation)
        self.probability = float(probability)

    def derivation_length(self):
        return len(self.derivation)

    def __repr__(self):
        compact_derivation = " ".join(self.derivation)
        return self.variable + ' -> ' + compact_derivation + ' (' + str(self.probability) + ')'

    def __eq__(self, other):
        try:
            return self.variable == other.variable and self.derivation == other.derivation
        except Exception:
            return False

class PCFG(object):
    def __init__(self, start_variable='S', rules=None):
        if rules is None:
            self.rules = {}
        else:
            self.rules = copy.deepcopy(rules)  # A dictionary that maps an str object to a list of PRule objects
        self.start = start_variable  # Start symbol of the grammar

    def add_rule(self, rule):
        """
        Adds a rule to dictionary of rules in grammar.
        """
        if rule.variable not in self.rules:
            self.rules[rule.variable] = []
        self.rules[rule.variable].append(rule)

    def remove_rule(self, rule):
        """
        Removes a rule from dictionary of rules in grammar.
        """
        try:
            self.rules[rule.variable].remove(rule)
        except KeyError:
            pass
        except ValueError:
            pass

    def eliminate_epsilon_rules(self, rule, black):
        """
        eliminate all rules of the form A → ε[p] (except A = S0)
        from the grammar
        :param rule: a rule of the form A → ε[p]
        :param black: "black list" of all epsilon rules variables that were removed
        :return: None
        """
        my_variable = rule.variable
        p = rule.probability
        self.remove_rule(rule)  # a. remove A --> epsilon[p]
        black.append(rule.variable)
        self.normalize(rule)  # b. normalize all remaining rules for A

        rhs = self.right_hand_side(my_variable, black)  # c. Find all rules where A appears in the right-hand side

        for vari in rhs:  # every variable found
            for r in rhs[vari]:  # every rule derivation that my_variable appears in
                deriv = r.derivation
                indices = self.indices_of_vari(deriv, my_variable)  # all occurrence indices of my_variable (list)
                new_rules = []  # a list of the required rule combinations
                m = len(indices)
                for k in range(1,
                               m + 1):  # all rules that can be obtained by applying the removed rule partially or fully
                    new_rules += self.remove_occurrences(k, indices, r, p)
                r.probability = self.new_prob_computation(r.probability, p, 0, m)  # update rule probability
                for new_rule in new_rules:
                    if self.exists(new_rule):  # If one of these rules already exists in the grammar we update its
                        # probability
                        existing = self.find_unknown_rule(new_rule)
                        existing.probability = existing.probability + new_rule.probability
                    else:
                        self.add_rule(new_rule)

    def exists(self, rule):
        """
        check if rule exists in the grammar
        :return: True if exists, else False
        """
        if rule.variable not in self.rules:
            return False
        for r in self.rules[rule.variable]:
            if r.derivation == rule.derivation:
                return True

    def find_unknown_rule(self, new_rule):
        """
        find and return a certain rule according
        to its derivation and variable
        """
        for rule in self.rules[new_rule.variable]:
            if rule == new_rule:
                return rule

    def remove_occurrences(self, k, indices, rule, p):
        """
        removes every combination of
        k-times occurrence of a variable
        from the derivation
        returns a list of the rules obtained
        by removing the occurrences.
        each rule as a tuple
        """
        combs = list(itertools.combinations(indices, k))  # all k-sized combination of indices to remove
        new_rules = []  # all new rules obtained by removing occurrences
        deriv = rule.derivation
        vari = rule.variable
        q = rule.probability
        m = len(indices)
        new_p = self.new_prob_computation(q, p, k, m)  # compute new probability for the rules
        for comb in combs:
            new_deriv = tuple([deriv[t] for t in range(len(deriv)) if
                               t not in comb])  # create new derivation from removing occurrences
            new_rule = PRule(vari, new_deriv, new_p)  # create new rule
            new_rules.append(new_rule)

        return new_rules

    @staticmethod
    def new_prob_computation(q, p, k, m):
        """
        compute: q * p_k * p_l
        :param q: probability of a rule
        :param p: probability of another rule
        :param k: k<m, number of removed occurrences of a variable
        :param m: total number of occurrences of a variable in a derivation
        :return: q * p_k * p_l
        """
        p_k = pow(p, k)
        p_l = pow(1 - p, m - k)
        return q * p_k * p_l

    @staticmethod
    def indices_of_vari(deriv, vari):
        """
        find the indices every occurrence of vari
        in a given derivation
        return a list of the indices
        """
        indices = []
        for v in range(len(deriv)):
            if deriv[v] == vari:
                indices.append(v)
        return indices

    def right_hand_side(self, my_variable, black):
        """
        find and return a dictionary of all rules where my_variable
        appears in the right-hand side
        """
        rhs = {}  # a dictionary of all the rules according to variable that my_variable appears in their derivations
        # {variable : rules with my_variable}
        for vari in self.rules:
            vari_list = []      # a list of all the rules in the derivations of vari that my_variable appears in them
            for rule in self.rules[vari]:
                if rule.variable not in black:
                    deriv = rule.derivation
                    if my_variable in deriv:
                        vari_list.append(rule)
            if vari_list:          # if my_variable appeared in at least one derivation
                rhs[vari] = vari_list   # add vari : vari_list to rhs

        return rhs

    @staticmethod
    def norm_computation(q, p):
        """
        compute q / (1 - p)
        :return: q / (1 - p)
        """
        return q / (1 - p)

    def normalize(self, curr_rule):
        """
        normalize all remaining rules for curr_rule
         each rule A → u[q] (where u ∈ (Σ ∪ V )∗) will become A → u[q/1−p]
        """
        my_variable = curr_rule.variable
        p = curr_rule.probability
        for rule in self.rules[my_variable]:    # each rule A → u[q] (where u ∈ (Σ ∪ V )∗)
            q = rule.probability
            rule.probability = self.norm_computation(q, p)  # will become A → u[q/1−p]

    def change_rule(self, rule, name):
        """
        change the given RHS of the
        given rule into a 2 symbols RHS
        remove the previous version of the rule
        from the grammar
        add the new version to the grammar
        return the RHS sequence that
        wasn't included in the new RHS version
        """
        prev_derivation = rule.derivation
        new_derivation = (prev_derivation[0], name)  # create new derivation
        updated_rule = PRule(rule.variable, new_derivation,
                             rule.probability)  # create a rule with the current variable and new derivation
        self.remove_rule(rule)  # remove current rule
        self.add_rule(updated_rule)  # add new rule
        return prev_derivation[1:]  # return derivation's "tail"

    def multiple_symbols(self, rule):
        """
        break the RHS of the given rule
        into a sequence of rules whose
        RHS <= 2 symbols
        """
        maxi = 0        # maxi =  max{i: v name = v_Ni, i > 0}
        vari = rule.variable
        for v in self.rules:
            if str(vari) + "_" in v:    # if variable of the form v_N exists in self.rules
                nv = v[2 + len(vari):]  # check its characters after N
                try:
                    nv = int(nv)        # check whether its characters form a number
                except ValueError:
                    pass
                if type(nv) == int:     # if so
                    if nv > maxi:
                        maxi = nv
        count = maxi + 1                # count will appear at the end of the new variable name
        length = len(rule.derivation)
        curr_rule = rule
        while length > 2:  # while the current rule's RHS contains more than 2 symbols
            name = vari + "_N" + str(count)  # generate a new name for a rule
            deriv = self.change_rule(curr_rule, name)  # change the current rule and return the end of its derivation
            curr_rule = PRule(name, deriv,
                              1)  # create a new rule with the returned derivation and the generated variable name
            self.add_rule(curr_rule)
            length = len(curr_rule.derivation)
            count += 1

    def is_variable(self, u):
        """
        return True if u is a variable
        else False
        """
        return u in self.rules

    def one_terminal(self, rule, side):
        """
        create a new rule of the form A → b[1] where b ∈ Σ
        if side:
        convert rule of the form A → bC[p] where b ∈ Σ and C ∈ V
        else:
        convert rule of the form: A → Bc[p] where B ∈ V and c ∈ Σ
        side: False when a variable is on the right and a terminal on the left
        True when a variable is on the left and a terminal on the left
        add created rules to the grammar
        return: converted rule
        """
        vari = rule.variable
        p = rule.probability
        deriv = rule.derivation
        ind = int(side)  # convert boolean value to int, just in case
        # ind == 1 if side == True, ind == 0 if side == False
        name = vari + "_Y" + str(ind)      # give a name to the new rule
        new_rule = PRule(vari, (deriv[ind], name), p)  # new rule to replace rule
        self.remove_rule(rule)
        self.add_rule(new_rule)
        self.add_rule(PRule(name, deriv[1 - ind], 1))  # create a new rule of the form A → b[1] where b ∈ Σ
        return new_rule

    def to_near_cnf(self):
        """
        Returns an equivalent near-CNF grammar.
        """
        near_cnf = PCFG(self.start, self.rules)  # generate a grammar with self.rules
        new_start = PRule("S_0", (self.start,), 1)  # 1. create new start variable
        near_cnf.add_rule(new_start)  # 1. add new start variable
        near_cnf.start = new_start.variable  # set as start variable

        # 2. eliminate epsilon rules
        go = True                      # a list of all epsilon rule
        black = []
        while go:
            epsilon_rules = []
            for vari in near_cnf.rules:
                for rule in near_cnf.rules[vari]:
                    if rule.derivation == () and rule not in black and rule.variable != near_cnf.start:
                        # if the rule is an epsilon rule (derivation ()) and it was already removed or it is S_0 --> ()
                        epsilon_rules.append(rule)
            for epsilon_rule in epsilon_rules:
                near_cnf.eliminate_epsilon_rules(epsilon_rule, black)

            go = epsilon_rules != []

        # 3. Convert “long rules”
        to_convert = []                         # a list of all rules with RHS > 2
        for vari in near_cnf.rules:  # a. Multiple symbols
            for rule in near_cnf.rules[vari]:
                if len(rule.derivation) > 2:
                    to_convert.append(rule)

        for rule in to_convert:
            near_cnf.multiple_symbols(rule)

        false_for_right = []                    # a list of all rules of the form A → Bc[p] where B ∈ V and c ∈ Σ
        true_for_left = []                      # a list of all rules of the form A → bC[p] where b ∈ Σ and C ∈ V
        both = []                               # a list of all rules of the form A → bc[p] where b,c ∈ Σ
        for vari in near_cnf.rules:  # b. Terminals
            for rule in near_cnf.rules[vari]:
                if len(rule.derivation) > 1:
                    res_first = near_cnf.is_variable(rule.derivation[0])
                    res_second = near_cnf.is_variable(rule.derivation[1])
                    if res_first and not res_second:  # rule of the form: A → Bc[p] where B ∈ V and c ∈ Σ
                        false_for_right.append(rule)

                    elif not res_first and res_second:  # rule of the form: A → bC[p] where b ∈ Σ and C ∈ V
                        true_for_left.append(rule)

                    elif not res_first and not res_second:  # r if both RHS variables are terminals
                        both.append(rule)

        for rule in false_for_right:
            near_cnf.one_terminal(rule,
                                  False)  # side == False when a variable is on the right and a terminal on the left

        for rule in true_for_left:
            near_cnf.one_terminal(rule,
                                  True)  # side == True when a variable is on the left and a terminal on the left

        for rule in both:
            new_rule = near_cnf.one_terminal(rule,
                                             False)  # first convert to the form A → bC[p] where b ∈ Σ and C ∈ V
            near_cnf.one_terminal(new_rule, True)  # then convert from the form A → bC[p] where b ∈ Σ and C ∈ V

        return near_cnf

    @staticmethod
    def table(row, col, val=None):
        """ create a row X col table"""
        mat = [[val]*col for x in range(row)]
        return mat

    def find_vars_to_terminal(self, w_j, var=True):
        """ find the variables that derive
        w_j and add them to a list (if var)
        or to a dictionary {variable_1: [w_j],... variable_k : [w_j]}
        (if not var)
        return the list or the dictionary
        according to the boolean flag"""
        varis = []
        matching = {}

        for variab in self.rules:
            for rule in self.rules[variab]:
                if w_j in rule.derivation:
                    if rule.derivation not in varis:  # if a derivation was not added
                        varis.append((rule.variable, rule.probability))  # add it to the variables list
                    matching[rule.variable] = ([(w_j, rule.probability)], 1)  # add each rule to the dictionary.
                    # value- a tuple of a list containing a tuple of w_j and its probability and 1 to so it will
                    # correspond to the shape of the non-terminal derivations in the bo table.
        if var:
            if varis:
                return varis  # if an empty list return None
        else:
            if matching:
                return matching  # if an empty dictionary return None

    @staticmethod
    def find_prob_in_cell(cell, var_to_find):
        """
        return the probability of the variable
        in the given cell if there
        else return 0
        """
        if cell:           # if not None
            for tups in cell:
                if tups[0] == var_to_find:
                    return tups[1]

        return 0

    def find_vars_to_vars(self, b, c, cell_to_compare, var=True):
        """ find the variables that derive
        the rule B C represented by b c respectively
        and add them to a list (if var)
        or to a dictionary {variable_1: [B, C],... variable_k : [B, C]}
        (if not var)
        return the list or the dictionary
        according to the boolean flag
        """
        varis = []
        matching = {}
        for var_b in b:  # find every combination of b and c
            for var_c in c:
                for variab in self.rules:
                    for rule in self.rules[variab]:
                        if (var_b[0], var_c[0]) == rule.derivation:  # found a variable that derives the current
                            # combination
                            b_prob = self.find_prob_in_cell(b, var_b[0])
                            c_prob = self.find_prob_in_cell(c, var_c[0])
                            to_compare = self.find_prob_in_cell(cell_to_compare, rule.variable)  # find the probability
                            # of the variable of the rule
                            if b_prob * c_prob * rule.probability > to_compare:  # if q·p1·p2
                                # is greater than the current probability of Z in (i, j):
                                app = True      # append to varis?
                                for num in range(len(varis)):
                                    if varis[num][0] == rule.variable:  # if a rule was add to varis
                                        app = False                     # don't append it to varis
                                        if varis[num][1] < b_prob * c_prob * rule.probability:  # if new probability
                                            varis[num] = list(varis[num])   # is better, replace variable probability
                                            varis[num][1] = b_prob * c_prob * rule.probability
                                            varis[num] = tuple(varis[num])
                                if app:
                                    varis.append((rule.variable, b_prob * c_prob * rule.probability))
                                # add it to the var-prob list
                            matching[rule.variable] = ([var_b, var_c], b_prob * c_prob * rule.probability)
                            # add each rule to the dictionary
        if var:
            if varis:
                return varis  # if an empty list return None
        else:
            if matching:
                return matching  # if an empty dictionary return None

    def find_unit_rules(self, vari):
        """
        find the unit rules that derive vari
        return a list of those rules
        """
        units = []
        for variab in self.rules:
            for rule in self.rules[variab]:
                if len(rule.derivation) == 1:   # if a unit rule
                    if vari in rule.derivation:  # if vari is the derivation
                        units.append(rule)

        return units

    def binary_rules(self, t, bp, j, s):
        """
        find every rule of the form A --> BC[p]
        in every partition in the given range (i+1, j)
        add the the rules that follow the probability
        requirements to t and to bp
        """
        for k in range(s+1, j):
            b = t[s][k]
            c = t[k][j]
            if b and c:
                fill_seq = self.find_vars_to_vars(b, c, t[s][j])  # find every A, A--> BC in G such that:
                # t[i][j](A.probability) < P(A → BC) · t[i][k](B.probability) · t[k][j](C.probability)
                # used for t, might be redundant
                fill_bp = self.find_vars_to_vars(b, c, t[s][j], False)  # find every A, A--> BC in G, used for bp
                if fill_seq is not None:  # if at least one matching rule was found
                    if t[s][j] is not None:  # (start block) if the cell was't empty
                        self.check_if_rule_was_added_list(t, s, j, fill_seq)
                    else:  # if the cell was empty
                        t[s][j] = fill_seq  # (end block)
                    # block might be redundant
                    if bp[s][j] is not None:  # if the cell was't empty
                        self.check_if_rule_was_added_dic(bp, s, j, fill_bp)
                    else:  # if the cell was empty
                        bp[s][j] = fill_bp

    @staticmethod
    def check_if_rule_was_added_list(t, s, j, cell):
        """
        when t[s][j] isn't empty
        for each variable in cell that is found
        in t[s][j], replace t[s][j][variable] to cell[variable]
        because its probability is larger (because it was added in
        find_vars_to_vars)
        for each variable in cell that is not
        found in t[s][j], add cell[variable]
        to t[s][j]
        :param t: table
        :param s:  row
        :param j:  column
        :param cell: rules to add to t
        :return: None
        """
        replace = [pair for pair in cell for exists in t[s][j] if pair[0] == t[s][j][0]]   # all tuples that there is
        # no need to add to t[s][j]
        delete = [pair for pair in t[s][j] for exists in cell if pair[0] == t[s][j][0]]
        do_add = [pair for pair in cell if pair not in replace]
        dont_change = [pair for pair in t[s][j] if pair not in delete]
        t[s][j] = replace + do_add + dont_change

    @staticmethod
    def check_if_rule_was_added_dic(bp, s, j, cell):
        """
        when bp[s][j] isn't empty
        for each variable in cell that is found
        in bp[s][j], replace bp[s][j][variable] to cell[variable]
        because its probability is larger (because it was added in
        find_vars_to_vars)
        for each variable in cell that is not
        found in bp[s][j], add cell[variable]
        to bp[s][j]
        :param bp: backpointers table
        :param s:  row
        :param j:  column
        :param cell: rules to add to bp
        :return: None
        """
        dont_add = []       # all items that there is no need to add to bp[s][j]
        for vari in cell:   # check if cell contains rules that appear in bp[s][j]
            if vari in bp[s][j]:
                bp[s][j][vari] = cell[vari]
                dont_add.append(vari)
        new_dic = {}
        for v in cell:
            if v not in dont_add:
                new_dic[v] = cell[v]

    def unary_rules(self, t, bp, j, s):
        """
        find every rule of the form A--> B[p]
        """
        if t[s+1][j] is not None:
            go = True
            to_add_later = []  # all relevant tuples: (unit rule, probability)
            units = []
            while go:          # while ∃A → B ∈ G s.t. t[i][j][A] < P(A → B) · t[i][j][B]
                go = False
                for each_var in t[s + 1][j]:
                    lst = self.find_unit_rules(each_var[0])   # find cell's unit rules
                    self.add_without_duplications(lst, units)
                if units:   # if unit rules are found
                    for each_var in t[s + 1][j]:
                        for u in units:
                            new_prob = u.probability * each_var[1]  # compute minimal probability
                            if u.derivation[0] == each_var[0]:
                                curr_prob = self.find_prob_in_cell(t[s + 1][j], u.variable)
                                if curr_prob < new_prob:
                                    # if u.probability · each_var[1] is greater that the current probability of the
                                    # u.variable in (i, j):
                                    if [u, new_prob, False] not in to_add_later:   # if a new unary rule was added
                                        go = True   # that means we should iterate one more time to check if there are
                                        to_add_later.append([u, new_prob, True])  # new unary rules

                for tup in to_add_later:  # after all new unary rules are collected
                    if tup[2] is True:
                        # update probabilities in cells for every variable that its probability wasn't updated
                        to_t = (tup[0].variable, tup[1])
                        t[s + 1][j].append(to_t)
                        bp[s + 1][j][tup[0].variable] = ([(tup[0].derivation[0], tup[0].probability)], tup[1])
                    tup[2] = False

    @staticmethod
    def add_without_duplications(list1, list2):
        """
        add list1's elements that don't
        appear in list2 to list2
        """
        for elem1 in list1:
            if elem1 not in list2:
                list2.append(elem1)

    def bp_no_prob(self, bp):
        """
        create and return a new backpointers table
        without probabilities
        """
        new_bp = self.table(len(bp), len(bp))
        for row in range(len(bp)):
            for coll in range(len(bp)):
                if bp[row][coll] is not None:       # for each cell in bp, if cell is not None:
                    new_dic = {}        # create a dictionary for the cell
                    new_bp[row][coll] = new_dic
                    for vari in bp[row][coll]:  # for each variable in the cell in bp
                        new_dic[vari] = []      # create a new list of values for the variable
                        for elem in range(len(bp[row][coll][vari][0])):
                            new_dic[vari].append(bp[row][coll][vari][0][elem][0])  # without its probability

        return new_bp

    def cky_parser(self, string):
        """
        Parses the input string given the grammar, using the probabilistic CKY algorithm.
        If the string has been generated by the grammar - returns a most likely parse tree for the input string.
        Otherwise - returns None.
        The CFG is given in near-CNF.
        """
        n = len(string) + 1
        t = self.table(n, n)  # each cell is either None or a list of variables
        # might be redundant because bp contains same variables and their derivation
        bp = self.table(n, n)  # backpointers table- bp[s][j] = {t[s][j][1] :
        # { [derivation],..... t[s][j][k] : [derivation] }
        for j in range(1, n):
            fill = self.find_vars_to_terminal(string[j-1])       # find every A, A--> w_j in G, used for t
            # might be redundant
            fill_bp = self.find_vars_to_terminal(string[j-1], False)    # find every A, A--> w_j in G, used for bp
            t[j-1][j] = fill
            bp[j-1][j] = fill_bp
            for s in range(j-2, -2, -1):    # go over every string partition
                self.binary_rules(t, bp, j, s+1)
                self.unary_rules(t, bp, j, s)   # find unit rules and add them to t and to bp

        new_bp = self.bp_no_prob(bp)
        if t[0][n-1] is not None:  # if the cell is not empty
            for pair in t[0][n-1]:
                if self.start in pair:    # is start variable is in that cell
                    root_prob = pair[1]
                    if root_prob < pow(2, -50):
                        return "the given grammar cannot derive the given string (low probability)"
                    tree = self.my_tree(new_bp, 0, n-1, root_prob, self.start)          # create a tree according to bp
                    return tree

        return "the given grammar cannot derive the given string"

    def my_tree(self, bp, row, col, root_prob, start):     # row = 0, col = last
        root = ptree.Node(start, [])
        tree = ptree.PTree(root, root_prob)
        self.tree_left(bp, root, row, col)  # start constructing from left, tree_left will build right tree as well
        return tree

    def tree_right(self, bp, root, row, col):
        if row >= len(bp) - 1 or col <= 0:  # "frame"- halting condition
            return None

        if len(bp[row][col][root.key]) < 2:     # root has no right child- finish
            return None

        need = bp[row][col][root.key][1]        # get root's right child
        curr = root.children
        curr.append(ptree.Node(need, []))             # add a brother to root's left child

        row = row + 1       # find child's derivation found down in the same column
        while row < len(bp) and (bp[row][col] is None or need not in bp[row][col]):
            row += 1
        self.tree_left(bp, curr[1], row, col)       # find left child of root's right child

    def tree_left(self, bp, root, row, col):
        if row >= len(bp) - 1 or col <= 0:  # "frame"- halting condition
            return None

        need = bp[row][col][root.key][0]    # get root's left child
        curr = root.children
        curr.append(ptree.Node(need, []))         # insert root's left child to the children list

        l_col = col    # find child's derivation found in the same row from left for constructing left path
        if len(bp[row][col][root.key]) == 2:    # if root has no right child, keep looking in the same column
            l_col -= 1                          # else go a column back
        while l_col > 0 and (bp[row][l_col] is None or need not in bp[row][l_col]):
            l_col -= 1

        self.tree_left(bp, curr[0], row, l_col)     # recursive call- find left child of root's left child

        if len(bp[row][col][root.key]) < 2:         # if root has no right child- finish
            return None

        self.tree_right(bp, root, row, col)         # find root's right child

    def is_valid_grammar(self):
        """
        Validates that the grammar is legal (meaning - the probabilities of the rules for each variable sum to 1).
        """
        epsilon = 0.0001
        for variable in self.rules:     # for each variable in the rules dictionary
            c = 0                       # initialize a counter to sum probabilities
            for rule in self.rules[variable]:   # for each LHS of the variable
                c += rule.probability           # add its probability to the counter
            if c < 1 - epsilon or c > 1 + epsilon:  # if c is not in the interval (1- epsilon, 1 + epsilon)
                return False
        return True


"""
 Show how to fire bullets.

 Sample Python/Pygame Programs
 Simpson College Computer Science
 http://programarcadegames.com/
 http://simpson.edu/computer-science/

 Explanation video: http://youtu.be/PpdJjaiLX6A
"""
import pygame
import random
import math

# Define some colors
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
YELLOW = (255,255,0)

# Set the height and width of the screen
screen_width = 700
screen_height = 400

# --- Classes

class Block(pygame.sprite.Sprite):
    """ This class represents the block. """
    def __init__(self, color):
        # Call the parent class (Sprite) constructor
        super().__init__()

        '''
        self.image = pygame.Surface([20, 15])
        self.image.fill(color)

        self.rect = self.image.get_rect()
        '''
        # Load the image
        self.image = pygame.image.load("images/meteor.png").convert()

        # Set our transparent color
        self.image.set_colorkey(BLACK)

        # Fetch the rectangle object that has the dimensions fo the image
        self.rect = self.image.get_rect();

    def update(self):
        # Move the block one step down
        self.rect.y +=1

class Player(pygame.sprite.Sprite):
    """ This class represents the Player. """

    def __init__(self):
        """ Set up the player on creation. """
        # Call the parent class (Sprite) constructor
        super().__init__()

        # Speed in pixels per frame
        self.xspeed = 0
        self.yspeed = 0

        '''
        self.image = pygame.Surface([20, 20])
        self.image.fill(RED)

        self.rect = self.image.get_rect()

        # Speed in pixels per frame
        self.xspeed = 0
        self.yspeed = 0

        # Initalise position of the player at the bottom of
        # the screen
        self.rect.x = 350
        self.rect.y = 370
        '''
        # Load the image
        self.image = pygame.image.load("images/player.png").convert()

        # Set our transparent color
        self.image.set_colorkey(BLACK)

        # Fetch the rectangle object that has the dimensions fo the image
        self.rect = self.image.get_rect();

        # Set initial coordinates
        self.rect.x = 350
        self.rect.y = 370

    def update(self):
        """ Update the player's position. """

        '''
        # Get the current mouse position. This returns the position
        # as a list of two numbers.
        pos = pygame.mouse.get_pos()

        # Set the player x position to the mouse x position
        self.rect.x = pos[0]
        '''
        self.rect.x += self.xspeed
        self.rect.y += self.yspeed

class Bullet(pygame.sprite.Sprite):
    """ This class represents the bullet . """
    def __init__(self):
        # Call the parent class (Sprite) constructor
        super().__init__()

        self.image = pygame.Surface([4, 10])
        self.image.fill(YELLOW)

        self.rect = self.image.get_rect()

    def update(self):
        """ Move the bullet. """
        self.rect.y -= 3
        print("bullet.y = %d" % self.rect.y)

# --- Create the window

# Initialize Pygame
pygame.init()

screen = pygame.display.set_mode([screen_width, screen_height])

# --- Background image
background = pygame.image.load("images/saturn.jpg").convert()

# Before the loop, load the sounds:
click_sound = pygame.mixer.Sound("sound/laser5.ogg")

# --- Sprite lists

# This is a list of every sprite. All blocks and the player block as well.
all_sprites_list = pygame.sprite.Group()

# List of each block in the game
block_list = pygame.sprite.Group()

# List of each bullet
bullet_list = pygame.sprite.Group()

# --- Create the sprites

for i in range(50):
    # This represents a block
    block = Block(BLUE)

    # Set a random location for the block
    block.rect.x = random.randrange(screen_width)
    block.rect.y = random.randrange(screen_height - 100)

    # Add the block to the list of objects
    block_list.add(block)
    all_sprites_list.add(block)

# Create a red player block
player = Player()
all_sprites_list.add(player)

# Loop until the user clicks the close button.
done = False
start = False

# Used to manage how fast the screen updates
clock = pygame.time.Clock()

# Select the font to use, size, bold, italics
font = pygame.font.SysFont('comicsansms', 25, True, False)
# Render the text. "True" means anti-aliased text.
game_over_text = font.render("Game over!!!!", True, WHITE)

score = 0

# -------- Main Program Loop -----------
while not done:
    # --- Event Processing
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            done = True
        elif event.type == pygame.KEYDOWN:
            if start != True:
                start = True
            # Figure out if it was an arrow key. If so
            # adjust speed.
            if event.key == pygame.K_LEFT:
                player.xspeed = -3
            elif event.key == pygame.K_RIGHT:
                player.xspeed = 3
            elif event.key == pygame.K_UP:
                player.yspeed = -3
            elif event.key == pygame.K_DOWN:
                player.yspeed = 3

             # Firing bullets when space bar is pressed
            elif event.key == pygame.K_SPACE:
                print("Firing bullet ...!")
                bullet = Bullet()
                # Set the bullet so it is where the player is
                bullet.rect.x = player.rect.x + player.rect.width/2
                bullet.rect.y = player.rect.y
                # Add the bullet to the lists
                all_sprites_list.add(bullet)
                bullet_list.add(bullet)
                click_sound.play()

            # User let up on a key
        elif event.type == pygame.KEYUP:
            if start != True:
                start = True
            # If it is an arrow key, reset vector back to zero
            if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT:
                player.xspeed = 0
            elif event.key == pygame.K_UP or event.key == pygame.K_DOWN:
                player.yspeed = 0

    # --- Game logic
    if  start:
        # Call the update() method on all the sprites
        all_sprites_list.update()

        # Calculate mechanics for each bullet
        for bullet in bullet_list:

            # See if it hit a block
            block_hit_list = pygame.sprite.spritecollide(bullet, block_list, True)

            # For each block hit, remove the bullet and add to the score
            for block in block_hit_list:
                bullet_list.remove(bullet)
                all_sprites_list.remove(bullet)
                score += 1
                print(score)

            # Remove the bullet if it flies up off the screen
            if bullet.rect.y < -10:
                bullet_list.remove(bullet)
                all_sprites_list.remove(bullet)

        block_player_list = pygame.sprite.spritecollide(player, block_list, True)
        if len(block_player_list) != 0:
            final_message = "Game Over !!!"
            done = True

        for block in block_list:
            if block.rect.y >= screen_height:
                block.rect.y = 0
                block.rect.x = random.randrange(screen_width)

        if len(block_list) == 0:
            final_message = "You won!!!"
            done = True;

    # --- Draw a frame

    # Clear the screen
    #screen.fill(BLACK)
    # Draw background first
    screen.blit(background, [0,0])

    # Draw all the spites
    all_sprites_list.draw(screen)

    # Write current score
    screen.blit(font.render("Score:" + str(score), True, WHITE), [10,10])

    # Go ahead and update the screen with what we've drawn.
    pygame.display.flip()

    # --- Limit to 20 frames per second
    clock.tick(60)

# Clear the screen
screen.fill(WHITE)

screen.blit(font.render(final_message, True, BLACK), [280,200])
pygame.display.flip()
pygame.time.delay(3000)

pygame.quit()


import argparse

def chrom_to_number(chrom):
    def to_int(tail):
        if tail.isnumeric():
            return int(tail)
        else:
            if tail.lower() == 'X':
                return 23
            elif tail.lower() == 'Y':
                return 24
            else:
                raise ValueError(f'unrecognized chromosome: {chrom}')

    if chrom.lower().startswith('chr'):
        return to_int(chrom[3:])
    else:
        return to_int(chrom)

def partition(bim_filename, out):
    with open(bim_filename, 'r') as file, open(out, 'w') as out_file:
        for line in file:
            toks = line.split()
            chrom = toks[0]
            rsid = toks[1]
            out_file.write(f'{rsid} {chrom_to_number(chrom)}\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('bim_file', type=str)
    parser.add_argument('--out', '-o', type=str)
    args = parser.parse_args()
    partition(args.bim_file, args.out)


import collections
import fnmatch
import logging
import os
import re
import subprocess

from devil.android import decorators
from devil.android import device_errors
from devil.android import device_utils
from devil.android.sdk import fastboot
from devil.utils import timeout_retry

logger = logging.getLogger(__name__)

_DEFAULT_TIMEOUT = 30
_DEFAULT_RETRIES = 3
_FASTBOOT_REBOOT_TIMEOUT = 10 * _DEFAULT_TIMEOUT

PartitionInfo = collections.namedtuple('PartitionInfo', [
    'image',     # The image name to look for. Can contain *.
    'optional',  # If this image is optional.
    'restart',   # If to restart the device after flashing the image.
])

# These partitions need to be flashed before calling flashall
# so that the device can pass the requirement check.
_VERIFICATION_PARTITIONS = collections.OrderedDict([
    ('bootloader', PartitionInfo(
        image='bootloader*.img',
        optional=False,
        restart=True,
    )),
    ('radio', PartitionInfo(
        image='radio*.img',
        optional=False,
        restart=True,
    )),
])

# These partitions will be flashed when wipe is set to true.
_WIPE_PARTITIONS = collections.OrderedDict([
    ('cache', PartitionInfo(
        image='cache.img',
        optional=True,
        restart=False,
    )),
])

def _FindAndVerifyPartitionsAndImages(partitions, directory):
  """Validate partitions and images.

  Validate all partition names and partition directories. Cannot stop mid
  flash so its important to validate everything first.

  Args:
    partitions: partitions to be tested.
    directory: directory containing the images.

  Returns:
    Dictionary with exact partition, image name mapping.
  """

  files = os.listdir(directory)
  return_dict = collections.OrderedDict()

  def find_file(pattern):
    for filename in files:
      if fnmatch.fnmatch(filename, pattern):
        return os.path.join(directory, filename)
    return None

  for partition, partition_info in partitions.items():
    image_file = find_file(partition_info.image)
    if image_file:
      return_dict[partition] = image_file
    elif not partition_info.optional:
      raise device_errors.FastbootCommandFailedError(
          [],
          '',
          message='Failed to flash device. Could not find image for %s.' %
          partition_info.image)
  return return_dict

class FastbootUtils(object):

  _FASTBOOT_WAIT_TIME = 1
  _BOARD_VERIFICATION_FILE = 'android-info.txt'

  def __init__(self,
               device=None,
               fastbooter=None,
               default_timeout=_DEFAULT_TIMEOUT,
               default_retries=_DEFAULT_RETRIES):
    """FastbootUtils constructor.

    Example Usage to flash a device:
      fastboot = fastboot_utils.FastbootUtils(device)
      fastboot.FlashDevice('/path/to/build/directory')

    Args:
      device: A DeviceUtils instance. Optional if a Fastboot instance was
        passed.
      fastbooter: A fastboot.Fastboot instance. Optional if a DeviceUtils
        instance was passed.
      default_timeout: An integer containing the default number of seconds to
        wait for an operation to complete if no explicit value is provided.
      default_retries: An integer containing the default number or times an
        operation should be retried on failure if no explicit value is provided.
    """
    if not device and not fastbooter:
      raise ValueError("One of 'device' or 'fastbooter' must be passed.")

    if device:
      self._device = device
      self._serial = str(device)
      self._board = device.product_board
      if not fastbooter:
        self.fastboot = fastboot.Fastboot(self._serial)

    if fastbooter:
      self._serial = str(fastbooter)
      self.fastboot = fastbooter
      self._board = fastbooter.GetVar('product')
      if not device:
        self._device = device_utils.DeviceUtils(self._serial)

    self._default_timeout = default_timeout
    self._default_retries = default_retries

  def __str__(self):
    return self._serial

  def IsFastbootMode(self):
    return self._serial in (str(d) for d in self.fastboot.Devices())

  @decorators.WithTimeoutAndRetriesFromInstance()
  def WaitForFastbootMode(self, timeout=None, retries=None):
    """Wait for device to boot into fastboot mode.

    This waits for the device serial to show up in fastboot devices output.
    """
    timeout_retry.WaitFor(self.IsFastbootMode,
                          wait_period=self._FASTBOOT_WAIT_TIME)

  @decorators.WithTimeoutAndRetriesFromInstance(
      min_default_timeout=_FASTBOOT_REBOOT_TIMEOUT)
  def EnableFastbootMode(self, timeout=None, retries=None):
    """Reboots phone into fastboot mode.

    Roots phone if needed, then reboots phone into fastboot mode and waits.
    """
    if self.IsFastbootMode():
      return
    self._device.EnableRoot()
    self._device.adb.Reboot(to_bootloader=True)
    self.WaitForFastbootMode()

  @decorators.WithTimeoutAndRetriesFromInstance(
      min_default_timeout=_FASTBOOT_REBOOT_TIMEOUT)
  def Reboot(self,
             bootloader=False,
             wait_for_reboot=True,
             timeout=None,
             retries=None):
    """Reboots out of fastboot mode.

    It reboots the phone either back into fastboot, or to a regular boot. It
    then blocks until the device is ready.

    Args:
      bootloader: If set to True, reboots back into bootloader.
    """
    if bootloader:
      self.fastboot.RebootBootloader()
      self.WaitForFastbootMode()
    else:
      self.fastboot.Reboot()
      if wait_for_reboot:
        self._device.WaitUntilFullyBooted(timeout=_FASTBOOT_REBOOT_TIMEOUT)

  def _VerifyBoard(self, directory):
    """Validate as best as possible that the android build matches the device.

    Goes through build files and checks if the board name is mentioned in the
    |self._BOARD_VERIFICATION_FILE| or in the build archive.

    Args:
      directory: directory where build files are located.
    """
    files = os.listdir(directory)
    board_regex = re.compile(r'require board=([\w|]+)')
    if self._BOARD_VERIFICATION_FILE in files:
      with open(os.path.join(directory, self._BOARD_VERIFICATION_FILE)) as f:
        for line in f:
          m = board_regex.match(line)
          if m and m.group(1):
            return self._board in m.group(1).split('|')

          logger.warning('No board type found in %s.',
                         self._BOARD_VERIFICATION_FILE)
    else:
      logger.warning('%s not found. Unable to use it to verify device.',
                     self._BOARD_VERIFICATION_FILE)

    zip_regex = re.compile(r'.*%s.*\.zip' % re.escape(self._board))
    for f in files:
      if zip_regex.match(f):
        return True

    return False

  def _FlashPartitions(self, partitions, directory, force=False):
    """Flashes all given partitions with all given images.

    Args:
      partitions: An ordered dict of partitions to flash.
      directory: Directory where all partitions can be found.
      force: boolean to decide to ignore board name safety checks.

    Raises:
      device_errors.CommandFailedError(): If image cannot be found or if bad
          partition name is give.
    """
    if not self._VerifyBoard(directory):
      if force:
        logger.warning('Could not verify build is meant to be installed on '
                       'the current device type, but force flag is set. '
                       'Flashing device. Possibly dangerous operation.')
      else:
        raise device_errors.CommandFailedError(
            'Could not verify build is meant to be installed on the current '
            'device type. Run again with force=True to force flashing with an '
            'unverified board.')

    flash_image_files = _FindAndVerifyPartitionsAndImages(partitions, directory)
    for partition, image_files in flash_image_files.items():
      logger.info('Flashing %s with %s', partition, image_files)
      self.fastboot.Flash(partition, image_files)
      if partitions[partition].restart:
        self.Reboot(bootloader=True)

  def FlashDevice(self, directory, wipe=False, wait_for_reboot=False):
    """Flash device with build in |directory|.

    Directory must contain bootloader, radio, and other necessary files from
    an android build. This is a dangerous operation so use with care.

    Note when wipe is set to true, we erase "userdata" partition, and flash the
    partitions in _WIPE_PARTITIONS to achieve the wipe purpose. We don't use
    the flag "-w" in flashall command as it can cause old devices like Nexus 5X
    stuck on the encryption phrase.

    Args:
      directory: Directory with build files.
      wipe: If set to true, wipe the device data by erasing "userdata"
        partitions and flash partitions in _WIPE_PARTITIONS.
      wait_for_reboot: If set to true, wait for the device to be online after
        sending reboot command.
    """

    # If a device is wiped, then it will no longer have adb keys so it cannot be
    # communicated with to verify that it is rebooted. It is up to the user of
    # this script to ensure that the adb keys are set on the device after using
    # this to wipe a device.
    self.EnableFastbootMode()
    self._FlashPartitions(_VERIFICATION_PARTITIONS, directory)
    try:
      logger.info('Flashing all.')
      for line in self.fastboot.FlashAll(directory, skip_reboot=True):
        logger.info('  %s', line)
    except subprocess.CalledProcessError as e:
      raise device_errors.FastbootCommandFailedError(
          [], '', message='Failed to flashall: %s' % str(e))
    if wipe:
      # Flashing the default userdata.img will cause the device to have only
      # 10GB of internal storage. To avoid this, we directly erase it and rely
      # on the recovery to auto format it.
      logger.info('Erasing "userdata" partition.')
      self.fastboot.Erase('userdata')
      self._FlashPartitions(_WIPE_PARTITIONS, directory)
    self.Reboot(wait_for_reboot=wait_for_reboot)


import os
import uuid
import json
import hashlib
import tempfile

import tornado.httpclient
import tornado.web
import tornado.httputil
from tornado import gen
from tornado.httpclient import (
    AsyncHTTPClient,
    HTTPError,
)
from pymysql.err import IntegrityError

from PIL import (
    Image,
    ImageFont,
    ImageDraw,
)

import util
import models
from settings import cdn_settings
from .. import base

class ImageUploadHandler(base.APIBaseHandler):
    def prepare(self):
        if self.request.method == 'POST':
            try:
                self.image_data = self.request.files['image']
            except Exception as e:
                raise base.JSONHTTPError(400) from e

    @base.authenticated(status=("confirmed", "reviewed",))
    @gen.coroutine
    def post(self):
        try:
            file = self.save_image(self.image_data)
        except Exception as e:
            raise base.JSONHTTPError(415) from e
        image = models.Image.query.filter_by(filename=file['name']).first()

        if not image:
            author = util.generate_cos_signature()
            comp_file = self.compress_file(file)
            crop_file = self.crop_file(comp_file)
            yield self.upload_image(comp_file, author)
            yield self.upload_image(crop_file, author)
            yield self.upload_image(file, author)
            image = self.create_image(file['name'])
        else:
            file['file'].close()
        self.set_status(201)
        self.finish(
            json.dumps(
                image.format_detail()
            )
        )

    @base.db_success_or_500
    def create_image(self, filename):
        image = models.Image(
            filename=filename,
            user=self.current_user
        )
        if image not in self.current_user.images:
            self.current_user.images.append(image)
        self.session.add(image)

        return image

    def save_image(self, metas):
        temp = tempfile.NamedTemporaryFile('wb+', delete=True)
        for meta in metas:
            temp.write(meta['body'])
        image_name = metas[0]['filename']
        temp.seek(0)
        hash_value = hashlib.md5(temp.read()).hexdigest()
        temp.seek(0)
        self.add_watermark(temp)

        return {
            'name': "{}.{}".format(
                hash_value,
                image_name.split('.')[-1]
            ),
            'file': temp
        }

    @gen.coroutine
    def upload_image(self, file, author):
        file['file'].seek(0)
        fields = (('op', 'upload'), ('insertOnly', '0'))
        content_type, body = util.encode_multipart_formdata(fields,
                                                            (('filecontent', file['name'], file['file'].read()),))
        headers = {
            'Content-Type': content_type,
            'Content-Length': str(len(body)),
            'Authorization': author
        }

        url = cdn_settings['cos_host'] + cdn_settings['bucket'] + r'/image/' + file['name']

        request = tornado.httpclient.HTTPRequest(url=url,
                                                 method="POST",
                                                 headers=headers,
                                                 body=body,
                                                 validate_cert=False)

        try:
            response = yield AsyncHTTPClient().fetch(request)
        except HTTPError as e:
            print(e.response)
            raise base.JSONHTTPError(e.code)

        file['file'].close()

    @staticmethod
    def compress_file(file):
        temp = tempfile.NamedTemporaryFile('wb+', delete=True)
        file['file'].seek(0)
        temp.write(file['file'].read())
        img = Image.open(temp.name)
        w, h = img.size
        if w >= h:
            img.resize((1080, int(1080.0 * h / w))).save(temp.name, img.format)
        elif w < h:
            img.resize((int(1080.0 * w / h), 1080)).save(temp.name, img.format)
        temp.seek(0)

        return {
            'file': temp,
            'name': 'comp_' + file['name']
        }

    @staticmethod
    def crop_file(file):
        temp = tempfile.NamedTemporaryFile('wb+', delete=True)
        file['file'].seek(0)
        temp.write(file['file'].read())
        img = Image.open(temp.name)
        w, h = img.size
        if w >= h:
            img.crop((w/2 - h/2, 0, w/2 + h/2, h)).save(temp.name, img.format)
        elif w < h:
            img.crop((0, h/2 - w/2, w, h/2 + w/2)).save(temp.name, img.format)

        temp.seek(0)

        return {
            'file': temp,
            'name': 'crop_' + file['name']
        }

    def add_watermark(self, file):
        img = Image.open(file.name)
        text = "©youpai/{}".format(self.current_user.name)
        rgba_img = img.convert('RGBA')
        font = ImageFont.truetype(self.application.settings['image_font'], 32)

        text_overlay = Image.new('RGBA', rgba_img.size, (255, 255, 255, 0))
        img_draw = ImageDraw.Draw(text_overlay)
        text_size_x, text_size_y = img_draw.textsize(text, font=font)
        text_xy = (int(rgba_img.size[0]/2 - text_size_x/2), int(rgba_img.size[1] - 2*text_size_y))
        img_draw.text(text_xy, text, font=font, fill=(0, 0, 0, 128))
        img_with_text = Image.alpha_composite(rgba_img, text_overlay)

        img_with_text.save(file.name, img.format)
        file.seek(0)


import nltk

nltk.download('movie_reviews')
import random
import pickle
from nltk.corpus import movie_reviews
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.naive_bayes import MultinomialNB,GaussianNB,BernoulliNB
from sklearn.linear_model import LogisticRegression,SGDClassifier
from sklearn.svm import  SVC,LinearSVC,NuSVC

# for votting
from nltk.classify import ClassifierI
from statistics import mode

class VoteClassifier(ClassifierI):
    def __init__(self,*classifiers):
        self._classifiers=classifiers
    def classify(self, features):
        votes=[]
        for c in self._classifiers:
            v=c.classify(features)    #classifier gula psitive or negative return
            votes.append(v)           #krte thakbe but statistics r mode most occurance ta output dibe
        return mode(votes)

    def confidence(self,features):
        votes = []
        for c in self._classifiers:
            v = c.classify(features)
            votes.append(v)
        choice_votes=votes.count(mode(votes)) #how many most popular votes
        conf=choice_votes/(len(votes))
        return conf

'''
    pos-|-> .txt    neg-|-> .txt
        |-> .txt        |-> .txt
        |-> .txt        |-> .txt

    list = allwords in movie review
    Req dictionary={words,count}

    final words=unique_word

                       final words
                          |
                        /  \
                      /      \
            training set    testing set

'''
print(movie_reviews)

documents = [(list(movie_reviews.words(fileid)), category)
             for category in movie_reviews.categories()
             for fileid in movie_reviews.fileids(category)]

random.shuffle(documents)
print(documents[0])

# print(documents[1])

all_words = []

for w in movie_reviews.words():
    all_words.append(w.lower())

# Frequency count
all_words = nltk.FreqDist(all_words)
# print(all_words.most_common(15))

# print(all_words["stupid"])

# train with top3k words
word_features = list(all_words.keys())[:3000]

def find_features(document):
    words = set(document)
    features = {}
    for w in word_features:
        features[w] = (w in words)  # if word is in document it is true otherwise false
    return features

print(find_features(movie_reviews.words('neg/cv000_29416.txt')))

# documents theke word niye dekhbo
featuresets = [(find_features(rev), category) for (rev, category) in documents]

print(featuresets[:5])

training_sets = featuresets[:1900]
testing_sets = featuresets[1900:]

# Now make a classifier.
# classifier =nltk.NaiveBayesClassifier.train(training_sets)

classifier_f = open("naivebayes.pickle", "rb")
classifier = pickle.load(classifier_f)
classifier_f.close()

print("Original Naive Bayes Algo accuracy:", (nltk.classify.accuracy(classifier, testing_sets)) * 100)
classifier.show_most_informative_features(15)  # want to show 15 most informative features

'''
How to save your train algorithm?

'''
# save_classifier=open("naivebayes.pickle","wb")
# pickle.dump(classifier,save_classifier)
# save_classifier.close()

#create a classifier using sklearn
MNB_classifier=SklearnClassifier(MultinomialNB( ))
MNB_classifier.train(training_sets)
print("MNB classifier Algo accuracy:", (nltk.classify.accuracy(MNB_classifier, testing_sets)) * 100)

# Gaus_classifier=SklearnClassifier(GaussianNB( ))
# Gaus_classifier.train(training_sets)
#
#
# print("Gaussian learn classifier Algo accuracy:", (nltk.classify.accuracy(Gaus_classifier, testing_sets)) * 100)

bernou_classifier=SklearnClassifier(BernoulliNB( ))
bernou_classifier.train(training_sets)

print("Bernouli classifier Algo accuracy:", (nltk.classify.accuracy(bernou_classifier, testing_sets)) * 100)

voted_classifier=VoteClassifier(MNB_classifier,bernou_classifier,classifier)
print("voted classifier",(nltk.classify.accuracy(voted_classifier,testing_sets))*100)
print("Classification:",voted_classifier.classify(testing_sets[0][0]),"Confidence %:",voted_classifier.confidence(testing_sets[0][0]))

def sentiment(text):
    feats=find_features(text)
    return voted_classifier.classify(feats),voted_classifier.confidence(feats)


import datetime
import numpy

from . import mean_average_precision_calculator as map_calculator
from . import average_precision_calculator as ap_calculator

def flatten(l):
    """ Merges a list of lists into a single list. """
    return [item for sublist in l for item in sublist]

def calculate_hit_at_one(predictions, actuals):
    """Performs a local (numpy) calculation of the hit at one.

  Args:
    predictions: Matrix containing the outputs of the model.
      Dimensions are 'batch' x 'num_classes'.
    actuals: Matrix containing the ground truth labels.
      Dimensions are 'batch' x 'num_classes'.

  Returns:
    float: The average hit at one across the entire batch.
  """
    top_prediction = numpy.argmax(predictions, 1)
    hits = actuals[numpy.arange(actuals.shape[0]), top_prediction]
    return numpy.average(hits)

def calculate_precision_at_equal_recall_rate(predictions, actuals):
    """Performs a local (numpy) calculation of the PERR.

  Args:
    predictions: Matrix containing the outputs of the model.
      Dimensions are 'batch' x 'num_classes'.
    actuals: Matrix containing the ground truth labels.
      Dimensions are 'batch' x 'num_classes'.

  Returns:
    float: The average precision at equal recall rate across the entire batch.
  """
    aggregated_precision = 0.0
    num_videos = actuals.shape[0]
    for row in numpy.arange(num_videos):
        num_labels = int(numpy.sum(actuals[row]))
        top_indices = numpy.argpartition(predictions[row], -num_labels)[-num_labels:]
        item_precision = 0.0
        for label_index in top_indices:
            if predictions[row][label_index] > 0:
                item_precision += actuals[row][label_index]
        item_precision /= top_indices.size
        aggregated_precision += item_precision
    aggregated_precision /= num_videos
    return aggregated_precision

def calculate_gap(predictions, actuals, top_k=20):
    """Performs a local (numpy) calculation of the global average precision.

  Only the top_k predictions are taken for each of the videos.

  Args:
    predictions: Matrix containing the outputs of the model.
      Dimensions are 'batch' x 'num_classes'.
    actuals: Matrix containing the ground truth labels.
      Dimensions are 'batch' x 'num_classes'.
    top_k: How many predictions to use per video.

  Returns:
    float: The global average precision.
  """
    gap_calculator = ap_calculator.AveragePrecisionCalculator()
    sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, actuals, top_k)
    gap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
    return gap_calculator.peek_ap_at_n()

def top_k_by_class(predictions, labels, k=20):
    """Extracts the top k predictions for each video, sorted by class.

  Args:
    predictions: A numpy matrix containing the outputs of the model.
      Dimensions are 'batch' x 'num_classes'.
    k: the top k non-zero entries to preserve in each prediction.

  Returns:
    A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
    are lists of lists of floats. 'true_positives' is a list of scalars. The
    length of the lists are equal to the number of classes. The entries in the
    predictions variable are probability predictions, and
    the corresponding entries in the labels variable are the ground truth for
    those predictions. The entries in 'true_positives' are the number of true
    positives for each class in the ground truth.

  Raises:
    ValueError: An error occurred when the k is not a positive integer.
  """
    if k <= 0:
        raise ValueError("k must be a positive integer.")
    k = min(k, predictions.shape[1])
    num_classes = predictions.shape[1]
    prediction_triplets = []
    for video_index in range(predictions.shape[0]):
        prediction_triplets.extend(top_k_triplets(predictions[video_index], labels[video_index], k))
    out_predictions = [[] for v in range(num_classes)]
    out_labels = [[] for v in range(num_classes)]
    for triplet in prediction_triplets:
        out_predictions[triplet[0]].append(triplet[1])
        out_labels[triplet[0]].append(triplet[2])
    out_true_positives = [numpy.sum(labels[:, i]) for i in range(num_classes)]

    return out_predictions, out_labels, out_true_positives

def top_k_triplets(predictions, labels, k=20):
    """Get the top_k for a 1-d numpy array. Returns a sparse list of tuples in
  (prediction, class) format"""
    m = len(predictions)
    k = min(k, m)
    indices = numpy.argpartition(predictions, -k)[-k:]
    return [(index, predictions[index], labels[index]) for index in indices]

class EvaluationMetrics(object):
    """A class to store the evaluation metrics."""

    def __init__(self, num_class, top_k):
        """Construct an EvaluationMetrics object to store the evaluation metrics.

    Args:
      num_class: A positive integer specifying the number of classes.
      top_k: A positive integer specifying how many predictions are considered per video.

    Raises:
      ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
        not be constructed.
    """
        self.sum_hit_at_one = 0.0
        self.sum_perr = 0.0
        self.sum_loss = 0.0
        self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(num_class)
        self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
        self.top_k = top_k
        self.num_examples = 0

    #def accumulate(self, predictions, labels, loss):
    def accumulate(self, loss, predictions, labels):
        """Accumulate the metrics calculated locally for this mini-batch.

    Args:
      predictions: A numpy matrix containing the outputs of the model.
        Dimensions are 'batch' x 'num_classes'.
      labels: A numpy matrix containing the ground truth labels.
        Dimensions are 'batch' x 'num_classes'.
      loss: A numpy array containing the loss for each sample.

    Returns:
      dictionary: A dictionary storing the metrics for the mini-batch.

    Raises:
      ValueError: An error occurred when the shape of predictions and actuals
        does not match.
    """
        batch_size = labels.shape[0]
        mean_hit_at_one = calculate_hit_at_one(predictions, labels)
        mean_perr = calculate_precision_at_equal_recall_rate(predictions, labels)
        mean_loss = numpy.mean(loss)

        # Take the top 20 predictions.
        sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, labels, self.top_k)
        self.map_calculator.accumulate(sparse_predictions, sparse_labels, num_positives)
        self.global_ap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))

        self.num_examples += batch_size
        self.sum_hit_at_one += mean_hit_at_one * batch_size
        self.sum_perr += mean_perr * batch_size
        self.sum_loss += mean_loss * batch_size

        return {"hit_at_one": mean_hit_at_one, "perr": mean_perr, "loss": mean_loss}

    def get(self):
        """Calculate the evaluation metrics for the whole epoch.

    Raises:
      ValueError: If no examples were accumulated.

    Returns:
      dictionary: a dictionary storing the evaluation metrics for the epoch. The
        dictionary has the fields: avg_hit_at_one, avg_perr, avg_loss, and
        aps (default nan).
    """
        if self.num_examples <= 0:
            raise ValueError("total_sample must be positive.")
        avg_hit_at_one = self.sum_hit_at_one / self.num_examples
        avg_perr = self.sum_perr / self.num_examples
        avg_loss = self.sum_loss / self.num_examples

        aps = self.map_calculator.peek_map_at_n()
        gap = self.global_ap_calculator.peek_ap_at_n()

        epoch_info_dict = {}
        return {"avg_hit_at_one": avg_hit_at_one, "avg_perr": avg_perr, "avg_loss": avg_loss, "aps": aps, "gap": gap}

    def clear(self):
        """Clear the evaluation metrics and reset the EvaluationMetrics object."""
        self.sum_hit_at_one = 0.0
        self.sum_perr = 0.0
        self.sum_loss = 0.0
        self.map_calculator.clear()
        self.global_ap_calculator.clear()
        self.num_examples = 0


from django.shortcuts import render, redirect
from user_profile.models import UserInfo
from .models import OrderItem,Order
from cart.cart import Cart
from pharmacy.report import render_to_pdf
import braintree
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse,reverse_lazy
from django.utils.decorators import method_decorator
from django.views import generic
from . import forms
from pharmacy import settings
from django.utils.translation import ugettext_lazy as _
from .forms import    CheckoutForm
from django.http import HttpResponse
from django.conf import settings
from decimal import Decimal
from django.core.mail import send_mail
from django.template.loader import render_to_string

def order_info(request):
    user=request.user
    info=UserInfo.objects.filter(user=request.user)
    cart = Cart(request)
    total = 0
    for item in cart:
        total+=(item['price']* int(item['quantity']))
    total= str(total)
    if request.method=='POST':
        info = UserInfo.objects.get(pk=request.POST['info'])
        order = Order.objects.create(info=info)
        order.shiping_method=request.POST['shiping_method']
        order.save()
        request.session['order.id'] = order.id
        for item in cart:
            OrderItem.objects.create(order=order,
                                     drug=item['drug'],
                                      price=item['price'],
                                     quantity=item['quantity'])

            cart.clear()
        return redirect(reverse('order:checkout'))
        #return reverse_lazy('orders:checkout',kwargs={'pk':pk})
        #return render(request, 'orders/checkout.html', {'order': order})
    else:

        return render(request,'orders/info-chek.html',{'info':info,'cart':cart,'user':user, 'total':total})

"""
Adds simple form view, which communicates with Braintree.

There are four steps to finally process a transaction:

1. Create a client token (views.py)
2. Send it to Braintree (js)
3. Receive a payment nonce from Braintree (js)
4. Send transaction details and payment nonce to Braintree (views.py)

"""

class CheckoutView(generic.FormView):
    """This view lets the user initiate a payment."""
    form_class = forms.CheckoutForm
    template_name = 'orders/checkout.html'
    success_url = 'order:checkout-successful'

    #def total_amount(self):
     #   request=self.request
      #  order_id = request.session['order.id']
        #order= Order.objects.get(pk = order_id)
        #amount = order.get_total_cost()
        #shipment = Decimal(order.shiping_method)
        #total_amount = amount + shipment
        #return render(request, {'total_amount':total_amount})

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        # We need the user to assign the transaction
        self.user = request.user
        # Ha! There it is. This allows you to switch the
        # Braintree environments by changing one setting
        if settings.BRAINTREE_PRODUCTION:
            braintree_env = braintree.Environment.Production
        else:
            braintree_env = braintree.Environment.Sandbox

        # Configure Braintree
        braintree.Configuration.configure(
            braintree_env,
            merchant_id=settings.BRAINTREE_MERCHANT_ID,
            public_key=settings.BRAINTREE_PUBLIC_KEY,
            private_key=settings.BRAINTREE_PRIVATE_KEY,
        )

        # Generate a client token. We'll send this to the form to
        # finally generate the payment nonce
        # You're able to add something like ``{"customer_id": 'foo'}``,
        # if you've already saved the ID
        self.braintree_client_token = braintree.ClientToken.generate({})
        return super(CheckoutView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        request = self.request
        order_id = request.session['order.id']
        order = Order.objects.get(pk=order_id)
        amount = order.get_total_cost()
        shipment = Decimal(order.shiping_method)
        total_amount = (amount + shipment)
        ctx = super(CheckoutView, self).get_context_data(**kwargs)
        ctx.update({
            'braintree_client_token': self.braintree_client_token,
            'total_amount':total_amount,
            'order':order,
            'amount':amount,
        })
        return ctx

    def form_valid(self, form):
        # Braintree customer info
        # You can, for sure, use several approaches to gather customer infos
        # For now, we'll simply use the given data of the user instance
        customer_kwargs = {
            "first_name": self.user.first_name,
            "last_name": self.user.last_name,
            "email": self.user.email,

        }

        # Create a new Braintree customer
        # In this example we always create new Braintree users
        # You can store and re-use Braintree's customer IDs, if you want to
        result = braintree.Customer.create(customer_kwargs)
        if not result.is_success:
            # Ouch, something went wrong here
            # I recommend to send an error report to all admins
            # , including ``result.message`` and ``self.user.email``

            context = self.get_context_data()
            # We re-generate the form and display the relevant braintree error
            context.update({
                'form': self.get_form(self.get_form_class()),
                'braintree_error': u'{} {}'.format(
                    result.message, _('Please get in contact.'))
            })
            return self.render_to_response(context)
        # If the customer creation was successful you might want to also
        # add the customer id to your user profile
        customer_id = result.customer.id
        """
        Create a new transaction and submit it.
        I don't gather the whole address in this example, but I can
        highly recommend to do that. It will help you to avoid any
        fraud issues, since some providers require matching addresses

        """
        address_dict = {
            "first_name": self.user.first_name,
            "last_name": self.user.last_name,
            "street_address":"300 E longleaf dr",
            "extended_address": 'auburn',
            "locality": 'auburn',
            #"region": 'state_or_region',
            "postal_code": '36832',
            #"country_code_alpha2": 'alpha2_country_code',
            #"country_code_alpha3": 'alpha3_country_code',
            #"country_name": 'country',
            #"country_code_numeric": 'numeric_country_code',
        }
        # You can use the form to calculate a total or add a static total amount
        # I'll use a static amount in this example
        request = self.request
        order_id = request.session['order.id']
        order = Order.objects.get(pk=order_id)
        amount = order.get_total_cost()
        shipment = Decimal(order.shiping_method)
        total_amount = amount + shipment
        result = braintree.Transaction.sale({
            "customer_id": customer_id,
            "amount": total_amount,
            "payment_method_nonce": form.cleaned_data['payment_method_nonce'],
            "descriptor": {
                #Definitely check out https://developers.braintreepayments.com/reference/general/validation-errors/all/python#descriptor
                "name": "COMPANY*test",

           },
            "billing": address_dict,
            "shipping": address_dict,
            "options": {
                # Use this option to store the customer data, if successful
                'store_in_vault_on_success': True,
                # Use this option to directly settle the transaction
                # If you want to settle the transaction later, use ``False`` and later on
                # ``braintree.Transaction.submit_for_settlement("the_transaction_id")``
                'submit_for_settlement': True,
            },
        })
        if not result.is_success:
            # Card could've been declined or whatever
            # I recommend to send an error report to all admins
            # , including ``result.message`` and ``self.user.email``

            context = self.get_context_data()
            context.update({
                'form': self.get_form(self.get_form_class()),
                'braintree_error': _(
                   'Your payment could not be processed. Please check your'
                    ' input or use another payment method and try again.')
            })
            return self.render_to_response(context)

        # Finally there's the transaction ID
        # You definitely want to send it to your database
        transaction_id = result.transaction.id
        # Now you can send out confirmation emails or update your metrics
        # or do whatever makes you and your customers happy :)
        order.paid = True
        order.save()
        # report
        pdf = render_to_pdf('report/shop-report.html', {'order':order,'total':total_amount})  # todo send fax and sms and email
        # email
        msg = render_to_string('email/shop-factor.html',
                               {'order': order,
                                'total':total_amount})
        send_mail('Thank you', 'your order is successfuly submited',
                  settings.DEFAULT_FROM_EMAIL, [self.request.user.email], fail_silently=False,html_message=msg),
        return super(CheckoutView, self).form_valid(form)

    def get_success_url(self):
        return reverse(self.success_url)

def Checkout_Successfull(request):
    return render(request, 'orders/created.html')


import os
import re
import threading
import time

def get_gateway_win():
    """获取网关的IP,不完整只能获取第一个网关IP"""
    cmd = 'ipconfig/all|find "IPv4"'
    lines = os.popen(cmd).readlines()
    if lines:
        for line in lines:
            result = re.findall(
                r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b", line)
            if result:
                return result[0]

def get_gateway_linux(net_ifac = None):
    if net_ifac is None:
        cmd = 'netstat -rn'
    else:
        cmd = 'netstat -rn |grep %s'%net_ifac
    lines = os.popen(cmd).readlines()
    if lines:
        for line in lines:
            result = re.findall(
                r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b", line)
            if result:
                return result[0]

def valid_ip_mac(ip):
    ping_cmd = 'ping -n 2 {}|find "100%"'.format(ip)
    lines = os.popen(ping_cmd).readlines()
    if lines:
        return None
    arp_cmd = 'arp -a {}|find "{}"'.format(ip,ip)
    lines = os.popen(arp_cmd).readlines()
    if lines:
        result = re.findall(
            r'[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}', lines[0])
        if result:
            return str(result[0]).replace('-', '').upper()

def find_all_valid_ip_mac():
    gateway = get_gateway_win()
    if gateway is None:
        return
    index = str(gateway).rindex('.')
    ip_pre = str(gateway)[0:index+1]
    for i in range(2,255):
        ip = ip_pre+str(i)
        mac = valid_ip_mac(ip)
        if mac:
            print(ip, mac)

def arp_a_win(gateway=None):
    if gateway is None:
        gateway = get_gateway_win()
    if gateway is None:
        return
    index = str(gateway).rindex('.')
    ip_pre = str(gateway)[0:index + 1]

    cmd = 'arp -a'
    lines = os.popen(cmd).readlines()
    res = []
    for line in lines:
        ip = None
        mac = None
        result = re.findall(
            r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b", line)
        if result:
            ip = result[0]
        result = re.findall(
            r'[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}', line)
        if result:
            mac = str(result[0]).replace('-', '').upper()
        if not ip is None and not mac is None and ip_pre in ip:
            res.append((ip, mac))
    return res

def arp_a_linux(gateway=None):
    if gateway is None:
        gateway = get_gateway_linux('wlan0')
    if gateway is None:
        return
    index = str(gateway).rindex('.')
    ip_pre = str(gateway)[0:index + 1]

    cmd = 'arp -a'
    lines = os.popen(cmd).readlines()
    res = []
    for line in lines:
        ip = None
        mac = None
        result = re.findall(
            r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b", line)
        if result:
            ip = result[0]
        result = re.findall(
            r'[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}-[0-9a-f]{2}', line)
        if result:
            mac = str(result[0]).replace('-', '').upper()
        if not ip is None and not mac is None and ip_pre in ip:
            res.append((ip, mac))
    return res

def ping_all_ip():
    global current
    current = 0

    def get_next():
        global current
        current += 1
        return current

    def ping():
        num = get_next()
        while num < 225:
            cmd = 'ping -n 1 192.168.10.%d' % num
            os.popen(cmd).readlines()
            # print(cmd)
            num = get_next()
    threads_live = []
    for i in range(224):
        thread = threading.Thread(target=ping)
        thread.start()
        threads_live.append(thread)
    while threads_live:
        temp = []
        for thread in threads_live:
            if thread.is_alive():
                temp.append(thread)
        threads_live = temp

print(time.time())
ping_all_ip()
print(time.time())
print(arp_a_win())
print(time.time())

# print(arp_a_win())
# find_all_valid_ip_mac()


#!/usr/bin/python3
#-*- coding:utf-8 -*-
"""
PyQt5 教程

菜单栏

by 张强 2019.10.1
"""
import sys,os
from PyQt5.QtWidgets import QMainWindow,QApplication,QAction,qApp
from PyQt5.QtGui import QIcon

print(os.getcwd())
class Example(QMainWindow):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        if sys.platform =='win32':
            exitAct = QAction(QIcon('exit.png'), '&Exit', self)
        else:
            exitAct = QAction(QIcon('/Users/zhangqiang/PycharmProjects/PyQt5/菜单和工具栏/exit.png'),'&Exit',self)

        exitAct.setShortcut('Ctrl+Q')
        exitAct.setStatusTip('Exit application')
        exitAct.triggered.connect(qApp.quit)

        self.statusBar()

        menubar = self.menuBar()
        fileMenu = menubar.addMenu('&File')
        fileMenu.addAction(exitAct)

        self.setGeometry(300,300,300,200)
        self.setWindowTitle('简单菜单')
        self.show()

if __name__ == '__main__':

    app = QApplication(sys.argv)
    exe = Example()
    sys.exit(app.exec_())


# 큐,덱 알고리즘
# 백준18258
#단어공부
import sys
from collections import deque
input=sys.stdin.readline
N=int(input())
queue=deque()
def command(str):

    if str.rsplit(' ')[0]=='push':

        queue.append(int(str.split(' ')[1].rstrip()))

    elif str.rstrip()=='pop':

        if len(queue) == 0:
            print(-1)
        else:

            num=queue.popleft()
            print(num)
    elif str.rstrip() == 'size':

        print(len(queue))

    elif str.rstrip() == 'empty':
        if len(queue)==0:
            print(1)
        else:
            print(0)
    elif str.rstrip() == 'front':
        if len(queue)==0:
            print(-1)
        else:
            num=queue.popleft()
            print(num)
            queue.appendleft(num)

    elif str.rstrip() == 'back':
        if len(queue)==0:
            print(-1)
        else:
            num = queue.pop()
            print(num)
            queue.append(num)

for i in range(N):
    command(input())


import pandas as pd
import numpy as np
import math

#Reading Primary Data
datasetP = pd.read_csv('../data/AllBooks_baseline_DTM_Labelled.csv')

# print(datasetP.shape)

#Dropping the Empty Row from data
datasetP = datasetP.drop([13,13])
# print(datasetP.shape)

#print(datasetP.head())

dftMat = np.asarray(datasetP)

#Separating the Labels
label = dftMat[:, 0]
#print(label)

#Separating the DFT Matrix
X = dftMat[:, 1:]

#Removing the Term after _ Buddishm_Ch14 --> Buddishm
for i in range (0, len(label)):
    listName = label[i].split('_')
    label[i] = listName[0];

#print(label)
# print(X.shape)

df = np.zeros(( X.shape[1]))

#Calculating the df, Document Frequecy for Each term -> Number of Documents in which the term is present
#If Frequency >0 it means the term is present in the document, and thus will in the np array obtain by the boolean condition

# print(df.shape)
for i in range (0 , X.shape[1]):
    temp= X[:, i]
    df[i] = temp[temp > 0].shape[0]

print("DF Calculated")
#df = np.reshape(df, (1, len(df)))
#print(df)

#Storing the value of the formula in a temporary variable
temp = (1 + X.shape[0])/(1 + df)

for i in range(0, len(temp)):
    temp[i] = math.log(temp[i])

# print(temp.shape)

idf = temp;
tfidf = np.zeros((X.shape))

#Final TDFIF matrix calcualtion by the given formula
for i in range(0, X.shape[0]):
    #print(X[i])
    tfidf[i] = X[i]*idf

count = 0

print("TF-IDF Calculated")

vec_len = np.zeros((X.shape[0]))

#Calculating the vector length for all the vectors
for i in range (0, X.shape[0]):
    vec_len[i] = math.sqrt(tfidf[i].dot(tfidf[i]))

#print(vec_len == 0)

#Normalisation done for the final Array. Diving by the vector length -> that is sqrt of sum of squares

for i in range(0, X.shape[0]):
    tfidf[i] = tfidf[i]/vec_len[i]
print("Normalised TFIDF Calculated")

#Saving the tfdif matrix as a csv for the next assignment
np.savetxt('../data/tfidf.csv', tfidf)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from typing import Dict, Any, List
import warnings

from aws_lambda_powertools import Logger, Metrics, Tracer
from aws_lambda_powertools.utilities.typing import LambdaContext
import boto3
from crhelper import CfnResource

from controltowersetup import (
    AccessAnalyzer,
    GuardDuty,
    Macie,
    Organizations,
    RAM,
    ServiceCatalog,
)

warnings.filterwarnings("ignore", "No metrics to publish*")

tracer = Tracer()
logger = Logger()
metrics = Metrics()
helper = CfnResource(json_logging=True, log_level="INFO", boto_level="INFO")
CT_AUDIT_ACCOUNT_NAME = "Audit"

SERVICE_ACCESS_PRINCIPALS = {
    "backup.amazonaws.com",
    "config.amazonaws.com",
    "config-multiaccountsetup.amazonaws.com",
    "guardduty.amazonaws.com",
    "macie.amazonaws.com",
}

DELEGATED_ADMINISTRATOR_PRINCIPALS = {
    "access-analyzer.amazonaws.com",
    "config-multiaccountsetup.amazonaws.com",
    "guardduty.amazonaws.com",
    "macie.amazonaws.com",
}

@tracer.capture_method
def setup_organization(
    organizations: Organizations,
    regions: List[str] = None,
    audit_account_id: str = None,
) -> bool:
    """
    Set up the organization in multiple regions
    """
    logger.info(f"Enabling GuardDuty, Security Hub and Macie in regions: {regions}")

    # enable all organizational policy types
    organizations.enable_all_policy_types()

    # attach an AI service opt-out policy
    organizations.attach_ai_optout_policy()

    # enable Service Catalog access to the organization
    ServiceCatalog().enable_aws_organizations_access()

    # enable various AWS service principal access to the organization
    organizations.enable_aws_service_access(SERVICE_ACCESS_PRINCIPALS)

    # enable RAM sharing to the organization
    RAM().enable_sharing_with_aws_organization()

    if not audit_account_id:
        logger.error("Unable to find Control Tower Audit account")
        return True

    # Create organization IAM access analyzer in the Control Tower Audit account
    AccessAnalyzer.create_org_analyzer(audit_account_id)

    for region in regions:
        guardduty = GuardDuty(region)

        # delegate GuardDuty administration to the Control Tower Audit account
        guardduty.enable_organization_admin_account(audit_account_id)

        # update the GuartDuty organization configuration to register new accounts automatically
        guardduty.update_organization_configuration(audit_account_id)

        macie = Macie(region)

        # delegate Macie administration to the Control Tower Audit account
        macie.enable_organization_admin_account(audit_account_id)

        # update the Macie organization configuration to register new accounts automatically
        macie.update_organization_configuration(audit_account_id)

    # Register the Control Tower Audit account as a delegated administer on AWS services
    organizations.register_delegated_administrator(
        audit_account_id, DELEGATED_ADMINISTRATOR_PRINCIPALS
    )

    return True

@tracer.capture_method
def get_all_regions() -> List[str]:
    """
    Return all regions
    """
    ec2 = boto3.client("ec2")
    regions = [
        region["RegionName"]
        for region in ec2.describe_regions(
            Filters=[{"Name": "opt-in-status", "Values": ["opt-in-not-required"]}],
            AllRegions=False,
        )["Regions"]
    ]
    return regions

@helper.create
@helper.update
def create(event: Dict[str, Any], context: LambdaContext) -> bool:
    logger.info("Got Create or Update")

    properties = event.get("ResourceProperties", {})
    regions = properties.get("Regions", "").split(",")
    if not regions:
        regions = get_all_regions()

    organizations = Organizations()

    audit_account_id = organizations.get_audit_account_id()

    return setup_organization(organizations, regions, audit_account_id)

@helper.delete
def delete(event: Dict[str, Any], context: LambdaContext) -> None:
    logger.info("Got Delete")

@metrics.log_metrics(capture_cold_start_metric=True)
@tracer.capture_lambda_handler
@logger.inject_lambda_context(log_event=True)
def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> None:

    if event.get("eventName") == "SetupLandingZone":
        regions = [event["awsRegion"]]
        audit_account_id = None
        accounts = (
            event.get("serviceEventDetails", {})
            .get("setupLandingZoneStatus", {})
            .get("accounts", [])
        )
        for account in accounts:
            if account["accountName"] == CT_AUDIT_ACCOUNT_NAME:
                audit_account_id = account["accountId"]
        return setup_organization(regions, audit_account_id)

    helper(event, context)


from django.core.management import BaseCommand

from common.models import Status
from user_apps.models import Plan

class Command(BaseCommand):
    help = "Data Migrations For Subscription Plans"

    def handle(self, *args, **kwargs):
        """Data migrations for Common Statuses"""
        TABLE_NAME = "Plan"
        record = Plan.objects.get_or_create(
            id=1, plan_name="Free", plan_price=0
        )
        record = Plan.objects.get_or_create(
            id=2, plan_name="Standard", plan_price=10
        )
        record = Plan.objects.get_or_create(
            id=3, plan_name="Pro", plan_price=25
        )

        self.stdout.write(
            self.style.SUCCESS(f"Data migrations for '{TABLE_NAME}' ran successfully!")
        )

        TABLE_NAME = "Status"
        record = Status.objects.get_or_create(
            id=1, name="Active", code="A", description="Record is active"
        )
        record = Status.objects.get_or_create(
            id=2, name="Deleted", code="D", description="Record is deleted"
        )
        self.stdout.write(
            self.style.SUCCESS(f"Data migrations for '{TABLE_NAME}' ran successfully!")
        )