#!/usr/bin/python
# -*- coding: utf-8 -*-

from collections import OrderedDict, defaultdict
from .utils import LowerStringKey
from ..lexicon.manual import ManualLexicon
import Levenshtein
from fim import fim

_UID = 0

def _uid():
    global _UID
    _UID += 1
    return _UID

# ------- DEFINITIONS----------------------------------------------------------

MODS = frozenset(['mod', 'pnmod'])
OBJS = frozenset(['subj', 's', 'obj'])
MR = frozenset(['obj2', 'desc', 'mod', 'pnmod', 'subj', 's', 'obj'])
CONJ = frozenset(['conj'])
JJ = frozenset(['JJ', 'JJR', 'JJS'])
NN = frozenset(['NN', 'NNS'])
# NN = frozenset(['NN', 'NNS', 'NNP'])

# adding "n't" and "'t" for tokenization compatibility
NEGATIONS = ['not', "n't", "'t", 'however', 'but', 'despite', 'though',
             'except', 'although', 'oddly', 'aside']
CONJUNCTIONS = ['and', 'or']
PRODUCT_PHRASES = [['compare', 'to'], ['compare', 'with'],
                   ['compare', None, 'to'], ['compare', None, None, 'to'],
                   ['compare', None, 'with'], ['compare', None, None, 'with'],
                   ['better', 'than'], ['worse', 'than']]
DEALER_PHRASES = [['shop', 'with'], ['buy', 'from']]
NEGATION_LIMIT = 5
Q = 2
K = 1

BASIC_SEEDS = ManualLexicon('Base Seeds')
BASIC_SEEDS["good"] = 1.0
BASIC_SEEDS["bad"] = -1.0

# ------- MAIN CLASS ----------------------------------------------------------

class DoublePropagation(object):

    def __init__(self,
                 seeds=BASIC_SEEDS,
                 negations=NEGATIONS,
                 negation_limit=NEGATION_LIMIT,
                 conjunctions=CONJUNCTIONS,
                 product_phrases=PRODUCT_PHRASES,
                 dealer_phrases=DEALER_PHRASES,
                 Q=Q,
                 K=K,
                 key=LowerStringKey(),
                 match=90):
        """
        Input:
            - seeds            : dict of seed opinion words to use, (each is
                                 an object with .string, .orientation and
                                 .stem attributes)
                                 default=liu_lexicon
            - negations        : list of strings, words that convey negation
                                 default=NEGATIONS
            - negation_limit   : int, maximum distance to look for negations
                                 default=NEGATION_LIMIT
            - conjunctions     : list of strings, conjunctions to check aspects
                                 in a clause, default=CONJUNCTIONS
            - product_phrases  : lisf of lists of strings, used in product
                                 phrase pruning, default=PRODUCT_PHRASES
            - dealer_phrases   : DEALER_PHRASES
            - Q                : int, Q parameter
            - K                : int, K parameter
            - key              : function, to apply to each word before
                                 extraction, LowerStringKey=lambda x: x.string.lower
            - match            : int (0-100), percentage of similarity for matching
                                 terms (use 100 for exact), default=90
        Output:
            tuple: (list aspects, list opinion_words)
        """

        self.params = {"seeds": seeds,
                       "negations": negations,
                       "negation_limit": negation_limit,
                       "conjunctions": conjunctions,
                       "product_phrases": product_phrases,
                       "dealer_phrases": dealer_phrases,
                       "Q": Q,
                       "K": K,
                       "key": key,
                       "match": match}

        if match < 100:
            self.matching = ExactMatching(key=key)
        else:
            self.matching = FuzzyMatching(match=match, key=key)

    def extract(self, sentences):
        """
        Returns the extracted aspects and opinion words.
        Input:
            - sentences: list of sentences
        Output:
            - tuple(list aspects, list opinion_words)
        """
        seeds = self.params["seeds"]
        negations = self.params["negations"]
        negation_limit = self.params["negation_limit"]
        conjunctions = self.params["conjunctions"]
        product_phrases = self.params["product_phrases"]
        dealer_phrases = self.params["dealer_phrases"]
        Q = self.params["Q"]
        K = self.params["K"]
        key = self.params["key"]
        matching = self.matching

        global Aspect
        global OpinionWord

        Aspect = type("Aspect", (Aspect,), {"negations": negations,
                                            "negation_limit": negation_limit})
        OpinionWord = type("OpinionWord", (OpinionWord,), {"negations": negations,
                                                           "negation_limit": negation_limit})

        aspects, opinion_words = double_propagation(sentences, seeds, Aspect, OpinionWord, matching, key)
        clause_pruning(sentences, aspects, opinion_words, conjunctions)
        product_pruning(sentences, aspects, opinion_words, product_phrases, dealer_phrases)
        phrase_aspects = get_phrase_aspects(aspects, Q, K, Aspect, matching, key)
        aspects += phrase_aspects

        # reset sentences
        for sentence in sentences:
            del sentence.aspects
            del sentence.opinion_words

        return aspects, opinion_words

# ------- MATCHING FUNCTIONS --------------------------------------------------

class ExactMatching(object):

    def __init__(self, key=lambda x: x.string.lower()):
        """
        Strictly compares the given token to term objects. For comparison,
        applies the function given in "key" parameter to the
        Token/tuple of Tokens.

        Input:
            key   : function to apply to the token,
            default=lambda x: x.string.lower()
        """
        self.key = key

    def __call__(self, token_tuple, terms):
        """
        Input:
            token_tuple : Token or tuple of Token objects
            terms       : term or iterable terms of terms to match

        Output:
            Returns None if no match is found.
            Returns the first term that matches the given token.
        """
        if not hasattr(terms, '__iter__'):
            terms = [terms]
        if not isinstance(token_tuple, tuple):
            token_tuple = (token_tuple,)
        try:
            token_tuple = tuple(self.key(token) for token in token_tuple)
        except Exception as e:
            token_tuple = tuple(str(token) for token in token_tuple)

        for term in terms:
            if any([token_tuple == term_i for term_i in term]):
                # print "compared", token, best_term
                return term
        else:
            # print "compared", token, None
            return None

class FuzzyMatching(object):

    def __init__(self, match=90, key=lambda x: x.string.lower()):
        """
        Fuzzy matching between the given token and term objects. For comparison
        applies the function given in the "key" parameter to the Token/tuple
        of Tokens. Parameter match defines the minimum similarity ratio for
        a match when comparing.

        Input:
            match : minimum similarity for fuzzy matching (%)
            key   : function to apply to the token,
                    default=lambda x: x.string.lower()
        """
        self.match = match
        self.key = key

    def __call__(self, token_tuple, terms):
        """
        Input:
            token_tuple : Token or tuple of Token objects
            terms       : term or iterable of terms to match

        Output:
            Returns None if no match is found.
            Returns the first matched in case many of them show the same
            similarity ratio.
        """
        if not hasattr(terms, '__iter__'):
            terms = [terms]
        if not isinstance(token_tuple, tuple):
            token_tuple = (token_tuple,)
        try:
            token_tuple = tuple(self.key(token) for token in token_tuple)
        except Exception:  # as e
            token_tuple = tuple(str(token) for token in token_tuple)

        best_term = None
        best_ratio = 0

        for term in terms:
            ratio = max([Levenshtein.ratio(unicode(" ".join(token_tuple)),
                                           unicode(" ".join(term_i)))*100
                         for term_i in term])
            if ratio >= self.match and ratio > best_ratio:
                best_term = term
                best_ratio = ratio

        return best_term

# ------- UTIL FUNCTIONS ------------------------------------------------------

def equals(rel_a, rel_b):
    """
    Here equivalent specifically means mod is the same as pnmod,
    and s or subj is the same as obj)
    """
    if rel_a == rel_b:
        return True
    elif (rel_a == 'mod' or rel_b == 'mod') \
    and (rel_a == 'pnmod' or rel_b == 'pnmod'):
        return True
    elif (rel_a == 's' or rel_b == 's') \
    and (rel_a == 'subj' or rel_b == 'subj'):
        return True
    elif (rel_a == 'subj' or rel_b == 'subj') \
    and (rel_a == 'obj' or rel_b == 'obj'):
        return True
    elif (rel_a == 'subj' or rel_b == 'subj') \
    and (rel_a == 's' or rel_b == 's'):
        return True
    else:
        return False

def stanford_to_minipar(relation):
    if relation == "amod":
        return "mod"
    elif "prep" in relation:
        return "pnmod"
    # to include collapsed dependencies
    elif "nsubj" in relation:
        return "subj"
    # to include collapsed dependencies
    elif "csubj" in relation:
        return "subj"
    elif relation == "xsubj":
        return "obj"
    elif relation == "dobj":
        return "obj2"
    elif relation == "iobj":
        return "desc"
    # to include collapsed dependencies
    elif "conj" in relation:
        return "conj"
    else:
        return relation

# ------- EXTRACTION RULES ----------------------------------------------------

def apply_rule_1_1(sentence, opinion_words, matching):
    """
    Extracts aspects based on known opinion words and direct dependency
    using the rule: (we don't force the pos-tag of the matched
    token to be in JJ)

    T → O-Dep → O
        s.t
            O ∈ { O }
            O-Dep ∈ { MR }
            POS(T) ∈ { NN }
        then  t = T

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    aspects = []
    for relation in sentence.relations:
        head = relation.head
        label = stanford_to_minipar(relation.label)
        dependent = relation.dependent
        match = matching(dependent, opinion_words)
        if head.pos in NN \
        and label in MR \
        and match:
            aspects.append((match, dependent, head))
    return aspects

def apply_rule_1_2(sentence, opinion_words, matching):
    """
    Extracts aspects based on known opinion words and indirect dependency using the rule:
    (we don't force the pos-tag of the matched token to be in JJ)

    O ← O-Dep ← H → T-Dep → T
    s.t.
        H any word
        O ∈ { O }
        O/T-Dep ∈ { MR }
        POS(T) ∈ { NN }
    then t = T

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)

    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    aspects = []
    for token in sentence:
        match = matching(token, opinion_words)
        if match \
        and token.head:
            pivot, label_a = token.head
            if stanford_to_minipar(label_a) in MR:
                for dependent, label_b in pivot.dependents:
                    if stanford_to_minipar(label_b) in MR \
                    and dependent is not token  \
                    and dependent.pos in NN:
                        # print token
                        # print 'v ' + label_a
                        # print pivot
                        # print '^ ' + label_b
                        # print dependent
                        # print ""
                        aspects.append((match, token, dependent))
    return aspects

def apply_rule_2_1(sentence, aspects, matching):
    """
    Extracts opinion words based on known aspects and direct dependency.
    (here we force the pos-tag of the matched token to be in NN)

    T → O-Dep → O
    s.t.
        T ∈ { T }
        O-Dep ∈ { MR }
        POS(O) ∈ { JJ }
    Output: o = O

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    opinion_words = []
    for relation in sentence.relations:
        head = relation.head
        label = stanford_to_minipar(relation.label)
        dependent = relation.dependent
        if head.pos in NN:
            match = matching(head, aspects)
            if match \
            and dependent.pos in JJ \
            and label in MR:
                # print head, label, dependent
                opinion_words.append((match, head, dependent))
    return opinion_words

def apply_rule_2_2(sentence, aspects, matching):
    """
    Extracts opinion words based on known aspects and indirect dependency.
    (here we force the pos-tag of the matched token to be in NN)

    O ← O-Dep ← H → T-Dep → T
    s.t.
        T ∈ { T }
        O/T-Dep ∈ { MR }
        POS(O) ∈ { JJ }
    Output: o = O

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    opinion_words = []
    for token in sentence:
        if token.pos in NN \
        and token.head:
            match = matching(token, aspects)
            if match:
                pivot, label_a = token.head
                if stanford_to_minipar(label_a) in MR:
                    for dependent, label_b in pivot.dependents:
                        if stanford_to_minipar(label_b) in MR \
                        and dependent is not token  \
                        and dependent.pos in JJ:
                            # print token
                            # print 'v ' + label_a
                            # print pivot
                            # print '^ ' + label_b
                            # print dependent
                            # print ""
                            opinion_words.append((match, token, dependent))
    return opinion_words

def apply_rule_3_1(sentence, aspects, matching):
    """
    Extracts aspects using known aspects and the conjunction relationship.
    (here we force the pos-tag of the matched token to be in NN)

    T_i(j) → T_i(j)-Dep → T_j(i)
    s.t.
        T_j(i) ∈ { T }
        T_i(j)-Dep ∈ { CONJ }
        POS(T_i(j)) ∈ { NN }
    Output: t = T_i(j)

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    new_aspects = []
    for relation in sentence.relations:
        head = relation.head
        if head.pos in NN:
            label = stanford_to_minipar(relation.label)
            dependent = relation.dependent
            if label in CONJ \
            and dependent.pos in NN:
                match = matching(head, aspects)
                if match:
                    # print head, label, dependent
                    new_aspects.append((match, head, dependent))
    return new_aspects

def apply_rule_3_2(sentence, aspects, matching):
    """
    Extracts aspects using known aspects and indirect same class dependency.
    (here we force the pos-tag of the matched token to be in NN)

    T_i ← T_i-Dep ← H → T_j-Dep → T_j
    s.t
        H any word.string
        T_i ∈ { T }
        T_i -Dep == T_j-Dep
        POS(T_j) ∈ { NN }
    Then, t = T_j

    Returns a list of extracted tuples:
         (existing Term, matched Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    new_aspects = []
    for token in sentence:
        if token.pos in NN \
        and token.head:
            match = matching(token, aspects)
            if match:
                pivot, label_a = token.head
                label_a = stanford_to_minipar(label_a)
                for dependent, label_b in pivot.dependents:
                    label_b = stanford_to_minipar(label_b)
                    if equals(label_a, label_b) \
                    and dependent is not token  \
                    and dependent.pos in NN:
                        # print token
                        # print 'v ' + label_a
                        # print pivot
                        # print '^ ' + label_b
                        # print dependent
                        # print ""
                        new_aspects.append((match, token, dependent))
    return new_aspects

def apply_rule_4_1(sentence, opinion_words, matching):
    """
    Extract opinion words based on known opinion words and the conjunction
    relationship.
    Ex: “Bill is big and honest” conj(big, honest)
    (we don't force the pos-tag of the matched opinion word to be in JJ)

    O_i(j) → O_i(j)-Dep → O_j(i)
    s.t.
        O_j(i) ∈ { O } ,
        O_i(j)-Dep ∈ { CONJ }
        POS(O_i(j)) ∈ { JJ }
    Output: o = O_i(j)

    Returns a list of extracted tuples:
         (existing Token, extracted Token)
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    new_opinion_words = []
    for relation in sentence.relations:
        head = relation.head
        label = stanford_to_minipar(relation.label)
        dependent = relation.dependent
        match = matching(head, opinion_words)
        if label in CONJ \
        and dependent.pos in JJ \
        and match:
            # print head, label, dependent
            new_opinion_words.append((match, head, dependent))
    return new_opinion_words

def apply_rule_4_2(sentence, opinion_words, matching):
    """
    Extract opinion words based on known opinion words and indirect dependency.
    (we don't force the pos-tag of the matched token to be in JJ)

    O_i ← O_i-Dep ← H → O_j-Dep → O_j
    s.t.
        H any word
        O_i ∈ { O }
        O_i-Dep == O_j-Dep
        POS(O_j) ∈ { JJ }
    Output: o = O_j
    """
    # if not sentence.is_parsed:
    #    raise TypeError("Sentence is not parsed.")
    new_opinion_words = []
    for token in sentence:
        match = matching(token, opinion_words)
        if match \
        and token.head:
            pivot, label_a = token.head
            label_a = stanford_to_minipar(label_a)
            for dependent, label_b in pivot.dependents:
                label_b = stanford_to_minipar(label_b)
                if equals(label_a, label_b) \
                and dependent is not token \
                and dependent.pos in JJ:
                    # print token
                    # print 'v ' + label_a
                    # print pivot
                    # print '^ ' + label_b
                    # print dependent
                    # print ""
                    new_opinion_words.append((match, token, dependent))
    return new_opinion_words

def double_propagation(sentences, seeds, Aspect, OpinionWord,
                       matching=ExactMatching(), key=lambda x: x.stem):
    """
    Input:
        sentences   : list of :Sentences
        seeds       : dict of seeds
        Aspect      : class Aspect
        OpinionWord : class OpinionWord
        matching    : matching function
    """

    aspects = []
    opinion_words = []

    for seed in seeds.values():
        match = matching(seed, opinion_words)
        if match:
            match.add_seed(key, seed)
        else:
            opinion_word = OpinionWord(key, seed=seed)
            opinion_words.append(opinion_word)

    # prepare sentences
    for sentence in sentences:
        sentence.aspects = []
        sentence.opinion_words = []

    while True:
        aspects_i = []
        opinion_words_i = []

        # first loop, using all aspects
        for sentence in sentences:
            # extract aspects using all known opinion words and DD
            for existing, matched, extracted in apply_rule_1_1(sentence, opinion_words, matching):
                do_aspect(aspects, existing, matched, extracted,
                          aspects_i, Aspect, matching, key)

            # extract aspects using all known opinion words and ID
            for existing, matched, extracted in apply_rule_1_2(sentence, opinion_words, matching):
                do_aspect(aspects, existing, matched, extracted,
                          aspects_i, Aspect, matching, key)

            # extract opinion words based on all known opinion words and conjunction
            for existing, matched, extracted in apply_rule_4_1(sentence, opinion_words, matching):
                do_opinion_word(opinion_words, existing, matched, extracted,
                                opinion_words_i, OpinionWord, matching, key)

            # extract opinion words based on all known opinion words and ID
            for existing, matched, extracted in apply_rule_4_2(sentence, opinion_words, matching):
                do_opinion_word(opinion_words, existing, matched, extracted,
                                opinion_words_i, OpinionWord, matching, key)

        aspects += aspects_i
        opinion_words += opinion_words_i

        # print "Iter 1"
        # print(len(aspects_i))
        # print(len(opinion_words_i))
        # print "---"

        aspects_star = []
        opinion_words_star = []

        # second loop, using recently extracted aspects
        for sentence in sentences:
            # extract aspects using recently aspects and conjunction
            for existing, matched, extracted in apply_rule_3_1(sentence, aspects_i, matching):
                do_aspect(aspects, existing, matched, extracted,
                          aspects_star, Aspect, matching, key)

            # extracts aspects using recently extracted aspects and class DD
            for existing, matched, extracted in apply_rule_3_2(sentence, aspects_i, matching):
                do_aspect(aspects, existing, matched, extracted,
                          aspects_star, Aspect, matching, key)

            # extracts opinion words based on recently extracted aspects and DD
            for existing, matched, extracted in apply_rule_2_1(sentence, aspects_i, matching):
                do_opinion_word(opinion_words, existing, matched, extracted,
                                opinion_words_star, OpinionWord, matching, key)

            # extract opinion words based on recently extracted aspects and ID
            for existing, matched, extracted in apply_rule_2_2(sentence, aspects_i, matching):
                do_opinion_word(opinion_words, existing, matched, extracted,
                                opinion_words_star, OpinionWord, matching, key)

        aspects += aspects_star
        opinion_words += opinion_words_star

        aspects_i += aspects_star
        opinion_words_i += opinion_words_star

        # print "Iter 2"
        # print(len(aspects_star))
        # print(len(opinion_words_star))
        # print "---"

        if len(aspects_i) == 0 and len(opinion_words_i) == 0:
            break

    return aspects, opinion_words

def do_aspect(aspects, existing, matched, extracted,
              new_aspects, Aspect, matching, key):
    """
    aspects     : dict of Aspect objects
    existing    : existing Term (OpinionWord or Aspect) used in rule
    matched     : Token matching the Term used in rule
    extracted   : Token extracted by rule
    matching    : matching function
    new_aspects : dict to add new Aspects
    """
    # append the matched Token to the existing Term
    appended = existing.append(key, existing, (matched,))
    extracted = (extracted,)
    # check if we have a "new" Aspect that matches the extracted Token
    new_aspect = matching(extracted, new_aspects)
    if new_aspect:
        # if new_match, add Token to matched "new" Aspect]
        new_aspect.append(key, appended, extracted)
    else:
        # check if we have an existing Aspect that matches the extracted Token
        aspect = matching(extracted, aspects)
        # if matches, add Token to matched Aspect
        if aspect:
            aspect.append(key, appended, extracted)
        else:
            # if new add it to "new" Aspects
            new_aspect = Aspect(key, appended, extracted)
            new_aspects.append(new_aspect)

def do_opinion_word(opinion_words, existing, matched, extracted,
                    new_opinion_words, OpinionWord, matching, key):
    """
    Function that processes the extracted opinion word.
    opinion_words     : dict of OpinionWord objects
    existing          : existing Term (OpinionWord or Aspect) used in rule
    matched           : Token matching the Term used in rule
    extracted         : Token extracted by rule
    matching          : matching function
    new_opinion_words : dict to add new OpinionWords here
    """
    # append the matched Token to the existing Term
    appended = existing.append(key, existing, (matched,))
    extracted = (extracted,)

    # check if we have a "new" OpinionWord that matches the extracted Token
    new_opinion_word = matching(extracted, new_opinion_words)
    if new_opinion_word:
        # if new_match, add Token to matched "new" OpinionWord
        new_opinion_word.append(key, appended, extracted)
    else:
        # check if we have an existing OpinionWord that matches
        # the extracted Token
        opinion_word = matching(extracted, opinion_words)
        # if matches, add Token to matched OpinionWord
        if opinion_word:
            opinion_word.append(key, appended, extracted)
        else:
            # if new add it to "new" OpinionWords
            new_opinion_word = OpinionWord(key, appended, extracted)
            new_opinion_words.append(new_opinion_word)

# ------- ORIENTATION RULES ---------------------------------------------------

# In theory, some functions below can be added as methods to Sentence or
# Document objects in module "reps.py". I decided not to do so to enhance
# clarity and for completeness of the implementation of DoublePropagation

def has_near_negation(token, negations, limit):
    sentence = token.sentence
    pos_1 = token.index-limit if token.index-limit >= 0 else 0
    pos_2 = token.index+limit if token.index+limit <= len(sentence) else len(sentence)
    return negations_between(negations, sentence=sentence,
                             pos_1=pos_1, pos_2=pos_2)

def negations_between(negations, **kwargs):
    """
    Counts the negations between the given tokens/positions
    Only kwargs.
    """
    token_1 = kwargs.get('token_1', None)
    token_2 = kwargs.get('token_1', None)
    sentence = kwargs.get('sentence', None)
    pos_1 = kwargs.get('pos_1', 0)
    pos_2 = kwargs.get('pos_2', None)

    if token_1 and token_2:
        if token_1.sentence is not token_2.sentence:
            raise Exception('Tokens need to belong to the same sentence.')
        sentence = token_1.sentence
        pos_1 = min(token_1.index, token_2.index)
        pos_2 = max(token_1.index, token_2.index)
        return len([token for token in sentence[pos_1:pos_2]
                    if token.string.lower() in negations])
    elif sentence:
        if pos_2 is None:
            pos_2 = len(sentence)
        return len([token for token in sentence[pos_1:pos_2]
                    if token.string.lower() in negations])
    else:
        raise Exception('You have to provide either 2 tokens or a sentence')

def review_orientation(review):
    orientation = 0.0
    for sentence in review.sentences:
        for opinion_word in sentence.opinion_words:
            # print opinion_word
            orientation += opinion_word.orientation
    return 1.0 if orientation > 0 else -1.0

# ------- PRUNING FUNCTIONS ---------------------------------------------------

def has_conjunction(sentence, start, end, conjunctions):
    for word in conjunctions:
        if word in [token.string.lower() for token in sentence[start:end]]:
            return True
    else:
        return False

def max_support(aspects):
    return max(aspects, key=lambda a: a.term.support)

def clean_empty_terms(terms, min=0):
    for index, term in enumerate(terms):
        if term.support <= min:
            terms.pop(index)

def clear_extracted_by_pruned(terms, deletes):
    """Recursively delete objects that were extracted by pruned aspects"""
    new_deletes = []
    while True:
        for term in terms:
            for sentence in term.sentences:
                existing, extracted = sentence.history[0]
                if existing in deletes:
                    delete = sentence.delete()
                    new_deletes.append(delete)
        if new_deletes:
            deletes = new_deletes
        break

def clause_pruning(sentences, aspects, opinion_words, conjunctions):
    deletes = []
    for sentence in sentences:
        sentence_aspects = sentence.aspects
        if len(sentence_aspects) > 1:
            clauses = sentence.clauses
            if len(clauses) == 0:
                max_aspect = max_support(sentence_aspects)
                for aspect in sentence_aspects:
                    if aspect != max_aspect:
                        delete = aspect.delete()
                        deletes.append(delete)
                sentence.aspects = [max_aspect]

            else:
                positions = [0]
                for clause in clauses:
                    positions += [clause.start, clause.stop]
                positions += [len(sentence)]
                # reset sentence.aspects
                sentence.aspects = []
                for i in range(len(positions)-1):
                    start, stop = positions[i:i+2]
                    segment_aspects = [aspect for aspect in sentence_aspects
                                      if start <= aspect.positions[0] < stop]
                    if len(segment_aspects) > 1 and not has_conjunction(sentence, start, stop, conjunctions):
                        max_aspect = max_support(segment_aspects)
                        for aspect in segment_aspects:
                            if aspect != max_aspect:
                                delete = aspect.delete()
                                deletes.append(delete)
                        sentence.aspects.append(max_aspect)

    clean_empty_terms(aspects)
    clear_extracted_by_pruned(aspects, deletes)
    clear_extracted_by_pruned(opinion_words, deletes)
    clean_empty_terms(aspects)
    clean_empty_terms(opinion_words)

def product_pruning(sentences, aspects, opinion_words,
                    product_phrases, dealer_phrases):
    deletes = []
    for sentence in sentences:
        for phrase in product_phrases:
            hit = sentence.contains(phrase, key=lambda x: x.stem)
            if hit:
                # get the nearest noun folowing the prhase
                start, stop = hit
                nouns = [noun for noun in sentence.nouns if noun.index > stop]
                if nouns:
                    if len(nouns) > 1:
                        noun = min(nouns, key=lambda n: abs(n.index-stop))
                    else:
                        noun = nouns[0]
                else:
                    noun = None
                # do pruning
                if noun:
                    for aspect in sentence.aspects:
                        if aspect.tokens[0] is noun:
                            delete = aspect.delete()
                            deletes.append(delete)

        for phrase in dealer_phrases:
            hit = sentence.contains(phrase, key=lambda x: x.stem)
            if hit:
                # get the nearest noun before the phrase
                start, stop = hit
                nouns = [noun for noun in sentence.nouns if noun.index < start]
                if nouns:
                    if len(nouns) > 1:
                        noun = min(nouns, key=lambda n: abs(n.index-start))
                    else:
                        noun = nouns[0]
                else:
                    noun = None
                # do pruning
                if noun:
                    for aspect in sentence.aspects:
                        if aspect.tokens[0] is noun:
                            delete = aspect.delete()
                            deletes.append(delete)

    clean_empty_terms(aspects)
    clear_extracted_by_pruned(aspects, deletes)
    clear_extracted_by_pruned(opinion_words, deletes)
    clean_empty_terms(aspects)
    clean_empty_terms(opinion_words)

def get_phrase_aspects(aspects, Q, K, Aspect, matching, key):

    transactions = {}
    existings = defaultdict(list)

    for aspect in aspects:
        for ais in aspect.sentences:
            existing, extracted = ais.history[0]
            sentence_id = ais.sentence.id
            # use first position
            for token in ais.tokens:
                term = token[0]
                post_nouns = sorted([noun for noun in ais.sentence.nouns
                                     if noun.index > term.index],
                                    key=lambda x: abs(x.index-term.index))
                prev_nouns = sorted([noun for noun in ais.sentence.nouns
                                     if noun.index < term.index],
                                    key=lambda x: abs(x.index-term.index))
                prev_adjs = sorted([adj for adj in ais.sentence.adjectives
                                    if adj.index < term.index],
                                   key=lambda x: abs(x.index-term.index))
                phrases = set()
                if prev_nouns:
                    for i in range(1, Q+1):
                        phrase = sorted([term]+prev_nouns[:i], key=lambda x: x.index)
                        pphrase = tuple(key(token) for token in phrase)
                        phrases.add(pphrase)
                        existings[(pphrase, sentence_id)].append((existing, tuple(phrase)))

                if post_nouns:
                    for i in range(1, Q+1):
                        phrase = sorted([term]+post_nouns[:i], key=lambda x: x.index)
                        pphrase = tuple(key(token) for token in phrase)
                        phrases.add(pphrase)
                        existings[(pphrase, sentence_id)].append((existing, tuple(phrase)))
                if prev_adjs:
                    for i in range(1, K+1):
                        phrase = sorted([term]+prev_adjs[:i], key=lambda x: x.index)
                        pphrase = tuple(key(token) for token in phrase)
                        phrases.add(pphrase)
                        existings[(pphrase, sentence_id)].append((existing, tuple(phrase)))
                if phrases:
                    ps = transactions.get(ais.sentence.id, None)
                    if ps:
                        ps |= phrases
                    else:
                        transactions[ais.sentence.id] = phrases

    new_aspects = []
    frequents = fim(transactions.values(), supp=2, zmax=1)
    for frequent, support in frequents:
        frequent = frequent[0]
        for id_sentence, transaction in transactions.items():
            if frequent in transaction:
                existing, extracted = existings[(frequent, id_sentence)][0]
                aspect = matching(frequent, new_aspects)
                if aspect:
                    aspect.append(key, existing, extracted)
                else:
                    aspect = Aspect(key, existing, extracted)
                    new_aspects.append(aspect)
    return new_aspects

# ------- BASE CLASSES --------------------------------------------------------

class Term(object):

    def __init__(self, existing, extracted):
        raise NotImplementedError()

    def __eq__(self, other):
        if type(self) == type(other) and self.id == other.id:
            return True
        return False

    @property
    def tiss(self):
        # dict of term in sentence Objects
        all_items = [(key, value) for review in self.reviews
                     for (key, value) in review.tiss.items()]
        return {key: value for (key, value) in all_items}

    @property
    def reviews(self):
        return self.tirs.values()

    @property
    def sentences(self):
        # dict of term in sentence Objects
        return [sentence for review in self.reviews
                for sentence in review.sentences]

    @property
    def sentence_ids(self):
        return self.tiss.values()

    @property
    def review_ids(self):
        return self.tirs.values()

    @property
    def support(self):
        return len(self.tiss)

    def in_sentence(self, sentence_id):
        return self.tiss[sentence_id]

    def in_review(self, review_id):
        return self.tirs[review_id]

class TermInReview(object):

    def __init__(self):
        raise NotImplementedError()

    def __repr__(self, *args, **kwargs):
        return self.__str__()

    @property
    def sentences(self):
        return self.tiss.values()

    @property
    def id(self):
        return self.term.id, self.review.id

    @property
    def support(self):
        return len(self.tiss)

    def delete(self):
        # print "deleting " + str(self)
        del self.term.tirs[self.review.id]

    def __eq__(self, other):
        if type(self) == type(other) and self.id == other.id:
            return True
        return False

class TermInSentence(object):

    def __init__(self, term_in_review, existing, extracted):
        self.sentence = extracted.sentence
        self.tir = term_in_review
        self.tokens = [extracted]

        # history keeps track of extraction process
        self.history = []
        self.history.append((existing, extracted))

    def __repr__(self, *args, **kwargs):
        return self.__str__()

    def __eq__(self, other):
        if type(self) == type(other) and self.id == other.id:
            return True
        return False

    @property
    def positions(self):
        return [token.index for token in self.tokens]

    @property
    def review(self):
        return self.tir

    @property
    def term(self):
        return self.tir.term

    @property
    def id(self):
        return self.term.id, self.sentence.id

    def append(self, existing, extracted):
        # print "appending to" + str(self)
        if extracted not in self.tokens:
            self.tokens.append(extracted)
        self.history.append((existing, extracted))
        return self

    def delete(self):
        # print "deleting " + str(self)
        deleted = self.review.tiss[self.sentence.id]
        del self.review.tiss[self.sentence.id]
        if self.review.support == 0:
            self.review.delete()
        return deleted

# ------- ASPECT CLASS --------------------------------------------------------

class Aspect(Term):

    negations = NEGATIONS
    negation_limit = NEGATION_LIMIT

    def __init__(self, key, existing, extracted):
        # dict of term in review Objects
        self.id = _uid()
        self.set = set([tuple(key(token) for token in extracted)])
        review_id = extracted[0].sentence.review.id
        self.is_compound = False

        if len(extracted) > 1:
            self.is_compound = True

        self.tirs = OrderedDict()
        self.tirs[review_id] = Aspect.AspectInReview(self, existing, extracted)

    def append(self, key, existing, extracted):
        self.set.add(tuple(key(token) for token in extracted))
        review_id = extracted[0].sentence.review.id

        tir = self.tirs.get(review_id, None)
        if tir:
            tis = tir.append(existing, extracted)
            return tis
        else:
            tir = Aspect.AspectInReview(self, existing, extracted)
            self.tirs[review_id] = tir
            return tir.tiss.values()[0]

    def __str__(self, *args, **kwargs):
        return "Aspect<{0}>".format(self.id)

    def __repr__(self, *args, **kwargs):
        return self.__str__()

    def __iter__(self):
        return iter(self.set)

    # ============================================================================================================

    class AspectInReview(TermInReview):

        def __init__(self, term, existing, extracted):
            self.term = term
            self.is_compound = term.is_compound
            self.review = extracted[0].sentence.review
            sentence_id = extracted[0].sentence.id
            # dict of term in sentence Objects
            self.tiss = OrderedDict()
            self.tiss[sentence_id] = Aspect.AspectInSentence(self, existing, extracted)

        def __str__(self, *args, **kwargs):
            return "Aspect<{0} in Review {1}>".format(*self.id)

        def append(self, existing, extracted):
            """
            Returns the AspectInSentence that was created
            when this function is called
            """
            sentence_id = extracted[0].sentence.id
            tis = self.tiss.get(sentence_id, None)
            if tis:
                tis = tis.append(existing, extracted)
                return tis
            else:
                tis = Aspect.AspectInSentence(self, existing, extracted)
                self.tiss[sentence_id] = tis
                return tis

        @property
        def orientation(self):
            """
            Orientation of an Aspect in review is obtained from the orientation
            of all the related AspectInSentence that were extracted either using
            other OpinionWordInSentence or other AspectInSentence.
            """
            orientation = 0.0
            for sentence in self.sentences:
                # get the "origin" of the AspectInSentence
                existing, extracted = sentence.history[0]
                if existing is not self.term:
                    orientation += sentence.orientation
            return 1.0 if orientation > 0 else -1.0

    # =============================================================================================================

    class AspectInSentence(TermInSentence):

        def __init__(self, term_in_review, existing, extracted):
            """
            Represents an Aspect that appears on a specific sentence.
            It may appear on different locations and each position may be extracted
            several times. set([ tuple(key(seed),) ])
            Extraction history is stored in the history attribute.
            """
            self.is_compound = term_in_review.is_compound
            self.sentence = extracted[0].sentence
            self.tir = term_in_review
            self.tokens = [extracted]

            # histoy keeps track of the
            self.history = []
            self.history.append((existing, extracted))

            self.sentence.aspects += [self]
            # print "creating " + str(self)

        def __str__(self, *args, **kwargs):
            return "Aspect<{0} in Sentence {1}>".format(*self.id)

        @property
        def orientation(self):
            """
            We use self.history and orientation
            rules to determine the orientation
            """
            if len(self.history) > 1:
                # do something maybe?
                pass

            # get the first item in history
            existing, extracted = self.history[0]

            # case 0: AspectInSentence extracted by self matching the Aspect
            if isinstance(existing, Aspect):
                # get orientation of Aspect in the current review
                return existing.in_review(self.review.review.id).orientation

            # case 1: aspect was extracted using known OpinionWordInSentence
            if isinstance(existing, OpinionWord.OpinionWordInSentence):
                # get orientation of OpinionWord (parent of OpinionWordInSentence)
                orientation = existing.term.orientation
                # check if there are negations near the OpinionWordInSentence
                # (we use the first position in which it was found)
                if has_near_negation(existing.tokens[0][0], Aspect.negations, Aspect.negation_limit):
                        return -1.0 * orientation
                return orientation

            # case 2: aspect extracted from known AspectInSentence
            if isinstance(existing, Aspect.AspectInSentence):
                # get the orientation of current AspectInReview
                orientation = existing.review.orientation
                if orientation:
                    n = negations_between(Aspect.negations,
                                          token_1=existing.tokens[0][0],
                                          token_2=extracted[0])
                    orientation *= ((-1.0)**n)
                return orientation

            raise Exception("Class matching failed")

# ------- OPINIONWORD CLASS ---------------------------------------------------

class OpinionWord(Term):

    negations = NEGATIONS
    negation_limit = NEGATION_LIMIT

    def __init__(self, key, existing=None, extracted=None, seed=None):
        # print "creating " + str(self)
        self.id = _uid()
        if seed:
            self.set = set([(key(seed),)])
            self.tirs = OrderedDict()
            self.is_seed = True
            self._orientation = seed.orientation
        else:
            # dict of term in review Objects
            self.set = set([tuple(key(token) for token in extracted)])
            review_id = extracted[0].sentence.review.id
            self.tirs = OrderedDict()
            self.tirs[review_id] = OpinionWord.OpinionWordInReview(self, existing, extracted)
            self.is_seed = False

    def add_seed(self, key, seed):
        self.set.add((key(seed),))

    @property
    def orientation(self):
        if self.is_seed:
            return self._orientation
        else:
            # return overall orientation over the corpus
            orientation = 0.0
            for sentence in self.sentences:
                # get the "origin" of the OpinionWordInSentence
                existing, extracted = sentence.history[0]
                if existing is not self:
                    orientation =+ sentence.orientation
            return 1.0 if orientation > 0 else -1.0

    def __str__(self, *args, **kwargs):
        return "OpinionWord<{0}>".format(self.id)

    def __repr__(self, *args, **kwargs):
        return self.__str__()

    def __iter__(self):
        return iter(self.set)

    def append(self, key, existing, extracted):
        self.set.add(tuple(key(token) for token in extracted))
        review_id = extracted[0].sentence.review.id
        tir = self.tirs.get(review_id, None)
        if tir:
            tis = tir.append(existing, extracted)
            return tis
        else:
            tir = OpinionWord.OpinionWordInReview(self, existing, extracted)
            self.tirs[review_id] = tir
            return tir.tiss.values()[0]

    # ==================================================================================================================

    class OpinionWordInReview(TermInReview):

        def __init__(self, term, existing, extracted):
            self.term = term
            self.review = extracted[0].sentence.review
            id_sentence = extracted[0].sentence.id

            # dict of term in sentence Objects
            self.tiss = OrderedDict()
            self.tiss[id_sentence] = OpinionWord.OpinionWordInSentence(self, existing, extracted)

        def __str__(self, *args, **kwargs):
            return "OpinionWord<{0} in Review {1}>".format(*self.id)

        def append(self, existing, extracted):
            """
            Returns the TermInSentence that was created
            when this function is called
            """
            id_sentence = extracted[0].sentence.id
            tis = self.tiss.get(id_sentence, None)
            if tis:
                tis = tis.append(existing, extracted)
                return tis
            else:
                tis = OpinionWord.OpinionWordInSentence(self, existing, extracted)
                self.tiss[id_sentence] = tis
                return tis

    # =======================================================================================================================

    class OpinionWordInSentence(TermInSentence):

        def __init__(self, term_in_review, existing, extracted):
            self.sentence = extracted[0].sentence
            self.tir = term_in_review
            self.tokens = [extracted]

            # histoy keeps track of the
            self.history = []
            self.history.append((existing, extracted))

            self.sentence.opinion_words.append(self)

        def __str__(self, *args, **kwargs):
            return "OpinionWord<{0} in Sentence {1}>".format(*self.id)

        @property
        def orientation(self):
            """
            We use self.history and orientation
            rules to determine the orientation
            """
            if self.term.is_seed:
                return self.term.orientation

            if len(self.history) > 1:
                # do something maybe?
                pass

            # get the first item in history
            existing, extracted = self.history[0]

            # case 0: OpinionWordInSentence extracted matching from OpinionWord
            if isinstance(existing, OpinionWord):
                # we just return the orientation of the OpinionWord
                return existing.orientation

            # case 1: OpinionWord was extracted using other OpinionWordInSentence
            if isinstance(existing, OpinionWord.OpinionWordInSentence):
                orientation = existing.term.orientation
                if orientation:
                    # count the negations between the OpinionWordInSentences
                    # for the existing one, we use its first-found position
                    n = negations_between(OpinionWord.negations,
                                          token_1=existing.tokens[0][0],
                                          token_2=extracted[0])
                    orientation *= ((-1.0)**n)
                return orientation

            # case 2: OpinionWord was extracted using a AspectInSentence
            if isinstance(existing, Aspect.AspectInSentence):
                # check if AspectInSentence has orientation
                orientation = existing.orientation
                if orientation:
                    # if orientation is not 0
                    # check if there are negations near this OpinionWordInSentence
                    # we get the position where it was first-found
                    if has_near_negation(extracted[0], OpinionWord.negations, OpinionWord.negation_limit):
                        orientation = -1.0 * orientation
                    return orientation
                else:
                    # if AspectInSentence has no orientation
                    # get the orientation of the review, based on OpinionWords
                    review = self.review.review
                    return review_orientation(review)

            raise Exception("Class matching failed")


# -*- coding: utf-8 -*-

"""
    Actions for CDNRequest rules
"""

import json

from collections import OrderedDict
from datetime import datetime, timedelta
from time import mktime

from django.forms.models import model_to_dict

from ...engine.actions import rule_action, BaseActions
from ...engine.fields import FIELD_TEXT
from ....models import History, Proof, Ticket, User, ReportItem
from ....utils.cache import redis_lock, RedisHandler
from ....tasks import enqueue_in, helpers

class DefaultCDNRequestActions(BaseActions):
    """
        This class implements actions for CDN requests
    """

    providers = {"cloudflare": {"email": "abuseteam@cloudflare.com"}}

    cache_expirations_days = 15
    redis_queue = "cdnrequest:{}:request"

    def __init__(self, report, domain_to_request):
        """
        """
        self.report = report
        self.domain_to_request = domain_to_request

    @rule_action(params=[{"fieldType": FIELD_TEXT, "name": "provider"}])
    def do_cdn_request(self, provider):
        """
            Make request to supported CDN providers
        """
        ticket, expiration = self._retrieve_from_cache(provider)

        # if already resolved, just attach report
        if expiration and datetime.now() < expiration:
            self.report.defendant = ticket.defendant
            self.report.service = ticket.service
            self.report.save()
            if all((self.report.defendant, self.report.service)):
                # rexecute report workflow
                self._rexecute_report_workflow(ticket)
        else:  # else, send request to provider
            ticket = self._send_request(provider)

        self.report.ticket = ticket
        self.report.status = "Attached"
        self.report.save(update_fields=["ticket", "status"])

        History.log_ticket_action(
            ticket=ticket, action="attach_report", new_ticket=False, report=self.report
        )

    @redis_lock("cdnrequest:lock")
    def _retrieve_from_cache(self, provider):
        """
            Check if there is already a pending/resolved similar request
        """
        expiration = ticket = None
        entries = RedisHandler.ldump(self.redis_queue.format(provider))

        for entry in entries:
            entry = json.loads(entry, object_pairs_hook=OrderedDict)
            if entry["domain"] == self.domain_to_request:
                ticket = Ticket.get(id=entry["request_ticket_id"])
                expiration = datetime.fromtimestamp(entry["expiration"])
                break

        return ticket, expiration

    @redis_lock("cdnrequest:lock")
    def _send_request(self, provider):
        """
            Send email request to CDN Provider
        """
        ticket = helpers.create_ticket(self.report, denied_by=None, attach_new=False)

        ticket.treatedBy = User.objects.get(username="abuse.robot")
        ticket.save()

        self._send_email_request(provider, ticket)
        self._update_cache(ticket, provider)

        return ticket

    def _send_email_request(self, provider, ticket):

        try:
            email = self.providers[provider]["email"]
        except KeyError:
            raise Exception("Unsupported CDN provider")

        proof_content = (
            self.report.reportItemRelatedReport.filter(itemType="FQDN").last().rawItem
        )

        Proof.create(ticket=ticket, content=proof_content)

        helpers.send_email(ticket, [email], "{}_ip_request".format(provider), lang="EN")

        ticket.save()

    def _update_cache(self, ticket, provider):

        entries = RedisHandler.ldump(self.redis_queue.format(provider))

        # Clear old entries
        for entry in entries:
            entry_json = json.loads(entry)
            if entry_json["domain"] == self.domain_to_request:
                RedisHandler.lrem(self.redis_queue.format(provider), entry)

        # Push task
        RedisHandler.rpush(
            self.redis_queue.format(provider),
            json.dumps(
                {
                    "domain": self.domain_to_request,
                    "request_ticket_id": ticket.id,
                    "expiration": self._get_expiration(),
                }
            ),
        )

    def _get_expiration(self):

        exp = datetime.now() + timedelta(days=self.cache_expirations_days)
        return int(mktime(exp.timetuple()))

    def _rexecute_report_workflow(self, ticket):

        # attach resolved URLs
        self.report.attach_url_matching_domain(self.domain_to_request)

        # attach resolved IP address
        item = (
            ticket.reportTicket.first()
            .reportItemRelatedReport.filter(itemType="IP")
            .last()
        )
        item = model_to_dict(item)
        item.pop("id")
        item["report"] = self.report
        ReportItem.create(**item)

        # schedule job
        enqueue_in(
            timedelta(seconds=10),
            "report.validate_with_defendant",
            report_id=self.report.id,
            timeout=3600,
        )


# K-means implementation for assignment 2
# William Pourmajidi

# Importing Required library for plotting numbers
import matplotlib.pyplot as plt
import seaborn as sns;
sns.set()  # for plot styling
# Importing Required library for working with arrays (key,value)
from sklearn.datasets.samples_generator import make_blobs

# Importing Required library for K-means
from sklearn.cluster import KMeans

X = np.array([[8, 4], [5, 4], [2, 4], [2, 6], [2, 8], [8, 6]])

kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
y_kmeans = kmeans.predict(X)
print(X)
print(y_kmeans)
plt.scatter(X[:, 0], X[:, 1]);
plt.show()
plt.scatter(X[:, 0], X[:, 1], c=y_kmeans, s=50, cmap='viridis')
plt.show()

centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.5);
plt.show()


import numpy as np

def discount_rewards(rewards, gamma):
    """
    Compute discounted rewards.
    :param rewards: list of rewards
    :param gamma: discount factor
    :return: list of discounted rewards
    """
    num_rewards = len(rewards)
    # set last discounted reward as last reward
    # this reduces branching
    discounted_rewards = np.zeros_like(rewards)
    discounted_rewards[num_rewards-1] = rewards[num_rewards-1]
    # reverse iterate over rewards
    for t in reversed(range(0, num_rewards-1)):
        # compute discounted reward as reward at current time step plus
        # discounted sum of discounted_rewards from next time step
        discounted_rewards[t] = rewards[t] + gamma * discounted_rewards[t+1]
    return discounted_rewards

def compare_lists_of_np_arrays(l1, l2):
    """Compare two lists of numpy arrays."""
    equal = True
    for i in range(len(l1)):
        equal = np.array_equal(l1[i], l2[i]) & equal

    return equal

def unzip_experience_buffer(buffer):
    """Unzip experience buffer into separate lists of observations, actions,
    rewards, and next observations."""
    obs, actions, rewards, next_obs, dones = [], [], [], [], []
    for s, a, r, s_, done in buffer:
        obs.append(s)
        actions.append(a)
        rewards.append(r)
        next_obs.append(s_)
        dones.append(done)

    return obs, actions, rewards, next_obs, dones


import json
import os

from requests_oauthlib import OAuth2Session
from requests_oauthlib.compliance_fixes.facebook import facebook_compliance_fix

from planevent import settings
from planevent.accounts import (
    models,
    tasks,
    auth,
)

os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = 'True'

def _get_oauth2_callback_url(request, provider):
    return request.route_url('oauth2_callback', provider=provider)

def login_oauth(request, provider):
    provider_settings = settings.OAUTH[provider]

    oauth = OAuth2Session(
        provider_settings.client_id,
        redirect_uri=_get_oauth2_callback_url(request, provider),
        scope=provider_settings.scope,
    )

    authorization_url, state = oauth.authorization_url(
        provider_settings.authorize_url,
    )

    return authorization_url

def process_oauth_callback(request, provider):
    provider_settings = settings.OAUTH[provider]

    oauth = OAuth2Session(
        provider_settings.client_id,
        redirect_uri=_get_oauth2_callback_url(request, provider),
        state=request.params['state'],
        scope=provider_settings.scope,
    )

    if provider == 'facebook':
        oauth = facebook_compliance_fix(oauth)

    token = oauth.fetch_token(
        provider_settings.access_token_url,
        authorization_response=request.url,
        client_secret=provider_settings.secret_key,
    )

    oauth.token = token

    response = oauth.get(provider_settings.user_info_url)
    provider_user = json.loads(response.content.decode('utf8'))

    account, is_new = process_oauth_user(provider, provider_user)

    auth._login(request, account)

    if is_new:
        return request.route_url('home') + '#/userProfile/firstLogging'
    else:
        return request.route_url('home')

def process_oauth_user(provider, provider_user):

    def fill_user_fields(account, provider_user, provider_settings):
        def fill_user_field(field):
            provider_field = provider_settings.mapper.get(field) or field
            if not getattr(account, field):
                setattr(account, field, provider_user.get(provider_field))

        fill_user_field('name')
        fill_user_field('first_name')
        fill_user_field('last_name')
        fill_user_field('gender')
        fill_user_field('link')

    account = models.Account.get_by_provider(provider, provider_user['id'])

    is_new = False
    if not account:
        email = provider_user.get('email')
        if email:
            account = models.Account.get_by_email(email)

    if not account:
        account = models.Account.create(email=provider_user.get('email'))
        is_new = True

    account.credentials.origin_id = provider_user.get('id'),
    account.credentials.provider = provider,

    provider_settings = settings.OAUTH[provider]

    fill_user_fields(account, provider_user, provider_settings)

    if is_new:
        tasks.send_welcome_email(account)

    return account, is_new


# !/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import re

__author__ = 'JrD'
__time__ = '2019.03.07'

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

def xmlparser(path=None):
    cwd = os.getcwd()
    absolute_path = os.path.join(cwd, path)
    if os.path.exists(absolute_path):
        filepath = (cwd+'/'+path.split('/')[-1].split('.')[0]+'.csv')
        if os.path.splitext(absolute_path)[-1].lower() == '.xml':
            with open('%s'%(filepath),'w') as write:
                try:
                    tree = ET.ElementTree(file=absolute_path)
                except ET.ParseError:
                    with open(absolute_path, 'a+') as file:
                        file.write('\n</nmaprun>')
                    tree = ET.ElementTree(file=absolute_path)
                root = tree.getroot()
                print('Nmap command: ' + root.attrib['args'])
                print('Start time: ' + root.attrib['startstr'])
                ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION = 'ADDR','PORT','STATE','SERVICE','PRODUCT','VERSION'
                column_name = ('%-20s %-7s %-10s %-15s %-20s %-30s' %(ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION))
                write.write('''%s,%s,%s,%s,%s,%s'''%(ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION))
                print(column_name)
                for host in root.iter('host'):
                    addr = host[1].attrib['addr']
                    for ports in host[3][1:]:
                        port, state, service, product, version = '','','','',''
                        try:
                            port = ports.attrib['portid']
                            state = ports[0].attrib['state']
                            service = ports[1].attrib['name']
                            product = ports[1].attrib['product']
                            version = ports[1].attrib['version']
                        except KeyError:
                            pass
                        print('%-20s %-7s %-10s %-15s %-20s %-30s'% (addr,port,state,service,product,version))
                        write.write('''\n%s,%s,%s,%s,%s,%s''' % (addr,port,state,service,product,version))

            print('Finished: %s' % filepath)
        else:
            print('[ERROR] File is not XML: %s' % path)
    else:
        print('[ERROR] File not found: %s' % path)

if __name__ == '__main__':
    print('''
 | \ | |  _ __ ___     __ _   _ __   \ \/ /  _ __     __ _   ___    ___   _ __
 |  \| | | r'_ ` _ \   / _` | | '_ \  \  /  | r'_ \  / _` | / __|  / _ \ | '__|
 | |\  | | | | | | | | (_| | | |_) |  /  \  | |_) | | (_| | \__ \ |  __/ | |
 |_| \_| |_| |_| |_|  \__,_| | .__/  /_/\_\ | .__/   \__,_| |___/  \___| |_|
                             |_|            |_|                                ''')
    if len(sys.argv) == 2:
        xmlpath = sys.argv[1]
        xmlparser(path=xmlpath)
    else:
        print('''[usage]:python NmapXpaser.py xxxxxx.xml''')
        sys.exit()


import os
import torch
import torch_geometric as pyg
import numpy as np
import networkx as nx
from utils import load_pickle

class GraphDataset(torch.utils.data.Dataset):
    """
    Graph Dataset. Collects NetworkX graph from a pre-defined folder and
    transforms them to Pytorch Geometric (pyg.data.Data()) instances.
    """
    def __init__(self, path, graph_type = 'zoning'):
        self.graph_path = os.path.join(path, 'graph_in' if 'zoning' in graph_type else 'graph_out')
        self.struct_path = os.path.join(path, 'struct_in')
        self.full_path = os.path.join(path, 'full_out')

        # include graph transformations if you like
        # self.graph_transform = graph_transform

    def __getitem__(self, index):

        # get access graph (name is index)
        graph_nx = load_pickle(os.path.join(self.graph_path, f'{index}.pickle'))

        # add images as graph attributes
        struct = np.load(os.path.join(self.struct_path, f'{index}.npy'))
        full = np.load(os.path.join(self.full_path, f'{index}.npy'))
        graph_nx.graph['struct'] = struct[np.newaxis, ...]
        graph_nx.graph['full'] = full[np.newaxis, ...]

        # transform networkx graph to pytorch geometric graph
        graph_pyg = pyg.utils.from_networkx(graph_nx)

        # transform graph if you like
        # graph_pyg = self.graph_transform(graph_pyg)

        return graph_pyg

    def __len__(self):
        return len(os.listdir(self.graph_path))


def avg_length(line):
    count=0
    total=0
    for word in line.split():
        length=len(word)
        total+=length
        count+=1
        average = total/count
    return average
n=input("Enter a sentence \n")
value=avg_length(n)
print("the average length of a word in a sentence is", value)


"""model definition"""
"""trained not GAN"""

from Net.ProCCaps import CapsNet_MR
# ------------
from torch import Tensor, device
from torch.utils.data import DataLoader
from torchvision import transforms
from Zhang_github.data_imagenet import  ValImageFolder
import utils_code.util_zhang as util_zhang
from utils_code.utils import *
import os
from PIL import Image, ImageFile
import numpy as np
import shutil
from tqdm import tqdm
ImageFile.LOAD_TRUNCATED_IMAGES = True
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="6"

additional_appendix = "ProCCaps"
routes = (32, 9, 9)
size = 224
root = 'test_dataset'
if os.path.exists(root):
  os.mkdir(root)
class RGB2LAB(object):
    def __init__(self):
        super(RGB2LAB,self).__init__()
    def __call__(self,img):
        (tens_rs_ab, tens_rs_l, tens_rxs_ab)= util_zhang.preprocess_img(img, HW=(size,size), resample=3)
        return tens_rs_ab, tens_rs_l, tens_rxs_ab
original_transform = transforms.Compose([RGB2LAB()])

"""Dataset to use"""
# val_dir_name, results_folder = '/media/TBData2/Datasets/COCO_stuff/val', "COCO_stuff"
val_dir_name, results_folder = '/media/TBData2/Datasets/ImageNet10k/Test', "Image10k_tiny"
# val_dir_name, results_folder = '/media/TBData2/Datasets/Places205/Test', "Places205"

"""batch_size"""
batch_size = 32
"""Init and load dataset"""
ImageDataset = ValImageFolder(val_dir_name, transform=original_transform)
dataloaders = DataLoader(ImageDataset, batch_size=batch_size, shuffle=False)

"""if you want to keep original dimension (enlarge the output of the model), you need orig_dim = True"""
orig_dim = False
extention, model_folder = 'png', "PL"
n_cuda, epoch = 0, str(65)
CUDA = "cuda:"+str(n_cuda)
device = device(CUDA)

"""Init model: the SSL models are in dataParallel box, the PL models are without dataParallel"""
model = CapsNet_MR(128, num_routes=routes)
depth = 'Q'
print(f"*The depth is {depth} ! Be aware of that*")

"""Checkpoint locations"""
model_path, appendix = os.path.join('models/WACV_model/checkpoint_ProCCaps_ImageNet_65.pth.tar'), "_wacv"  #<-PL_IN*

"""Resume pretrained model"""
model, epoch_start = resume_model(model_path, model, map_location=CUDA)
model = model.to(device)

"""Create folder for the coloured images"""
if not os.path.exists(os.path.join(root,results_folder)):
    os.mkdir(os.path.join(root,results_folder))
mother_folder = os.path.join(root,results_folder,epoch+"_"+model_folder+"_"+appendix)
mother_folder_orig_clone = mother_folder + '_orig_clone'
if os.path.exists(mother_folder):
    shutil.rmtree(mother_folder, ignore_errors=True)
if os.path.exists(mother_folder_orig_clone):
    shutil.rmtree(mother_folder_orig_clone, ignore_errors=True)
os.mkdir(mother_folder)
os.mkdir(mother_folder_orig_clone)

psnr_sum, psnr_sum_self = [], []
model.train()
for batch_id, (image_batch,index, target, name_path) in enumerate(tqdm(dataloaders)):
    ##extract the greyscale image
        img_ab_rs = image_batch[0].to(device)
        img_l_rs = image_batch[1].to(device)
        img_l_rs_v = img_l_rs.to(device)
        ##infer the colourisation channels
        if model_folder == 'SSL': img_ab_pred,_ = model(img_l_rs_v)
        elif model_folder == 'PL': img_ab_pred = model(img_l_rs_v, depth=depth)
        ##print check
        if batch_id == 0: print(img_ab_pred[0,:,:,:])
        ##for loop for image recomposition
        for j in range(len(img_l_rs_v)):
            if orig_dim:
                tens_orig = util_zhang.original_l(ImageDataset, index[j])
                L = tens_orig.unsqueeze(0).to(device)
                HW = tens_orig.shape[2:]
            else:
                L =  img_l_rs_v
                HW = img_ab_pred.shape[2:]

            img_rgb = util_zhang.postprocess_tens(L, img_ab_pred, j, mode='bilinear',HW_request=HW) #HW_request=img_ab_pred.shape[2:])#
            img_rgb = (img_rgb*255).astype(np.uint8)
            img_rgb_orig = util_zhang.postprocess_tens(L, img_ab_rs, j, mode='bilinear',HW_request=HW) #HW_request=img_ab_pred.shape[2:])#HW)
            img_rgb_orig = (img_rgb_orig * 255).astype(np.uint8)
            ##PSNR
            psnr_sum.append(psnr(img_rgb_orig, img_rgb))
            #Save output
            pil_image = Image.fromarray(img_rgb, 'RGB')
            if isinstance(target[j],Tensor):
                target_j = str(target[j].item())
            else:
                target_j = target[j]

            dstfolder = os.path.join(mother_folder, target_j)
            if not os.path.exists(dstfolder):
                os.mkdir(dstfolder)
                os.mkdir(os.path.join(mother_folder_orig_clone, target_j))
            pil_image.save(os.path.join(dstfolder, name_path[j].replace(extention,'png')))
            Image.fromarray(img_rgb_orig, 'RGB').save(os.path.join(mother_folder_orig_clone,
                                                                   target_j,
                                                                  name_path[j].replace(extention,'png')))
            del pil_image,img_rgb_orig,img_rgb,dstfolder#, tens_orig
        del image_batch,target,name_path
print(np.mean(psnr_sum))


# Data Representation - Assignment 3
# Author: Jamie Tohall

# Write a program that retrieves the dataset for the "exchequer account (historical series)"
# from the CSO, and stores it into a file called "cso.json"

import requests # Import relevant modules
import json

url = "https://ws.cso.ie/public/api.restful/PxStat.Data.Cube_API.ReadDataset/FIQ02/JSON-stat/2.0/en"
# url for the exchequer account (historical series) dataset, JSON format

def getalldata():
    response = requests.get(url)
    return response.json()

if __name__ == "__main__":

    with open("cso.json", "w") as fp:
        print(json.dumps(getalldata()),file=fp)


import os

import pandas

class CSVFileHandling:

    def __init__(self, path):
        self.path = path

    def create_file(self):
        with open(self.path, 'w') as file:
            file.write('')
            pass
            # json.dumps(file)
            # file.write(data)

    def delete_file(self):
        if os.path.exists(self.path):
            os.remove(self.path)

    def read_file(self):
        pandas.read_csv(self.path)

    def write_file(self,content1):
        df = pandas.DataFrame(content1)
        df.to_csv(self.path)
        # with open(path,'w') as file:
        #     file.write()

    def append_file(self,content2):
        old_data = pandas.read_csv(self.path)
        new_data = pandas.DataFrame(content2)
        combined_data = pandas.concat((old_data, new_data), ignore_index=True)
        print(combined_data)

path = "data.csv"
file_handle = CSVFileHandling(path)
# file_handle.create_file()
# file_handle.delete_file()
file_handle.read_file()
content1 = {
    'Name': ['John', 'Alice', 'Bob'],
    'Age': [30, 25, 28],
    'City': ['New York', 'Los Angeles', 'Chicago']
}
content2 = {
    'Name': ['Eve', 'Tom'],
    'Age': [35, 27],
    'City': ['San Francisco', 'Seattle']
}
# file_handle.write_file(content1)
file_handle.append_file(content2)


print('This program counts an averege value for given numbers')
print('and gives this one from given numbers,')
print('that is the nearest average value.')
numbers = []
number = int(input('Write a number: '))

while number != 's':
   numbers.append(int(number))
   number = input('Write a number or "s" if you do not want to write more numbers: ')

sum = 0

for j in numbers:
	sum += int(j)

avg = float(sum) / len(numbers)

index = 0

for j in numbers:
	if abs(int(j) - avg) < abs(int(numbers[index]) - avg):
		index = numbers.index(j)

print(numbers[index])


import Variable_sus as vs
import util
import os
import numpy as np
import openpyxl
import sys

alpha = 0.8 # 怀疑度系数
beta = 0.8  # 输出系数
data_path = r'D:\fault_loc\ITSP-data'
res_file = r''
res_file_temp = os.path.join('result', 'vsbfl_BUCT_sus.xlsx')  # 请先自己创建这个文件

def find_pair_by_tag(dir_path):
    '''
    根据数据集寻找每份错误代码对应的正确代码
    '''
    pair_info = {}
    tag_files = os.listdir(dir_path)
    for i in tag_files:
        tag_file = os.path.join(dir_path, i)
        lines = util.read_file(tag_file)
        wa_file = lines[0].replace('\n', '')
        ac_file = lines[1].replace('\n', '')
        pair_info[wa_file] = ac_file
    # for file in pair_info:
    #     print(file, pair_info[file])
    return pair_info

def find_pair_by_res(file_path):
    '''
    根据匹配结果寻找每份错误代码对应的正确代码
    '''
    pair_info = {}
    if file_path.split('.')[-1] != 'xlsx' and file_path.split('.')[-1] != 'xls':
        print('not an execl file')
        return
    wb = openpyxl.load_workbook(file_path)
    for i, sheet in enumerate(wb):
        if i == 0:
            continue
        # print(sheet.title)
        buggy_name = ''
        for j, row in enumerate(sheet.rows):
            if j == 0:
                continue
            if row[0].value != None:
                buggy_name = row[0].value
            if row[1].value.find('[') >= 0:
                ac_name_list = eval(row[1].value)
                pair_info[buggy_name] = ac_name_list[0]
    #     break
    # print(pair_info)
    return pair_info

def find_possiable_correct_line(wa_res, ac_res):
    '''
    寻找配对的变量
    '''
    # print(wa_res)
    weight, weight_LCS = vs.add_weight(wa_res, ac_res)
    vars_pair = util.cal_KM(weight) #二分图最大完备匹配
    # print(vars_pair)
    # print(weight_LCS)
    possiable_correct_line = {}
    for var in vars_pair:
        possiable_correct_line[var] = weight_LCS[var][vars_pair[var]['var']]
    # possiable_correct_line分为n个列表，n是测试用例的数量，每个列表中的数字为
    # 错误代码变量值序列中能和正确代码匹配的值在整个值序列中的位置
    # print(possiable_correct_line)
    return possiable_correct_line, vars_pair

def cal_sus(wa_file, ac_file, test_dir_path, language):
    '''
    计算怀疑度
    '''
    wa_res, ac_res = vs.get_sequences(wa_file, ac_file, test_dir_path, language)
    # print(wa_res)
    # print(ac_res)
    possiable_correct_line, vars_pair = find_possiable_correct_line(wa_res, ac_res)
    # print(possiable_correct_line)
    lines = util.read_file(wa_file)
    file_length = len(lines)
    line_sus = list(np.ones(file_length + 1))
    for var in possiable_correct_line:
        line_nums = possiable_correct_line[var]
        # print(var, line_nums)
        for i, test_case in enumerate(wa_res):
            # print(i, test_case['info'])
            if var not in test_case['info']:
                continue
            info = test_case['info'][var]
            line_num = line_nums[i]
            possiable_fault_line = range(0, len(info))
            possiable_fault_line = list(filter(lambda x: x not in line_num, possiable_fault_line))
            possiable_fault_line = list(map(lambda x: info[x]['line'], possiable_fault_line))
            line_num = list(map(lambda x: info[x]['line'], line_num))
            # print(var, i, line_num, len(ac_res[i]['info'][var]) - len(line_nums[i]))
            if test_case['res'] == False:
                for j in possiable_fault_line:
                    line_sus[int(j)] *= alpha
                # 这个补丁是为了解决计算不足的问题
                # 这里列表直接用-1还是有危险的，因为可能列表长度为0
                if(len(possiable_fault_line) == 0 and info[-1]['value'] != ac_res[i]['info'][vars_pair[var]['var']][-1]['value']):
                    print(var, i, info)
                    line_sus[int(info[-1]['line'])] *= alpha
            else:
                for j in line_num:
                    line_sus[int(j)] *= (2-alpha)
    # print(line_sus)
    if line_sus.count(min(line_sus)) >= file_length * beta:
        for i, line in enumerate(lines):
            flag = False
            if language == 'cpp' or language == 'c':
                flag = (util.find_pos('cin', line) or util.find_pos('printf', line))
            elif language == 'py':
                flag = util.find_pos('print', line)
            # print(line)
            # print(util.find_pos('printf', line))
            if flag == True:
                # print(line, i)
                line_sus[i + 1] *= alpha
    print(line_sus)
    return line_sus

def run_file(file_path, ac_file, test_dir_path, language):
    '''
    计算程序的最后怀疑度列表
    '''
    print(file_path, ac_file)
    line_sus = cal_sus(file_path, ac_file, test_dir_path, language)
    line_rank = []
    for i,sus in enumerate(line_sus):
        if i == 0:
            continue
        line_rank.append({
            'sus': sus,
            'no': i
        })
    line_rank.sort(key=lambda s: s['sus'])
    # print(line_rank)
    final_rank = []
    tmp_list = []
    for i in range(len(line_rank)):
        if i != 0 and abs(line_rank[i]['sus'] - line_rank[i-1]['sus']) > 0.0001:
            final_rank.append(tmp_list)
            tmp_list = []
        tmp_list.append(line_rank[i]['no'])
    if len(tmp_list) != 0:
        final_rank.append(tmp_list)
    print(final_rank)
    return final_rank, line_sus

def run_dir(file_dir_path, pair_info, test_dir_path):
    '''
    计算某个文件夹内所有文件的最后怀疑度列表
    '''
    file_list = os.listdir(file_dir_path)
    wb = openpyxl.load_workbook(res_file)
    if sys.platform == "linux":
        problem_id = file_dir_path.split('/')[-2]
    else:
        problem_id = file_dir_path.split('\\')[-2]
    wb.create_sheet(problem_id)
    ws = wb[problem_id]
    ws.append({'a':'name', 'b':'suspicion_rank', 'c':'suspicion'})
    for file in file_list:
        file_type = file.split('.')[-1]
        if file_type == 'c' or file_type == 'cpp' or file_type == 'py':
            wa_file_path = os.path.join(file_dir_path, file)
            ac_file_path = os.path.join(data_path, str(problem_id), 'AC_'+file_type, pair_info[file])
            # print(wa_file_path, ac_file_path)
            try:
                final_rank, line_sus = run_file(wa_file_path, ac_file_path, test_dir_path, file_type)
                ws.append({'a':file, 'b':str(final_rank), 'c': str(line_sus)})
                # util.add_file(res_file, file + '    ' + str(final_rank) + '    ' + str(VSBFL_rank) + '\n')
            except Exception:
                print(Exception)
                ws.append({'a':file, 'b':'[]', 'c':'[]'})
                continue
                # util.add_file(res_file, file + '    ' + 'contains error\n')
        # break
    wb.save(res_file)
    return

if __name__ == "__main__":
    # wa_file = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\WA_c\wa.c'
    # ac_file = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\AC_c\ac.c'
    # test_dir_path = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\TEST_DATA_TCG1'
    # wa_file = r'D:\fault_loc\ITSP-data\2871\WA_c\278496_buggy.c'
    # ac_file = r'D:\fault_loc\ITSP-data\2871\AC_c\278496_correct.c'
    # test_dir_path = r'D:\fault_loc\ITSP-data\2871\TEST_DATA_TCG1'
    # rank = run_file(wa_file, ac_file, test_dir_path, 'cpp')

    res_file = res_file_temp
    # problem_list = [2810, 2811, 2812, 2813, 2824, 2825, 2827, 2828, 2830, 2831, 2832, 2833, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871]
    problem_list = ['2174', '3310', '3904', '3905', '3919', '3922', '3924']

    for problem in problem_list:
         # problem = 2810
        print(problem)
        pair_info = find_pair_by_tag(os.path.join(data_path, str(problem), 'Tag_c'))
        # pair_info = find_pair_by_res(r'D:\fault_loc\VSFL-TCG\result\cluster_op2.xlsx')
        dir_path = os.path.join(data_path, str(problem), 'WA_c')
        test_dir_path = os.path.join(data_path, str(problem), 'TEST_DATA_TCG1')
        run_dir(dir_path, pair_info, test_dir_path)
        # cal_time(dir_path)
    #     break

'''
[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0736, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
[[1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [7]]
'''


import random
"""#usr/bin/python3
longeurmois=[[31],[28,29],[31],[30],[31],[30],[31],[31],[30],[31],[30],[31]]
mois=int(input("numéro du mois"))
if len(longeurmois[mois-1])==1:
    print("il y a ",longeurmois[mois-1][0]," jours dans le mois numéro ",mois,".",sep="")
else:
    print("il y a entre ",longeurmois[mois-1][0]," et ",longeurmois[mois-1][1]," jours dans le mois ",mois,".",sep="")
"""
def troisnplusun(n):
    if n%2==0:
        return n//2
    return 3*n+1

po=1
n=1
while True:
    i=n
    k=i
    while i!=1:
        i=troisnplusun(i)
        if i>10**po:
            print(10**po)
            po+=1
        if i<k:
            break
    n+=1


from random import randint, choice
traps = ('"','\\','{','}','`','№','\'')
symbolAlpha = [chr(x) for x in range(65,91)]
symbolCrypt = ('!','@','#','$','%','^','&','*',')','(','~',
'_','+','-','=','<','>','?','/','[',']',',','|',':',';','.')
keys = dict(zip(symbolAlpha, symbolCrypt))

cryptMode = input("[E]ncrypt|[D]ecrypt: ").upper()
if cryptMode not in ['E','D']:
	print("Error: mode is not Found!"); raise SystemExit

startMessage = list(input("Write the message: ").upper())

def encryptDecrypt(mode, message, final = ""):
	if mode == 'E':
		length = len(message) // 4
		for _ in range(length):
			message.insert(randint(0,len(message)),choice(traps))
		for symbol in message:
			if symbol in keys:
				final += keys[symbol]
			else: final += symbol
	else:
		for symbol in message:
			for key in keys:
				if symbol == keys[key]:
					final += key
	return final
print("Final message:",encryptDecrypt(cryptMode, startMessage))


import argparse
from pathlib import Path
import shutil

def parse_path(path_string: str) -> Path:
    path_string = Path(path_string).resolve()
    return path_string

if __name__ == '__main__':
    save_help = 'File path to store the augmented training data and the '\
                'original validation and test data'
    parser = argparse.ArgumentParser()
    parser.add_argument("augmented_data_fp", type=parse_path,
                        help='File path to the augmented training data')
    parser.add_argument("test_val_folder_fp", type=parse_path,
                        help='File path to the folder containing the validation and test data')
    parser.add_argument("save_folder", type=parse_path, help=save_help)
    args = parser.parse_args()

    save_folder = args.save_folder
    save_folder.mkdir(parents=True, exist_ok=True)
    new_train_fp = Path(save_folder, 'train.json')
    shutil.copy(args.augmented_data_fp, new_train_fp)
    val_fp = Path(args.test_val_folder_fp, 'val.json')
    new_val_fp = Path(save_folder, 'val.json')
    shutil.copy(val_fp, new_val_fp)
    test_fp = Path(args.test_val_folder_fp, 'test.json')
    new_test_fp = Path(save_folder, 'test.json')
    shutil.copy(test_fp, new_test_fp)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb  9 23:44:57 2019

@author: Qi Bi
"""

# In[1]:

# -*- coding: utf-8 -*-
from tfdata import *
import numpy as np
import tensorflow as tf

# In[2]:

def weight_variable(shape, name):
    with tf.variable_scope(name) as scope:
        weights = tf.get_variable(name='weights',
                                  shape=shape,
                                  trainable=True,
                                  initializer=tf.truncated_normal_initializer(stddev=0.01))
        #REGULARIZATION_RATE=0.000001
        #regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
        #tf.add_to_collection('losses', regularizer(weights))
        return weights

# In[3]:

def bias_variable(shape, name):
    with tf.variable_scope(name) as scope:
        biases = tf.get_variable(name='biases',
                                 shape=shape,
                                 trainable=True,
                                 initializer=tf.constant_initializer(0.01))

        return biases

# In[4]:

def conv2d(input, in_feature_dim, out_feature_dim, kernel_size, stride, with_bias=True, name=None):
    W = weight_variable([kernel_size, kernel_size, in_feature_dim, out_feature_dim], name=name)
    conv = tf.nn.conv2d(input, W, [1, stride, stride, 1], padding='SAME')
    if with_bias:
        return conv + bias_variable([out_feature_dim], name=name)
    return conv

# In[7]:

def avg_pool(input, s, stride):
    return tf.nn.avg_pool(input, [1, s, s, 1], [1, stride, stride, 1], 'SAME')

# In[8]:

def max_pool(input, s, stride):
    return tf.nn.max_pool(input, [1, s, s, 1], [1, stride, stride, 1], 'SAME')

# In[9]:

def loss(logits, targets):
    # Get rid of extra dimensions and cast targets into integers
    targets = tf.squeeze(tf.cast(targets, tf.int32))
    # Calculate cross entropy from logits and targets
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets)
    # Take the average loss across batch size
    #cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') + tf.add_n(tf.get_collection('losses'))
    cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
    return cross_entropy_mean

# In[10]:

# Train step
def train(loss_value, model_learning_rate):
    # Create optimizer
    # my_optimizer = tf.train.MomentumOptimizer(model_learning_rate, momentum=0.9)

    my_optimizer = tf.train.AdamOptimizer(model_learning_rate)
    # Initialize train step
    train_step = my_optimizer.minimize(loss_value)
    return train_step

# In[11]:

# Accuracy function
def accuracy_of_batch(logits, targets):
    # Make sure targets are integers and drop extra dimensions
    targets = tf.squeeze(tf.cast(targets, tf.int32))
    # Get predicted values by finding which logit is the greatest
    batch_predictions = tf.cast(tf.argmax(logits, 1), tf.int32)
    # Check if they are equal across the batch
    predicted_correctly = tf.equal(batch_predictions, targets)
    # Average the 1's and 0's (True's and False's) across the batch size
    accuracy = tf.reduce_mean(tf.cast(predicted_correctly, tf.float32))
    return accuracy

# In[12]:

def load_with_skip(data_path, session, skip_layer):
    data_dict = np.load(data_path, encoding="bytes").item()
    for key in data_dict:
        if key not in skip_layer:
            with tf.variable_scope(key, reuse=True):
                for subkey, data in zip(('weights', 'biases'), data_dict[key]):
                    get_var = tf.get_variable(subkey).assign(data)
                    session.run(get_var)

# In[13]:

def fc(x, num_in, num_out, name):
    with tf.variable_scope(name) as scope:
        Wfc = weight_variable([num_in, num_out], name=name)
        bfc = bias_variable([num_out], name=name)

        tf.summary.histogram(name + "/weights", Wfc)
        tf.summary.histogram(name + "/biases", bfc)

        act = tf.nn.xw_plus_b(x, Wfc, bfc, name=name + '/op')

        return act

def fcnobias(x, num_in, num_out, name):
    with tf.variable_scope(name) as scope:
        Wfc = weight_variable([num_in, num_out], name=name)
        #bfc = bias_variable([num_out], name=name)
        tf.summary.histogram(name + "/weights", Wfc)
        #tf.summary.histogram(name + "/biases", bfc)

        #act = tf.nn.xw_plus_b(x, Wfc, bfc, name=name + '/op')
        act=tf.matmul(x, Wfc,name=name + '/nobias')
        return act

### ResBlock

cardinality = 32 # how many split ?

def transform_layer(input,out_feature_dim,stride,is_training,name):
    channel=input.shape[3]
    x=input
    # 1*1 conv
    x=conv2d(x,channel,out_feature_dim,1,1,is_training, name=name + '/transform_conv_1_1')
    # batch norm
    if is_training:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training,
                                                   updates_collections=tf.GraphKeys.UPDATE_OPS, scope=name+ '/bn_1')
    else:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training, scope=name+ '/bn_1')
    ## relu
    x = tf.nn.relu(x)

    ## 3*3 conv
    x=conv2d(x,out_feature_dim,out_feature_dim,3,stride,is_training, name=name + '/conv_layer_3_3')
    # batch norm
    if is_training:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training,
                                                   updates_collections=tf.GraphKeys.UPDATE_OPS, scope=name+ '/bn_2')
    else:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training, scope=name+ '/bn_2')
    ## relu
    x = tf.nn.relu(x)
    return x

def transition_layer(input,out_feature_dim, is_training,name):
    channel=input.shape[3]
    x=input
    ## 1*1 conv
    x=conv2d(x,channel,out_feature_dim,1,1,is_training, name=name + '/transit_conv_layer_1_1')
    ## batch norm
    if is_training:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training,
                                                   updates_collections=tf.GraphKeys.UPDATE_OPS, scope=name+ '/bn_1')
    else:
        x = tf.contrib.layers.batch_norm(x, decay=0.99, scale=True, is_training=is_training, scope=name+ '/bn_1')
    return x

def split_layer(input,out_feature_dim, cardinality, stride, is_training, name):
    x=input

    layers_split = list()

    dim=out_feature_dim//cardinality

    for i in range(cardinality) :
        splits = transform_layer(x, dim, stride, is_training, name=name + '_splitN_' + str(i))
        layers_split.append(splits)

    return tf.concat(layers_split, axis=3)

### ResNeXt50
def ResBlockfirstlayer(input, out_feature_dim, cardinality, name, downsample=True,  is_training=True):
    input_dim=input.shape[3]

    if input_dim==out_feature_dim:
        channel=input_dim//2
        x=tf.pad(input, [[0, 0], [0, 0], [0, 0],[channel, channel]])
    else:
        x=input

    if downsample:
        stride=2

        current=split_layer(input,out_feature_dim,cardinality,stride,is_training, name=name+'split_layer')
        current=transition_layer(current,2*out_feature_dim,is_training, name=name+'trans_layer')

        pad_x=avg_pool(x,2,2)

    else:
        stride=1

        current=split_layer(input,out_feature_dim,cardinality,stride,is_training, name=name+'split_layer')
        current=transition_layer(current,2*out_feature_dim,is_training,name=name+'trans_layer')

        pad_x=x

    x=current+pad_x
    return x

def ResBlocklayer(input, out_feature_dim, cardinality, name, downsample=False,  is_training=True):
    #input_dim=input.shape[3]

    x=input

    if downsample:
        stride=2
        #channel=input_dim//2

        current=split_layer(x,out_feature_dim,cardinality,stride,is_training, name=name+'split_layer')
        current=transition_layer(current,2*out_feature_dim,is_training, name=name+'trans_layer')

        pad_x=avg_pool(x,2,2)

    else:
        stride=1

        current=split_layer(x,out_feature_dim,cardinality,stride,is_training, name=name+'split_layer')
        current=transition_layer(current,2*out_feature_dim,is_training,name=name+'trans_layer')
        #current=squeeze_excitation_layer(current,2*out_feature_dim, reduction_ratio, is_training, name=name+'squeeze_layer')

        pad_x=x

    x=current+pad_x
    return x

### ResNeXt50  nosqueeze
def ResNeXt50(xs,is_training):
    current = tf.reshape(xs, [-1, 224, 224, 3])
    ## conv1   112*112
    current = conv2d(current, 3, 64, 7, 2, name='conv1')
    # batch norm
    if is_training:
        current = tf.contrib.layers.batch_norm(current, decay=0.99, scale=True, is_training=is_training,
                                                   updates_collections=tf.GraphKeys.UPDATE_OPS, scope='conv1/bn_1')
    else:
        current = tf.contrib.layers.batch_norm(current, decay=0.99, scale=True, is_training=is_training, scope='conv1/bn_1')
    # relu
    current = tf.nn.relu(current)

    ## conv2  56*56   128,128,256,C=32, 3
    current = max_pool(current, 3, 2)
    ### padding to 128-d
    current=tf.pad(current, [[0, 0], [0, 0], [0, 0],[32, 32]])

    current=ResBlockfirstlayer(current, 128,32,name='conv2_trans',downsample=False,is_training=is_training)
    for id in range(2):
        current=ResBlocklayer(current, 128,32,name='conv2_%d'%id,downsample=False,is_training=is_training)

    ## conv3 28*28  256,256,512, C=32, 4
    current=ResBlockfirstlayer(current, 256,32,name='conv3_trans',downsample=True,is_training=is_training)
    for id in range(3):
        current=ResBlocklayer(current, 256,32,name='conv3_%d'%id,downsample=False,is_training=is_training)

    ## conv4  14*14   512,512,1024   C=32, 6
    current=ResBlockfirstlayer(current, 512,32,name='conv4_trans',downsample=True,is_training=is_training)
    for id in range(5):
        current=ResBlocklayer(current, 512,32,name='conv4_%d'%id,downsample=False,is_training=is_training)

    ## conv5  7*7   1024,1024,2048,  C=32, 3
    current=ResBlockfirstlayer(current, 1024,32,name='conv5_trans',downsample=True,is_training=is_training)
    for id in range(2):
        current=ResBlocklayer(current, 1024,32,name='conv5_%d'%id,downsample=False,is_training=is_training)

    ## gloabl avearge pooling
    current = avg_pool(current, 7, 7)

    final_dim=2048

    current = tf.reshape(current, [-1, final_dim])
    output = fc(current, final_dim, 21, name='fc')

    return output

# In[18]:

def main():
    # Dataset path
    train_tfrecords = 'train.tfrecords'
    test_tfrecords = 'test.tfrecords'

    # Learning params  原来imagenet的学习率是0.001
    learning_rate = 0.001
    training_iters = 50400  # 一个epoch1680次
    batch_size = 20

    # Load batch
    train_img, train_label = input_pipeline(train_tfrecords, batch_size)
    test_img, test_label = input_pipeline(test_tfrecords, batch_size)

    # Model
    with tf.variable_scope('model_definition') as scope:
        train_output = ResNeXt50(train_img, is_training=True)
        scope.reuse_variables()
        test_output = ResNeXt50(test_img, is_training=False)

    # Loss and optimizer
    loss_op = loss(train_output, train_label)
    #l2_regularization = l2_reg()
    #loss_sum = loss_op + 0.0005 * l2_regularization

    tf.summary.scalar('loss', loss_op)

    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        train_op = train(loss_op, learning_rate)
        test_loss_op = loss(test_output, test_label)
        # train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss_op)

    # Evaluation
    train_accuracy = accuracy_of_batch(train_output, train_label)
    tf.summary.scalar("train_accuracy", train_accuracy)

    test_accuracy = accuracy_of_batch(test_output, test_label)
    tf.summary.scalar("test_accuracy", test_accuracy)

    # Init
    init = tf.global_variables_initializer()

    # Summary
    merged_summary_op = tf.summary.merge_all()

    # Create Saver
    # saver = tf.train.Saver(tf.trainable_variables())
    ### the default saver is tf.train.Saver() However use this leads to mistakes
    # saver = tf.train.Saver()

    ### new solution
    var_list = tf.trainable_variables()
    g_list = tf.global_variables()
    bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name]
    bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name]
    var_list += bn_moving_vars
    saver = tf.train.Saver(var_list=var_list,max_to_keep=1000)

    # Launch the graph
    with tf.Session() as sess:
        print('Init variable')
        sess.run(init)

        #load_ckpt_path = 'checkpoint/my-model.ckpt-346920'
        #saver.restore(sess, load_ckpt_path)

        summary_writer = tf.summary.FileWriter('logs', sess.graph)

        print('Start training')
        # coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess)
        for step in range(training_iters):
            step += 1
            # _, loss_value = sess.run([train_op, loss_op])
            # print('Generation {}: Loss = {:.5f}'.format(step, loss_value))
            # print(Wfc1value[1, 1], Wfc2value[1, 1])
            _, loss_value, test_loss_value = sess.run([train_op, loss_op, test_loss_op])
            print('Generation {}: Loss = {:.5f}     Test Loss={:.5f}'.format(step, loss_value, test_loss_value))

            # Display testing status
            if step % 40 == 0:
                acc1 = sess.run(train_accuracy)
                print(' --- Train Accuracy = {:.2f}%.'.format(100. * acc1))
                acc2 = sess.run(test_accuracy)
                print(' --- Test Accuracy = {:.2f}%.'.format(100. * acc2))

            if step % 40 == 0:
                summary_str = sess.run(merged_summary_op)
                summary_writer.add_summary(summary_str, global_step=step)
            if step % 840 == 0:
                saver.save(sess, 'checkpoint/my-model.ckpt', global_step=step)

        print("Finish Training and validation!")

        # coord.request_stop()
        # coord.join(threads)

# In[19]:

if __name__ == '__main__':
    main()