import re
result = dir(re)
str = "Python Kursu: Python Programlama Rehberiniz | 40 saat"
result = re.findall("Python",str) # Girilen pattern'dan kaç tane geçiyorsa tamamını listeler.
result = re.split(" ",str) # Girilen parametreye göre stringi böler.
result = re.sub(" ","-",str) # İlk parametredeki değeri ikinci ile değiştirir.
result = re.sub("\s","-",str) # \s regular expressions için boşluk karakteri yerine geçer.
ara = re.search("Programlama",str) # İçine girilen stringin İLK bulunduğu konumu match objesi olarak döndürür.
result = ara.span() # Başlangıç ve bitişi birlikte döndürür.
result = ara.start() # Sadece başlangıç değerini döndürür.
result = ara.end() # Sadece bitiş değerini döndürür.
result = ara.group() # Bulduğu ifadeyi döndürür.
result = ara.string # Arama işleminin nerede yapıldığını gösterir.

# Regular expressions
'''
Köşeli Parantezler [] arasına yazılan tüm karakterleri aranır.
    [abc] gibi                 [^abc] -> abc dışındaki karekterler
    [a-e] [a,b,c,d,e]          [^0-9] -> rakam olmayan karakterler.
    [1-9] [1,2,3,4,5,6,7,8,9]

. - Herhangi bir tek karakteri temsil eder
    re.findall(".",str)

^ - Belirtilen string  (tamamı) karakterle başlayıp başlamadığını kontrol eder.
    re.findall("^a",str)

$ - Belirtilen string (tamamı) karakterle bitiyor mu ?
    re.findall(t$,str)

* - Bir karakterin sıfır ya da daha fazla sayıda olup olmamasını kontrol eder.
        em*e => eme : one match
                emme : one match
                emmmmmmme: one match
                emte : No match (m'nin arkasından e gelmiyor.)

+ - Bir karakterin bir ya da daha fazla sayıda olmasını kontrol eder
        ma+n => man: One match
                maaan : One match
                main : No match a'nın arkasından n gelmiyor.

? - Bir karakterin sıfır ya da bir kez olup olmadığını kontrol eder (daha fazlası yok)
        ma?n = > man: One match
                 mn: One match
                 maaaaan: No match
{} - Karakter sayısını kontrol eder.
        al{2} => a karakterinin arkasındaki l iki defa tekrarlamalı.
        al{2,3} => a karakterinin arkasındaki l en az 2 en fazla 3 defa tekrarlamalı.
        [0-9]{2,4} => en az 2 en fazla 4 basamaklı sayılar.

| - ifadesi alternatif seçeneklerden en az birinin gerçekleşmesi anlamına gelir.

() - ifadesi grıplama yapmak için kullanılır.
        (a|b|c)xz => a,b,c karakterlerinin en az birinin arkasından xz gelmeli

\ - ifadesi özel karakterleri aramamızı sağlar.
        \$a $ karakterinin arkasında a karakteri arar.
            Regular exp. engine tarafından yorumlanmaz.

        \A belirtilen karakter stringin başındamı
            \Athe => The stringin başındamı

        \Z belirtilen karakter stringin sonundamı.
            \Zthe => The stringin sonundamı

\b - ifadesi belirtilen karakter kelimenin başında ya da sonunda mı ?
        \bthe => the kelimenin başında mı
        the\b => the kelimenin sonunda mı

\d - [0-9] ifadesi ile aynı anlama gelir rakam olanları arar.
        \D => 123abCD567

\D - [^0-9] ifadesi ile aynı anlama gelir rakam olmayanları arar.
        \D => 123abCD567

        \s boşluk karakterlerini arar.
        \S boşluk karakterleri dışındakileri arar.
        \w alfabetik karakterler, rakamlar ve alt çizgi karakteri
        \W \w nün tam tersi.
'''
new_Str = "Emmmre Tekin"
numberstr = "290 4500 12 336 55861"
str2 = "Balıkesir"
email = "emretekin_35@outlook.com"

result = re.findall("[abc]",str)
result = re.findall("[a-z]",str) # a-z arasındaki tüm harfler
result = re.findall("[0-9]",str) # tüm rakamlar
result = re.findall("[^a-z]",str) # alfabetik olmayan değerler.
result = re.findall("[^0-9]",str) # Rakam dışındaki değerler.
result = re.findall("....",str) # yazılan nokta kadar karakterleri gruplar.
result = re.findall("^Py",str) # Belirtilen string  (tamamı) karakterle başlayıp başlamadığını kontrol eder.
result = re.findall("t$",str) # Belirtilen string (tamamı) karakterle bitiyor mu ?
result = re.findall("Em*e",new_Str) # 0 ya da daha fazla sayıda olup olmadığını kontrol eder.
result = re.findall("Em+e",new_Str) # 1 ya da daha fazla sayıda olup olmadığını kontrol eder.
result = re.findall("Em?r",new_Str) # 0 ya da 1 kez olup olmadığını kontrol eder.
result = re.findall("m{2}",new_Str) # {} içine girilen değer kadar olup olmadığını kontrol eder.
result = re.findall("[0-9]{4}",numberstr)
result = re.findall("a|i",str2) # Or anlamına gelir
result = re.findall("(A|B|C)alı",str2) # Gruplama yapar
result = re.findall("\@o",email) # Özel karakter
result = re.findall("\ABalk",str2) # belirtilen ifadeyle başlayıp başlamadığını kontrol eder.
result = re.findall("sir\Z",str2) # belirtilen ifadeyle bitip bitmediğini kontrol eder.
result = re.findall("at\Z",str)
v2 = re.findall("\bBalı",new_Str)
v1 = re.findall("\d",str)
result = re.findall("\w",str)
result = re.findall("\W",str)
print(result)


from __future__ import print_function
# -*- coding: utf-8 -*-

import sys
sys.path.insert(0, '/home/kaminem64/itunes')
sys.path.insert(0, '/home/kaminem64/itunes/app')
import os
os.environ['DJANGO_SETTINGS_MODULE'] = 'itunes.settings'
import django
django.setup()

from app.models import *

import re
import gc
import gensim
import pickle
from string import punctuation
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import TweetTokenizer
from gensim import matutils
from gensim import models
from gensim.similarities import MatrixSimilarity, SparseMatrixSimilarity, Similarity
import datetime

GLOBAL_FIRST_DATE = datetime.datetime(2013, 11, 1)
GLOBAL_LAST_DATE = datetime.datetime(2016, 1, 1)

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

tokenizer = TweetTokenizer()
punc = re.compile('[%s]' % re.escape(punctuation))
only_english_letters = re.compile(r'[^a-zA-Z ]')

def tokenize_stemmer(text):
    if text:
        text = only_english_letters.sub('', text)
        if text:
            return [stemmer.stem(token) for token in tokenizer.tokenize(gensim.utils.to_unicode(text))]
        else:
            return []
    else:
        return []

def tokenize_lemmatizer(text):
    if text:
        text = punc.sub('', text)
        return [lemmatizer.lemmatizer(token) for token in tokenizer.tokenize(gensim.utils.to_unicode(text))]
    else:
        return []

num_topics = 50

# tfidf = pickle.load( open( "exports/tfidf.p", "rb" ) )
# corpus_gensim = pickle.load( open( "exports/corpus_gensim.p", "rb" ) )
# gensim_tfidf = list(corpus_gensim)
# vocab_gensim = pickle.load( open( "exports/vocab_gensim.p", "rb" ) )

app_ids = list(App.objects.filter(is_reviews_crawled=True, is_releasenotes_crawled=True, category__name='Productivity').order_by('id').values_list('store_app_id',flat=True))
print(len(app_ids), ' apps are being analyzed...')
# from random import shuffle
# shuffle(app_ids)
# app_ids = app_ids[:4000]
# print(app_ids)
tfidf_db_map = {}
class CleanDocuments(object):
    def __iter__(self):
        index = 0
        for document in ReviewReleaseNoteFlat.objects.filter(store_app_id__in=app_ids, is_review=True).order_by('id'):
            if index%10000 == 0:
                print(index)
            tfidf_db_map.update({document.id:index})
            index += 1
            yield document.body

# share stages for all models
vectorizer = TfidfVectorizer(ngram_range=(1, 3), tokenizer=tokenize_stemmer, min_df=3, max_df=0.3, binary=True, lowercase=True, strip_accents='ascii', stop_words='english')
tfidf = vectorizer.fit_transform(CleanDocuments())
print('Saving vectorizer...')
pickle.dump(vectorizer, open('exports/lda_vectorizer.p','wb'))
print('vectorizer saved!')
print('Saving tfidf...')
pickle.dump(tfidf, open('exports/lda_tfidf.p','wb'))
print('tfidf saved!')
print('Saving tfidf_db_map...')
pickle.dump(tfidf_db_map, open('exports/lda_tfidf_db_map.p','wb'))
print('tfidf_db_map saved!')
corpus_gensim = matutils.Sparse2Corpus(tfidf.T)
del(tfidf)
print('Saving corpus_gensim...')
pickle.dump(corpus_gensim, open('exports/lda_corpus_gensim.p','wb'))
print('corpus_gensim saved!')

vocab_gensim = {val: key for key, val in vectorizer.vocabulary_.items()} #gensim requires {id:word} dict not {word:id}
print('Saving vocab_gensim...')
pickle.dump(vocab_gensim, open('exports/lda_vocab_gensim.p','wb'))
print('vocab_gensim saved!')

lda = models.ldamulticore.LdaMulticore(corpus_gensim, workers=3, id2word=vocab_gensim, num_topics=num_topics, passes=5)#, alpha='auto') #auto needs the non-multicore version of LDA
print('Saving lda_model...')
lda.save('exports/lda.model')
print('lda_model saved!')


from werkzeug.wrappers import BaseResponse
import decimal

from app.channel.epaytong.deposit.callback import CallbackEpayTong
from app.channel.onepay.deposit.callback import CallbackOnePay
from app.enums.third_config import ThirdPayConfig
from app.libs.doc_response import ResponseDoc
from app.libs.error_code import ResponseSuccess
from app.logics.transaction.deposit_ctl import DepositTransactionCtl
from app.logics.transaction.withdraw_ctl import WithdrawTransactionCtl
from . import api
from flask_restplus import Resource
from config import EnvironEnum
from flask import current_app, request
from app.libs.ip_kit import IpKit

ns = api.namespace('callback', description='用户充值')

@ns.route('/epaytong/deposit')
@ResponseDoc.response(ns, api)
class EpayTongDeposit(Resource):

    def post(self):
        client_ip = IpKit.get_remote_ip()
        third_config = ThirdPayConfig.EpayTong_PAY_DEPOSIT.value

        if not EnvironEnum.is_local_evn(current_app.config['FLASK_ENV']):
            # 无论如何都记录一条log
            current_app.logger.info('epaytong deposit callback, ip: %s, data: %s', IpKit.get_remote_ip(), request.form)

        if not CallbackEpayTong.check_ip(client_ip):
            current_app.logger.fatal('ip not allow, client_ip: %s, data: %s, body: %s', client_ip, request.args,
                                     request.form)
            return BaseResponse('FAIlURE')

        sys_tx_id = request.form['order_no']
        order = DepositTransactionCtl.get_order(sys_tx_id)
        if not order:
            current_app.logger.error('epaytong: no found order, order id: %s' % (sys_tx_id))
            return BaseResponse('FAILURE')

        no_need_field = ['signType', 'sign']
        sorted_fields = sorted([k for k in request.form.keys()])
        request_str = "&".join(
            ["{}={}".format(k, request.form[k]) for k in sorted_fields if
             (request.form[k] == 0 or request.form[k] == "0" or request.form[k]) and k not in no_need_field])
        sign = request.form['sign']
        sign_str = request_str + third_config['secret_key']
        flag = CallbackEpayTong.check_sign(sign, sign_str)

        if not flag:
            current_app.logger.fatal('invalid sign, client_ip: %s, data: %s, body: %s', client_ip, request.args,
                                     request_str)
            return ResponseSuccess(code=500, message='签名错误').as_response()

        tx_amount = request.form['price']
        pwTradeId = request.form['notify_id']

        status = request.form['is_success']
        trade = request.form['trade_status']
        if status == 'T' and trade == "TRADE_FINISHED":
            if not DepositTransactionCtl.success_order_process(order, decimal.Decimal(tx_amount), pwTradeId):
                return BaseResponse('FAIlURE')
        else:
            if not DepositTransactionCtl.failed_order_process(order, decimal.Decimal(tx_amount), pwTradeId):
                return BaseResponse('FAIlURE')

        return BaseResponse('success')


import six

import logging
import sys
import traceback

import django
import django.conf
from django.db import connection
from django.utils.deprecation import MiddlewareMixin
from google.rpc import code_pb2

from opencensus.common import configuration
from opencensus.trace import (
    attributes_helper,
    execution_context,
    integrations,
    print_exporter,
    samplers,
)
from opencensus.trace import span as span_module
from opencensus.trace import status as status_module
from opencensus.trace import tracer as tracer_module
from opencensus.trace import utils
from opencensus.trace.propagation import trace_context_http_header_format

HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST']
HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD']
HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH']
HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE']
HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL']
HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE']
ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE']
ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME']
STACKTRACE = attributes_helper.COMMON_ATTRIBUTES['STACKTRACE']

REQUEST_THREAD_LOCAL_KEY = 'django_request'
SPAN_THREAD_LOCAL_KEY = 'django_span'

EXCLUDELIST_PATHS = 'EXCLUDELIST_PATHS'
EXCLUDELIST_HOSTNAMES = 'EXCLUDELIST_HOSTNAMES'

log = logging.getLogger(__name__)

class _DjangoMetaWrapper(object):
    """
    Wrapper class which takes HTTP header name and retrieve the value from
    Django request.META
    """

    def __init__(self, meta=None):
        self.meta = meta or _get_django_request().META

    def get(self, key):
        return self.meta.get('HTTP_' + key.upper().replace('-', '_'))

def _get_django_request():
    """Get Django request from thread local.

    :rtype: str
    :returns: Django request.
    """
    return execution_context.get_opencensus_attr(REQUEST_THREAD_LOCAL_KEY)

def _get_django_span():
    """Get Django span from thread local.

    :rtype: str
    :returns: Django request.
    """
    return execution_context.get_opencensus_attr(SPAN_THREAD_LOCAL_KEY)

def _get_current_tracer():
    """Get the current request tracer."""
    return execution_context.get_opencensus_tracer()

def _set_django_attributes(span, request):
    """Set the django related attributes."""
    django_user = getattr(request, 'user', None)

    if django_user is None:
        return

    user_id = django_user.pk
    user_name = django_user.get_username()

    # User id is the django autofield for User model as the primary key
    if user_id is not None:
        span.add_attribute('django.user.id', str(user_id))

    if user_name is not None:
        span.add_attribute('django.user.name', str(user_name))

def _trace_db_call(execute, sql, params, many, context):
    tracer = _get_current_tracer()
    if not tracer:
        return execute(sql, params, many, context)

    vendor = context['connection'].vendor
    alias = context['connection'].alias

    span = tracer.start_span()
    span.name = '{}.query'.format(vendor)
    span.span_kind = span_module.SpanKind.CLIENT

    tracer.add_attribute_to_current_span('component', vendor)
    tracer.add_attribute_to_current_span('db.instance', alias)
    tracer.add_attribute_to_current_span('db.statement', sql)
    tracer.add_attribute_to_current_span('db.type', 'sql')

    try:
        result = execute(sql, params, many, context)
    except Exception:  # pragma: NO COVER
        status = status_module.Status(
            code=code_pb2.UNKNOWN, message='DB error'
        )
        span.set_status(status)
        raise
    else:
        return result
    finally:
        tracer.end_span()

class OpencensusMiddleware(MiddlewareMixin):
    """Saves the request in thread local"""

    def __init__(self, get_response):
        super(OpencensusMiddleware, self).__init__(get_response)
        settings = getattr(django.conf.settings, 'OPENCENSUS', {})
        settings = settings.get('TRACE', {})

        self.sampler = (settings.get('SAMPLER', None)
                        or samplers.ProbabilitySampler())
        if isinstance(self.sampler, six.string_types):
            self.sampler = configuration.load(self.sampler)

        self.exporter = settings.get('EXPORTER', None) or \
            print_exporter.PrintExporter()
        if isinstance(self.exporter, six.string_types):
            self.exporter = configuration.load(self.exporter)

        self.propagator = settings.get('PROPAGATOR', None) or \
            trace_context_http_header_format.TraceContextPropagator()
        if isinstance(self.propagator, six.string_types):
            self.propagator = configuration.load(self.propagator)

        self.excludelist_paths = settings.get(EXCLUDELIST_PATHS, None)

        self.excludelist_hostnames = settings.get(EXCLUDELIST_HOSTNAMES, None)

        if django.VERSION >= (2,):  # pragma: NO COVER
            connection.execute_wrappers.append(_trace_db_call)

        # pylint: disable=protected-access
        integrations.add_integration(integrations._Integrations.DJANGO)

    def process_request(self, request):
        """Called on each request, before Django decides which view to execute.

        :type request: :class:`~django.http.request.HttpRequest`
        :param request: Django http request.
        """
        # Do not trace if the url is excludelisted
        if utils.disable_tracing_url(request.path, self.excludelist_paths):
            return

        # Add the request to thread local
        execution_context.set_opencensus_attr(
            REQUEST_THREAD_LOCAL_KEY,
            request)

        execution_context.set_opencensus_attr(
            'excludelist_hostnames',
            self.excludelist_hostnames)

        try:
            # Start tracing this request
            span_context = self.propagator.from_headers(
                _DjangoMetaWrapper(_get_django_request().META))

            # Reload the tracer with the new span context
            tracer = tracer_module.Tracer(
                span_context=span_context,
                sampler=self.sampler,
                exporter=self.exporter,
                propagator=self.propagator)

            # Span name is being set at process_view
            span = tracer.start_span()
            span.span_kind = span_module.SpanKind.SERVER
            tracer.add_attribute_to_current_span(
                attribute_key=HTTP_HOST,
                attribute_value=request.get_host())
            tracer.add_attribute_to_current_span(
                attribute_key=HTTP_METHOD,
                attribute_value=request.method)
            tracer.add_attribute_to_current_span(
                attribute_key=HTTP_PATH,
                attribute_value=str(request.path))
            tracer.add_attribute_to_current_span(
                attribute_key=HTTP_ROUTE,
                attribute_value=str(request.path))
            tracer.add_attribute_to_current_span(
                attribute_key=HTTP_URL,
                attribute_value=str(request.build_absolute_uri()))

            # Add the span to thread local
            # in some cases (exceptions, timeouts) currentspan in
            # response event will be one of a child spans.
            # let's keep reference to 'django' span and
            # use it in response event
            execution_context.set_opencensus_attr(
                SPAN_THREAD_LOCAL_KEY,
                span)

        except Exception:  # pragma: NO COVER
            log.error('Failed to trace request', exc_info=True)

    def process_view(self, request, view_func, *args, **kwargs):
        """Process view is executed before the view function, here we get the
        function name add set it as the span name.
        """

        # Do not trace if the url is excludelisted
        if utils.disable_tracing_url(request.path, self.excludelist_paths):
            return

        try:
            # Get the current span and set the span name to the current
            # function name of the request.
            tracer = _get_current_tracer()
            span = tracer.current_span()
            span.name = utils.get_func_name(view_func)
        except Exception:  # pragma: NO COVER
            log.error('Failed to trace request', exc_info=True)

    def process_response(self, request, response):
        # Do not trace if the url is excludelisted
        if utils.disable_tracing_url(request.path, self.excludelist_paths):
            return response

        try:
            span = _get_django_span()
            span.add_attribute(
                attribute_key=HTTP_STATUS_CODE,
                attribute_value=response.status_code)

            _set_django_attributes(span, request)

            tracer = _get_current_tracer()
            tracer.end_span()
            tracer.finish()
        except Exception:  # pragma: NO COVER
            log.error('Failed to trace request', exc_info=True)
        finally:
            return response

    def process_exception(self, request, exception):
        # Do not trace if the url is excluded
        if utils.disable_tracing_url(request.path, self.excludelist_paths):
            return

        try:
            if hasattr(exception, '__traceback__'):
                tb = exception.__traceback__
            else:
                _, _, tb = sys.exc_info()

            span = _get_django_span()
            span.add_attribute(
                attribute_key=ERROR_NAME,
                attribute_value=exception.__class__.__name__)
            span.add_attribute(
                attribute_key=ERROR_MESSAGE,
                attribute_value=str(exception))
            span.add_attribute(
                attribute_key=STACKTRACE,
                attribute_value='\n'.join(traceback.format_tb(tb)))

            _set_django_attributes(span, request)
        except Exception:  # pragma: NO COVER
            log.error('Failed to trace request', exc_info=True)


import unittest
import random  # NOQA
import importlib
def PEP(cls=None, *, testnum=True, module=True):
    def pep(cls):
        nonlocal testnum, module
        num = int(cls.__name__.split('PEP')[1])
        try:
            module = importlib.import_module(f'pep{num}')
        except ImportError:
            try:
                module = importlib.import_module(f'.pep{num}', 'rejected_peps')
            except ImportError:
                module = None
        if module is None:
            return unittest.skip('Import failure')(cls)
        if module:
            setattr(cls, f'pep{num}', module)
        setattr(cls, 'num', num)
        def test_pepnum(self):
            nonlocal num
            self.assertEqual(getattr(self, f'pep{num}').PEP, self.num)
        if testnum:
            cls.test_pepnum = test_pepnum
        return cls
    if callable(cls):
        return pep(cls)
    return pep
objects = (1,
           20.5,
           complex(6, 2),
           '666',
           b'777',
           (8, 9, 10),
           [8, 9, 10],
           {8, 9, 10},
           {'8': 9, '10': 11, '12': 13},
           memoryview(b'Hello, world!'),
           True,
           (x for x in range(25)),
           enumerate([555, 999, 777, -111]))
iterables = objects[5:9]

class TestInit(unittest.TestCase):
    def setUp(self):
        try:
            from . import pep, search, get, pepinfo, info, SUPPORTED
        except ImportError:
            self.skipTest(f'Import failure')
        self.pep, self.search, self.get = pep, search, get
        self.pepinfo, self.info = pepinfo, info
        self.sp = SUPPORTED
    def assertModuleEqual(self, first, pepnum: int, msg=None):
        # @PEP is run before test cases are run, so it's OK if TestInit runs before other cases.
        self.assertEqual(first, getattr(globals()[f'TestPEP{pepnum}'], f'pep{pepnum}'),
                         msg=msg)
    def test_pep(self):
        # Warning: SUPPORTED is not tested. Please make sure it is correct.
        for p in self.sp:
            try:
                self.assertModuleEqual(self.pep(p), p)
            except AttributeError:
                import sys
                print(f'Skipped PEP {p} in TestInit', file=sys.stderr)
        # Combinations tested in TestCombine.
    def test_search(self):
        self.assertListEqual(list(self.search('range literals')), [204])
        self.assertListEqual(list(self.search('and')), [281, 326])
        self.assertListEqual(list(self.search('A', strict=True)), [211, 313, 326, 349, 416])
        self.assertListEqual(list(self.search('a', 'for')), [276, 326])
    def test_search_any(self):
        self.assertListEqual(list(self.search.any('range literals', 'and')), [204, 281, 326])
    def test_search_one(self):
        self.assertEqual(self.search.one('iterator'), 276)
        with self.assertRaisesRegex(ValueError, 'More than 1 match'):
            self.search.one('int')
        with self.assertRaisesRegex(ValueError, 'No match found'):
            self.search.one('integer')
    def test_search_one_any(self):
        self.assertEqual(self.search.one.any('iterator', 'integer'), 276)
        with self.assertRaisesRegex(ValueError, 'More than 1 match'):
            self.search.one.any('int', 'iterator')
        with self.assertRaisesRegex(ValueError, 'No match found'):
            self.search.one.any('iterators', 'integer')
    def test_get(self):
        self.assertModuleEqual(self.get('iterator'), 276)
        self.assertModuleEqual(self.get.any('iterator', 'integer'), 276)
@PEP
class TestPEP204(unittest.TestCase):
    def test_all_given(self):
        params = {(1, 2, 3),
                  (0, 10, 2),
                  (-20, 125, 67),
                  (75, 26, -2),
                  (6, -3, -1),
                  (0, 0, 1),
                  }
        for p in params:
            with self.subTest(p=p):
                self.assertEqual(self.pep204[p[0]:p[1]:p[2]], range(*p))
    def test_omitted(self):
        params = ((1, 2),
                  (7,),
                  (4, 21),
                  (6, 17),
                  (1, 6),
                  [7, 60],
                  (-1,),
                  [16],
                  )
        for p in params:
            with self.subTest(p=p):
                if isinstance(p, list):
                    if len(p) == 1:
                        self.assertEqual(self.pep204[p[0]], range(*p))
                    else:
                        self.assertEqual(self.pep204[:p[0]:p[1]],
                                         range(0, *p))
                elif len(p) == 1:
                    self.assertEqual(self.pep204[:p[0]:], range(*p))
                elif len(p) == 2:
                    self.assertEqual(self.pep204[p[0]:p[1]], range(*p))
                else:
                    self.fail(f'Invalid argument {p!r}')
    def test_errors(self):
        for p in {(1, 2), (3, 4), (20, -6), (1024, 69927)}:
            with self.subTest(p=p):
                with self.assertRaisesRegex(ValueError,
                                            'Stop is required'):
                    self.pep204[p[0]::p[1]]
        for p in {(1.0, 2.5, 6.2), (49.4, 999.4, 7777.291),
                  (-1, 0.15, 5.4), (7.39, 9993, -1.0)}:
            with self.subTest(p=p):
                with self.assertRaises(TypeError):
                    self.pep204[p[0]:p[1]:p[2]]
@PEP
class TestPEP211(unittest.TestCase):
    def setUp(self):
        self.w = self.pep211.wrapper
        self.itertools = __import__('itertools')
    def assertEq(self, *args, **kwargs):
        try:
            a, b = list(args[0]), list(args[1])
        except TypeError:
            a, b = args[:2]
        return self.assertEqual(a, b, *args[2:], **kwargs)
    def test_call(self):
        for obj in objects:
            with self.subTest(obj=obj):
                self.assertEqual(self.w(obj)(), obj)
    def test_success(self):
        self.assertEq(self.w((6, 6, 6)) @ (7, 7, 7),
                      self.itertools.product((6, 6, 6), (7, 7, 7)))
        self.assertEq(self.w((-1, -9, 6, 2, 7)) @ self.w((94, 9)),
                      self.itertools.product((-1, -9, 6, 2, 7),
                                             (94, 9)))
        self.assertEq((3.14, 3.15) @ self.w((2.71, 2.72)),
                      self.itertools.product((3.14, 3.15),
                                             (2.71, 2.72)))
@PEP
class TestPEP212(unittest.TestCase):
    def test_indices(self):
        for i in iterables:
            with self.subTest(i=i):
                self.assertEqual(self.pep212.indices(i), range(len(i)))
    def test_irange(self):
        for i in iterables:
            with self.subTest(i=i):
                self.assertEqual(list(self.pep212.irange(i)),
                                list(enumerate(i)))
                self.assertEqual(list(self.pep212.irange(i)),
                                list(zip(range(len(i)), i)))
@PEP
class TestPEP259(unittest.TestCase):
    def setUp(self):
        self.sio = __import__('io').StringIO
    def test_regular(self):
        s = self.sio()
        self.pep259.print(file=s)
        self.assertEqual(s.getvalue(), '\n')
        del s
        s = self.sio()
        self.pep259.print(1, 2, 3, 4, file=s)
        self.assertEqual(s.getvalue(), '1 2 3 4\n')
        del s
        s = self.sio()
        self.pep259.print(787, '4949848373737', -1.5,
                          sep='\n', end='\t123\n', file=s)
        self.assertEqual(s.getvalue(), '787\n4949848373737\n-1.5\t123\n')
        del s
        s = self.sio()
        try:
            with self.assertRaisesRegex(TypeError, 'end must be None or a string'):
                self.pep259.print(3, end=2, file=s)
        finally:
            del s
    def test_omit(self):
        s = self.sio()
        self.pep259.print(1, 2, 3, 4, '\n', file=s)
        self.assertEqual(s.getvalue(), '1 2 3 4 \n')
        del s
        s = self.sio()
        self.pep259.print(27575, '\n\n', sep='\t', file=s)
        self.assertEqual(s.getvalue(), '27575\t\n\n')
        del s
@PEP
class TestPEP265(unittest.TestCase):
    def test_sorted(self):
        self.assertIs(self.pep265.sorted, sorted)
    def test_constants(self):
        self.assertEqual(self.pep265.ORIGINAL, -1)
        self.assertEqual(self.pep265.KEYS, 0)
        self.assertEqual(self.pep265.VALUES, +1)
    def test_itemlist(self):
        import math
        sp = self.pep265
        testdict = {1: 9, 5: 6, 3: 3}
        # Test ORIGINAL
        self.assertListEqual(sp.itemlist(testdict, sp.ORIGINAL),
                             list(testdict.items()))
        # Test KEYS & key
        self.assertListEqual(sp.itemlist(testdict, sp.KEYS,
                                         key=math.log2),
                             [(1, 9), (3, 3), (5, 6)])
        # Test VALUES & reverse
        self.assertListEqual(sp.itemlist(testdict, sp.VALUES, reverse=True),
                             [(1, 9), (5, 6), (3, 3)])
        # Test ORIGINAL & reverse
        self.assertListEqual(sp.itemlist(testdict, sp.ORIGINAL, reverse=True),
                             [(3, 3), (5, 6), (1, 9)])
        with self.assertRaises(TypeError):
            sp.itemlist({None: 1, 3: 2, 4: None}, sp.KEYS)
    def test_alias(self):
        self.assertIs(self.pep265.items, self.pep265.itemlist)
@PEP
class TestPEP276(unittest.TestCase):
    def test_subclass(self):
        self.assertIsInstance(self.pep276.int(20), int)
    def test_iter(self):
        self.assertTupleEqual(tuple(self.pep276.int(99)),
                              tuple(range(99)))
        self.assertTupleEqual(tuple(self.pep276.int('0')),
                              tuple(range(0)))
        self.assertTupleEqual(tuple(self.pep276.int(-3.14)),
                              tuple(range(0)))
@PEP
class TestPEP281(unittest.TestCase):
    def test_regular(self):
        from decimal import Decimal
        r = self.pep281.range
        self.assertTupleEqual(tuple(r(1, 6)),
                              (1, 2, 3, 4, 5))
        self.assertTupleEqual(tuple(r(7, 3, -1)),
                              (7, 6, 5, 4))
        self.assertTupleEqual(tuple(r(1, 9, 2)),
                              (1, 3, 5, 7))
        A = type('A', (), {'__index__': (lambda _: 2)})
        self.assertTupleEqual(tuple(r(A(), 5)),
                              (2, 3, 4))
        for a in [(1.0, 2.0, 0.5),
                  (Decimal('1')),
                  (complex(2, 7), 7)]:
            with self.subTest(a=a):
                with self.assertRaises(TypeError):
                    r(*a)
    def test_new(self):
        import itertools
        r = self.pep281.range
        for a, b, c in itertools.permutations(iterables, 3):
            with self.subTest(a=a, b=b, c=c):
                self.assertEqual(r(a, b, c), range(len(a),
                                                   len(b),
                                                   len(c)))
@PEP
class TestPEP294(unittest.TestCase):
    def test_helpers(self):
        # Success
        self.assertEqual(self.pep294.underscore('lambda'), 'lambda_')
        self.assertEqual(self.pep294.title('lambda'), 'Lambda')
        self.assertEqual(self.pep294.original('lambda'), 'lambda')
        # Errors
        with self.assertRaises(TypeError):
            self.pep294.underscore(1.5)
        with self.assertRaises(TypeError):
            self.pep294.title([666, 272])
        with self.assertRaises(TypeError):
            self.pep294.original(b'6666666')
    def test_apply(self):
        import types
        self.pep294.apply()
        self.assertIn('function', dir(types))
        self.assertIn('lambda_', dir(types))
        self.assertNotIn('lambda', dir(types))
        self.pep294.apply(rename=self.pep294.title)
        self.assertIn('Lambda', dir(types))
        self.pep294.apply(rename=self.pep294.original)
        self.assertIn('lambda', dir(types))
        # TODO: Check strict
@PEP
class TestPEP303(unittest.TestCase):
    def test_divmod(self):
        d = self.pep303.divmod
        self.assertEqual(d(200, 3, 7, 5, 9),
                         (0, 2, 3, 4, 1))
        self.assertEqual(d(373, 4), (93, 1))
        with self.assertRaises(TypeError):
            d(314, 159.265, '1521')
    def test_rdivmod(self):
        r = self.pep303.rdivmod
        self.assertEqual(r(200, 9, 5, 7, 3),
                         (0, 1, 4, 3, 2))
        self.assertEqual(r(373, 4), (93, 1))
        with self.assertRaises(TypeError):
            r(314, 159.265, '1521')
    def test_inverse(self):
        self.assertEqual(self.pep303.inverse_divmod([0, 1, 4, 3, 2],
                                                    9, 5, 7, 3), 200)
@PEP
class TestPEP313(unittest.TestCase):
    def test_to(self):
        f = self.pep313.to_roman
        # MODERN
        self.assertEqual(f(111), 'CXI')
        self.assertEqual(f(-49), '-IL')
        # CLASSIC
        self.assertEqual(f(4, self.pep313.CLASSIC), 'IIII')
        self.assertEqual(f(998, self.pep313.CLASSIC),
                         'DCCCCLXXXXVIII')
        # LARGE
        self.assertEqual(f(-1_000_000, self.pep313.LARGE), '-_M')
        self.assertEqual(f(500_000_000_000, self.pep313.LARGE), '___D')
    def test_from(self):
        f = self.pep313.from_roman
        self.assertEqual(f('CXI'), 111)
        self.assertEqual(f('IL'), 49)
        self.assertEqual(f('-IIII'), -4)
        self.assertEqual(f('DCCCCLXXXXVIII'), 998)
        self.assertEqual(f('_M'), 1_000_000)
        self.assertEqual(f('___D'), 500_000_000_000)
    def test_zeros(self):
        with self.assertRaisesRegex(ValueError, 'x cannot be 0'):
            self.pep313.to_roman(0)
        self.assertEqual(self.pep313.to_roman(0, zero='Z'), 'Z')
        self.pep313.default_zero = 'N'
        self.assertEqual(self.pep313.to_roman(0), 'N')
        self.assertEqual(self.pep313.from_roman('N'), 0)
        self.assertEqual(self.pep313.from_roman('O', zero='O'), 0)
    def test_fraction(self):
        from fractions import Fraction as F
        self.assertEqual(self.pep313.to_roman(F(3, 7)), 'III/VII')
        self.assertEqual(self.pep313.from_roman('-XIV/II'), F(-14, 2))
    def test_errors(self):
        invalid_index = type('Invalid', (),
                             {'__index__': (lambda _: '')})
        with self.assertRaises(ValueError):
            self.pep313.to_roman(10, 'spam')
        with self.assertRaisesRegex(ValueError, 'x cannot be 0'):
            self.pep313.to_roman(0)
        with self.assertRaises(TypeError):
            self.pep313.to_roman(invalid_index())
        with self.assertRaises(TypeError):
            self.pep313.from_roman(b'foo')
        with self.assertRaises(ValueError):
            self.pep313.from_roman('')
        with self.assertRaises(ValueError):
            self.pep313.from_roman('bar')
        with self.assertRaises(ValueError):
            self.pep313.from_roman('__')
@PEP
class TestPEP326(unittest.TestCase):
    def test_Min(self):
        Min = self.pep326.Min
        for obj in objects:
            with self.subTest(obj=obj):
                self.assertLess(Min, obj)
                self.assertLessEqual(Min, obj)
                self.assertFalse(Min >= obj)
        self.assertEqual(Min, Min)
        self.assertIs(Min, type(Min)())
    def test_Max(self):
        Max = self.pep326.Max
        for obj in objects:
            with self.subTest(obj=obj):
                self.assertGreater(Max, obj)
                self.assertGreaterEqual(Max, obj)
                self.assertFalse(Max <= obj)
        self.assertEqual(Max, Max)
        self.assertIs(Max, type(Max)())
    def test_alias(self):
        self.assertIs(self.pep326.Min, self.pep326.UniversalMinimum)
        self.assertIs(self.pep326.Max, self.pep326.UniversalMaximum)
@PEP
class TestPEP335(unittest.TestCase):
    def setUp(self):
        self.plain = self.pep335.plain
    def test_alias(self):
        self.assertIs(self.pep335.not_, self.pep335.NOT)
        self.assertIs(self.pep335.or_, self.pep335.OR)
        self.assertIs(self.pep335.and_, self.pep335.AND)
        self.assertIs(self.plain.and_, self.plain.AND)
        self.assertIs(self.plain.or_, self.plain.OR)
        self.assertIs(self.plain.not_, self.plain.NOT)
    ...
@PEP
class TestPEP336(unittest.TestCase):
    def setUp(self):
        self.nt = self.pep336.NoneType
    def test_singleton(self):
        n = self.nt()
        self.assertEqual(repr(n), 'None')
        self.assertIs(n, self.nt())
    def test_eq(self):
        self.assertEqual(self.nt(), self.nt())
        self.assertEqual(self.nt(), None)
        self.assertNotEqual(self.nt(), 10)
    def test_isNone(self):
        self.assertTrue(self.pep336.isNone(self.nt()))
        self.assertTrue(self.pep336.isNone(None))
        self.assertFalse(self.pep336.isNone(...))
@PEP
class TestPEP349(unittest.TestCase):
    def setUp(self):
        self.s = self.pep349.str
    def test_str(self):
        # Normal
        self.assertEqual(self.s('16'), '16')
        self.assertEqual(self.s(['1', 2]), "['1', 2]")
        self.assertEqual(self.s(b'pep349', 'utf-8'), 'pep349')
        self.assertEqual(self.s(str), "<class 'str'>")
        # Bytes
        class A:
            def __str__(self): return b'A'
        self.assertEqual(self.s(A()), b'A')
        self.assertEqual(self.s(b'A'), b'A')
        # Fallbacks & Errors
        class B:
            def __repr__(self): return 'B'
        class C:
            def __str__(self): return 1
        self.assertEqual(self.s(B()), 'B')
        with self.assertRaises(TypeError):
            self.s(C())
    def test_methods(self):
        self.assertEqual(self.s('16')[0], '1')
        self.assertFalse(self.s(['1', 2]).startswith('('))
        class A:
            def __str__(self): return b'A'
        self.assertEqual(self.s(A()).lower(), b'a')
        self.assertTrue(self.s.startswith('50', '5'))
    def test_isinstance_issubclass(self):
        self.assertTrue(isinstance('1', self.s))
        self.assertFalse(isinstance(-1, self.s))
        self.assertTrue(issubclass(str, self.s))
        self.assertTrue(issubclass(self.s, str))
@PEP
class TestPEP351(unittest.TestCase):
    def setUp(self):
        self.freeze = self.pep351.freeze
    def test_hashable(self):
        class H:
            def __hash__(self): return -3
            def __eq__(self, other): return isinstance(other, H)
        hashables = {0, 3.14, 'string', b'foo',
                     ((3, 4), 5), H()}
        for h in hashables:
            with self.subTest(h=h):
                self.assertEqual(self.freeze(h), h)
    def test_set(self):
        self.assertEqual(self.freeze({1, 2}), frozenset((1, 2)))
        self.assertEqual(self.freeze(set()), frozenset(()))
    def test_list(self):
        self.assertEqual(self.freeze([(6, 3), 'xxx']),
                         ((6, 3), 'xxx'))
        self.assertEqual(self.freeze([]), ())
    def test_dict(self):
        from types import MappingProxyType as MPT
        fd = TestPEP416.pep416.frozendict
        # MappingProxyType
        self.assertEqual(self.freeze({'a': 20, 10: -5}),
                         MPT({'a': 20, 10: -5}))
        # frozendict
        self.assertEqual(self.freeze({'a': 5, 'b': -20}, allow_frozendict=True),
                         fd(a=5, b=-20))
        self.assertIsNotNone(self.pep351._pep416)  # Lazy import
    def test_manual(self):
        class A:
            __hash__ = None
            def __freeze__(self): return 2
        self.assertEqual(self.freeze(A()), 2)
    def test_failure(self):
        class A:
            __hash__ = None
        with self.assertRaises(TypeError):
            self.freeze(A())
@PEP
class TestPEP416(unittest.TestCase):
    def setUp(self):
        self.d = self.pep416.frozendict
    def test_errors(self):
        d = self.d()
        for name in {'__setitem__', '__delitem__', 'clear',
                     'update', 'setdefault', 'pop', 'popitem'}:
            with self.subTest(name=name):
                with self.assertRaises(TypeError):
                    try:
                        getattr(d, name)()
                    except Exception as e:
                        try:
                            getattr(d, name)()
                        except Exception:
                            raise e from None
    def test_initread(self):
        d = self.d({'1': 20, -3: 77.})
        self.assertEqual(d['1'], 20)
        self.assertEqual(d.get(-3), 77//1.)
        self.assertEqual(d.get(23, 2), 2)
        self.assertTupleEqual(tuple(d.keys()), ('1', -3))
        self.assertEqual(d, d.copy())
@PEP
class TestPEP535(unittest.TestCase):
    def setUp(self):
        self._cmp = self.pep535._cmp
        self.cmp = self.pep535.cmp
        self.p = self.pep535
    def test_acmp(self):
        self.assertTrue(self._cmp(5, '<', 6))
        self.assertTrue(self._cmp(7.6, self.p.GT, -9.3))
        with self.assertRaisesRegex(ValueError, 'Invalid op'):
            self._cmp(6, '!<', 5)
        with self.assertRaisesRegex(ValueError, 'Invalid op'):
            self._cmp(6, '<>', 5)
        with self.assertRaisesRegex(TypeError, 'Invalid op'):
            self._cmp(6, 0, 5)
    def test_cmp(self):
        self.assertTrue(self.cmp(5, '<', 6, '<', 7))
        self.assertTrue(self.cmp(9, '>', 7, '<', 15, '!=', 90, '==', 90))
        self.assertFalse(self.cmp(5, '<', 9, '>', 7, '<', 6))
        self.assertTrue(self.cmp(5, '>', 7, '>', 9, and_func=(lambda x, y: True)))
        with self.assertRaisesRegex(ValueError, '^Invalid arguments'):
            self.cmp(5, '>', 3, '>')
        import sys as _sys
        if _sys.version_info >= (3, 11):
            class A:
                def __bool__(self):
                    raise ValueError('Value Invalid')
            class B:
                def __lt__(self, other):
                    return A()
            try:
                c = self.cmp(B(), '<', B(), '<', B())
            except ValueError as e:
                self.assertRegex(e.__notes__[0], 'Error occurred while falling back')
            else:
                self.fail(f"ValueError not raised during 'self.cmp(B(), '<', A())'; "
                          f'{c!r} returned')
    def test_numpy(self):
        try:
            import numpy as np  # NOQA
        except Exception:
            self.skipTest('numpy not found')
        else:
            i = np.arange(5)
            self.assertEqual(repr(self.cmp(0, '<', i, '<', 4, and_func=np.logical_and)),
                             repr(np.array([False, True, True, True, False], dtype=bool)))
@PEP
class TestPEP559(unittest.TestCase):
    def test_noop(self):
        noop = self.pep559.noop
        self.assertIsNone(noop())
        for i in iterables[:-1]:
            self.assertIsNone(noop(*i))
        self.assertIsNone(noop(**iterables[-1]))
        for o in objects:
            self.assertIsNone(noop(o))
@PEP
class TestPEP754(unittest.TestCase):
    def test_values(self):
        p = self.pep754
        self.assertIs(p.NaN, p.NaN)
        self.assertNotEqual(p.NaN, p.NaN)
        self.assertEqual(p.PosInf, float('inf'))
        self.assertEqual(p.NegInf, -float('inf'))
        self.assertNotEqual(-1, p.PosInf / p.NegInf)
    def test_functions(self):
        p = self.pep754
        self.assertTrue(p.isNaN(p.NaN))
        self.assertFalse(p.isNaN(0.5))
        self.assertFalse(p.isNaN(-2))
        self.assertTrue(p.isPosInf(p.PosInf))
        self.assertFalse(p.isPosInf(3.14))
        self.assertFalse(p.isPosInf(p.NegInf))
        self.assertTrue(p.isNegInf(p.NegInf))
        self.assertFalse(p.isNegInf(-2.718))
        self.assertFalse(p.isNegInf(p.NaN))
        self.assertTrue(p.isFinite(42))
        self.assertTrue(p.isFinite(-0.618))
        self.assertFalse(p.isFinite(p.PosInf))
        self.assertFalse(p.isFinite(p.NegInf))
        self.assertFalse(p.isFinite(p.NaN))
        self.assertTrue(p.isInf(p.PosInf))
        self.assertTrue(p.isInf(p.NegInf))
        self.assertFalse(p.isInf(p.NaN))
        self.assertFalse(p.isInf(-0.0))
@PEP
class TestPEP3140(unittest.TestCase):
    def setUp(self):
        self.s = self.pep3140.str
    def test_str(self):
        self.assertEqual(self.s(['1', '2']), '[1, 2]')
        self.assertEqual(self.s({'1': '2', 3: '4'}), '{1: 2, 3: 4}')
        self.assertEqual(self.s((16, 25, '43', 93)), '(16, 25, 43, 93)')
        from collections import defaultdict
        dd = defaultdict(int, {'1': 2})
        self.assertEqual(self.s(dd), "defaultdict(<class 'int'>, {1: 2})")
        self.assertEqual(self.s(str), "<class 'str'>")
    def test_methods(self):
        self.assertEqual(self.s(['1', '2'])[0], '[')
        self.assertFalse(self.s({'1': '2', 3: '4'}).startswith('('))
        self.assertEqual(self.s((16, 25, '43', 93)) + 'x', '(16, 25, 43, 93)x')
        self.assertTrue(self.s.startswith('50', '5'))
        self.assertEqual(self.s(b'50', 'utf-8'), '50')
    def test_isinstance_issubclass(self):
        self.assertTrue(isinstance('1', self.s))
        self.assertFalse(isinstance(-1, self.s))
        self.assertTrue(issubclass(str, self.s))
        self.assertTrue(issubclass(self.s, str))
class TestCombine(unittest.TestCase):
    def setUp(self):
        try:
            from . import combined
            from . import pep349, pep3140, pep204, pep212, pep281
        except ImportError:
            import combined
            import pep349, pep3140, pep204, pep212, pep281
        self.c = combined
        self.p349, self.p3140 = pep349, pep3140
        self.p204, self.p212, self.p281 = pep204, pep212, pep281
    def test349_3140(self):
        self.assertEqual(self.p349.str & self.p3140.str, self.c.str)
    def test204_281(self):
        self.assertEqual(self.p204 & self.p281.range, self.c.rliteral)
    def test212_281(self):
        self.assertEqual(self.p212.indices & self.p281.range, self.p212.indices)
    def test204_212_raises(self):
        with self.assertRaises(TypeError):
            self.p204 & self.p212.indices
def run(**kwargs):
    if 'v' in kwargs and 'verbosity' not in kwargs:
        kwargs['verbosity'] = kwargs.pop('v')
    kwargs.setdefault('verbosity', 2)
    kw = {'exit': False, 'module': __name__, **kwargs}
    unittest.main(**kw)
if __name__ == '__main__':
    run(v=2, exit=True)


# Senior Data Analyst - Technical Exercise

# Import packages
#region

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pycountry
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')

from scipy import stats

#endregion

# Import data
print('Please make sure your data set is stored in the same directory as this script!')
df = pd.read_excel('Interview_HW.xlsx')

# Data Quality
#region

print('Data Quality:')

## Get overview
display(df)
display(list(df))
display(df.shape)
display(df.describe().round(2))

## Data types
display(df.dtypes)
df_cleaned = df.convert_dtypes()
display(df_cleaned.dtypes)
df_cleaned['search_ts'] = pd.to_datetime(df_cleaned['search_ts'])
display(df_cleaned.dtypes)

## NAs
display(df_cleaned.isna().sum())
df_cleaned = df_cleaned.dropna()
print('Removed NAs')
display(df_cleaned.shape)

## Duplicates
list_columns = list(df)
df_deduped = df_cleaned.drop_duplicates()
print('Removed duplicates (all columns)')
display(df_deduped.shape)

list_columns.remove('user_ID')
df_deduped = df_deduped.drop_duplicates(list_columns)
print('Removed duplicates (all columns except ID)')
display(df_deduped.shape)

list_columns.remove('search_ts')
#df_deduped_strict = df_deduped.drop_duplicates(list_columns)
#print('Removed duplicates (all columns except ID and time stamp)')
#display(df_deduped_strict.shape)

## Outliers
### Get overview
list_columns_outliers = df.select_dtypes(include=np.number).columns.tolist()

for column_outliers in list_columns_outliers:
    plt.figure()
    sns.boxplot(data=df[column_outliers]).set_title(column_outliers + ' (raw)')

### Set maximum number of days
max_value = 365 # in days

### Days to departure
#### Remove negative values
df_no_outliers = df_deduped[df_deduped['days_to_departure'] >= 0]
print('Days to departure: removed negative values')
display(df_no_outliers.shape)

#### Remove unrealistic values
# df_no_outliers = df_no_outliers[df_no_outliers['days_to_departure'] <= max_value]
# print('Days to departure: removed unrealistic values')
# display(df_no_outliers.shape)

### Trip duration
#### Remove negative values
df_no_outliers = df_no_outliers[df_no_outliers['trip_duration'] >= 0]
print('Trip duration: removed negative values')
display(df_no_outliers.shape)

#### Remove unrealistic values
df_no_outliers = df_no_outliers[df_no_outliers['trip_duration'] <= max_value]
print('Trip duration: removed unrealistic values')
display(df_no_outliers.shape)

### Distance
#### Remove unrealistic values
earth_perimeter = 40000 # in kilometers
df_no_outliers['distance'] = df_no_outliers['distance'] / 1000

df_no_outliers = df_no_outliers[df_no_outliers['distance'] <= earth_perimeter]
print('Distance: removed unrealistic values')
display(df_no_outliers.shape)

### Get cleaned overview
display(df_no_outliers.describe().round(0))

for column_outliers in list_columns_outliers:
    plt.figure()
    sns.boxplot(data=df_no_outliers[column_outliers].values).set_title(column_outliers + ' (cleaned)')

#endregion

# Trends
#region

print('Trends:')

## Prepare data
df_trends = df_no_outliers

df_trends['search_ts'].describe(datetime_is_numeric=True)
df_trends['month'] = pd.DatetimeIndex(df_trends['search_ts']).month
df_trends['day'] = pd.DatetimeIndex(df_trends['search_ts']).day

criteria = [df_trends['day'].between(1, 10), df_trends['day'].between(11, 20), df_trends['day'].between(21, 31)]
categories = ['beginning', 'mid', 'end']

df_trends['day_cat'] = np.select(criteria, categories, 0)

df_trends['weekday'] = df_trends['search_ts'].dt.day_name()

list_alpha_2 = [i.alpha_2 for i in list(pycountry.countries)]
list_name = [i.name for i in list(pycountry.countries)]
dict_countries = dict(zip(list_alpha_2, list_name))
df_trends['user_country_source1'] = df_trends['user_country_source1'].map(dict_countries)
df_trends['user_country_source2'] = df_trends['user_country_source2'].map(dict_countries)

# Get top 5
display(df_trends['user_city_source1'].value_counts().nlargest(5))
display(df_trends['user_country_source1'].value_counts().nlargest(5))
display(df_trends['user_city_source2'].value_counts().nlargest(5))
display(df_trends['user_country_source2'].value_counts().nlargest(5))
display(df_trends['searched_destination'].value_counts().nlargest(5))

# Group by
display(df_trends.groupby(by=['month']).mean())
display(df_trends.groupby(by=['day_cat']).mean())
display(df_trends.groupby(by=['weekday']).mean())

display(df_trends.groupby(by=['month'])['user_city_source1', 'user_country_source1', 'user_city_source2', 'user_country_source2', 'searched_destination'].agg(pd.Series.mode))
display(df_trends.groupby(by=['day_cat'])['user_city_source1', 'user_country_source1', 'user_city_source2', 'user_country_source2', 'searched_destination'].agg(pd.Series.mode))
display(df_trends.groupby(by=['weekday'])['user_city_source1', 'user_country_source1', 'user_city_source2', 'user_country_source2', 'searched_destination'].agg(pd.Series.mode))

#endregion

# Correlations
#region

print('Correlations:')

## Encode categorical columns
df_trends_cat = df_trends.copy()

list_columns_categorical = ['user_city_source1', 'user_country_source1', 'user_city_source2', 'user_country_source2', 'website_language', 'searched_destination', 'day_cat', 'weekday']

for column_categorical in list_columns_categorical:
    df_trends_cat[column_categorical] = df_trends_cat[column_categorical].astype('category').cat.codes

## Set variables
list_variables_independent = ['user_city_source1', 'user_city_country1', 'user_city_source2', 'user_city_country2', 'website_language', 'month', 'day', 'day_cat', 'weekday']
list_variables_dependent = ['days_to_departure', 'trip_duration', 'searched_destination', 'distance']

## Check for normal distribution
for variable in list_variables_dependent:
    print(variable)

    k2, p = stats.normaltest(df_trends_cat[variable])
    alpha = 1e-3
    print("p = {:g}".format(p))

    # Null hypothesis: data is normally distributed
    if p < alpha:
        print("The null hypothesis can be rejected")
    else:
        print("The null hypothesis cannot be rejected")
    print()

    plt.figure()
    sns.histplot(data=df_trends_cat[variable]).set_title(variable)

## Spearman
def correlation_spearman(df):
    r = df.corr(method="spearman").round(2)
    plt.figure(figsize=(10,6))
    heatmap = sns.heatmap(r, vmin=-1, vmax=1, annot=True)
    plt.title("Spearman Correlation")

    return(r)

correlation_spearman(df_trends_cat)

#endregion


# Write-up: http://techn.ology.net/investigating-kasparov-chess-moves-with-scikit-learn/

import chess # https://python-chess.readthedocs.io/en/latest/
import chess.pgn
import re
import csv
import os
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import metrics
import matplotlib.pyplot as plt

def number_of_moves(node):
    i = 0
    while not node.is_end():
        i = i + 1
        next_node = node.variations[0]
        node = next_node
    return i

def get_nth_move(node, n):
    i = 0
    move = ''
    while not node.is_end():
        i = i + 1
        next_node = node.variations[0]
        if i == n:
            move = node.board().san(next_node.move)
            break
        node = next_node
    return move

def board_transform(b):
    fen = b.fen().split(' ')[0].split('/')
    values = {
        'p' : 10,
        'r' : 11,
        'n' : 12,
        'b' : 13,
        'q' : 14,
        'k' : 15,
        'P' : 20,
        'R' : 21,
        'N' : 22,
        'B' : 23,
        'Q' : 24,
        'K' : 25
    }
    transformed = []
    for row in fen:
        for r in list(row):
            if (re.search('^\d+$', r)):
                for i in range(1, int(r) + 1):
                    transformed.append('0')
            else:
                transformed.append(str(values.get(r)))
    return transformed

def get_pgn(path):
    file_list = []
    for fn in os.listdir(path):
        file_list.append(fn)
    return file_list

def create_csv(threshold):
    threshold = threshold * 2
    # Write CSV
    with open('kasparov.csv', 'wb') as csvfile:
        fh = csv.writer(csvfile)
        # Get the files to parse
        path = 'pgn'
        pgn_files = get_pgn(path)
        for pgn_file in pgn_files:
            # Read the game from the given PGN file
            file_name = path + '/' + pgn_file
            #print 'Reading: ' + file_name
            with open(file_name) as pgn:
                game = chess.pgn.read_game(pgn)
            # Decide what player is Kasperov
            if 'Kasparov' in game.headers['White']:
                player = 1
            else:
                player = 0
            # Get the number of game moves
            moves = number_of_moves(game)
            # Restrict to threshold
            if (threshold > 0) and (moves >= threshold):
                moves = threshold
            # Get a new board
            board = chess.Board()
            # For each move...
            for i in range(1, moves + 1):
                move = get_nth_move(game, i)
                # Write the CSV transformed board for the player move
                if (player and (i % 2)) or (not(player) and not(i % 2)):
                    t = board_transform(board)
                    # Prepend the move number
                    t.insert(0, i)
                    # Add the subsequent move
                    t.append(move)
                    fh.writerow(t)
                board.push_san(move) # Make the move

def train():
    data = pd.read_csv('kasparov.csv', header=None)
    X = data.loc[:, 0:list(data)[-2]]
    y = data.loc[:, list(data)[-1]]
    #print type(X), X.shape
    #print type(y), y.shape
    X_train, X_test, y_train, y_test = train_test_split(X, y)
    #print X_train.shape, y_train.shape
    #print X_test.shape, y_test.shape
    return X_train, X_test, y_train, y_test

def decision_tree(X_train, X_test, y_train, y_test):
    decisiontree = DecisionTreeClassifier()
    model = decisiontree.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    return metrics.accuracy_score(y_test, y_pred)

create_csv(0)
X_train, X_test, y_train, y_test = train()

def trial_and_error():

    # K Nearest Neighbors
    from sklearn.neighbors import KNeighborsClassifier
    knn = KNeighborsClassifier(n_neighbors=5)
    knn.fit(X_train, y_train)
    y_pred = knn.predict(X_test)
    print(metrics.accuracy_score(y_test, y_pred)) # 0.12101669195751139 for all moves in game

    # Better k?
    k_range = range(1, 26)
    scores = []
    for k in k_range:
        knn = KNeighborsClassifier(n_neighbors=k)
        knn.fit(X_train, y_train)
        y_pred = knn.predict(X_test)
        scores.append(metrics.accuracy_score(y_test, y_pred))

    plt.plot(k_range, scores)
    plt.xlabel('k')
    plt.ylabel('Accuracy')
    plt.show()

    # Multinomial Naive Bayes
    from sklearn.naive_bayes import MultinomialNB
    nb = MultinomialNB()
    nb.fit(X_train, y_train)
    y_pred = nb.predict(X_test)
    print(metrics.accuracy_score(y_test, y_pred)) # 0.07169954476479515 for all moves in game

    # Support Vector Machine
    from sklearn import svm
    classifier = svm.SVC(gamma=0.001)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    print(metrics.accuracy_score(y_test, y_pred)) # 0.15440060698027314 for all moves in game

    # Decision Tree
    from sklearn.tree import DecisionTreeClassifier
    accuracy = decision_tree(X_train, X_test, y_train, y_test)
    print(accuracy) # 0.17071320182094082 for all moves in game

    create_csv(3)
    X_train, X_test, y_train, y_test = train()
    accuracy = decision_tree(X_train, X_test, y_train, y_test)
    print(accuracy)

    create_csv(6)
    X_train, X_test, y_train, y_test = train()
    accuracy = decision_tree(X_train, X_test, y_train, y_test)
    print(accuracy)

    create_csv(12)
    X_train, X_test, y_train, y_test = train()
    accuracy = decision_tree(X_train, X_test, y_train, y_test)
    print(accuracy)

    k_range = range(1, 15)
    scores = []
    for k in k_range:
        create_csv(k)
        X_train, X_test, y_train, y_test = train()
        accuracy = decision_tree(X_train, X_test, y_train, y_test)
        print(accuracy)
        scores.append(accuracy)

    plt.plot(k_range, scores)
    plt.xlabel('Moves')
    plt.ylabel('Accuracy')
    plt.show()


from brain.ppo_tf import PPO
import numpy as np
# import tensorflow.contrib.eager as tfe
# tfe.enable_eager_execution()
brain = PPO()

actor_hidden = None
critic_hidden = None
while True:
    img, env_state, card_type, card_property = \
        np.random.rand(4, 192, 256, 3).astype(np.uint8), np.random.rand(4,4).astype(np.float), \
        np.random.rand(4, 4).astype(np.int), np.random.rand(4, 8).astype(np.float),
    action, _, actor_hidden = brain.select_action(img, env_state, card_type, card_property, actor_hidden)

    print(action)


import sqlite3

def Create_Lib_Tables(conn):
    conn.execute("drop table book_master")
    conn.execute("""CREATE TABLE book_master (
               book_id text primary key,
               ISBN text not null,
               book_name text not null,
               author text not null,
               genre text not null,
               availability text not null,
               reserve text,
               unique (book_id)
               )""")

    conn.execute("drop table member_master")
    conn.execute("""CREATE TABLE member_master (
           member_id text primary key,
           first_name text not null,
           last_name text not null,
           email text not null,
           unique (member_id)
           )""")

    conn.execute("drop table book_trans")
# For status the valid values are R-Reserved, O-Check-out, I-Check-in, "U"-Unreserve
    conn.execute("""CREATE TABLE book_trans (
               book_id text,
               member_id text,
               checkout_dt date,
               e_return_dt date,
               a_return_dt date,
               status text
               )""")
    conn.commit()

def Upload_Sample_Data(conn):
    conn.execute("INSERT INTO book_master VALUES ('HP19901', '0764598342123', 'Harry Potter and the Philosophers Stone', 'J. K. Rowling', 'Fantasy', 'Y', '')")
    conn.execute("INSERT INTO book_master VALUES ('HP19902', '0764598343321', 'Harry Potter and the Philosophers Stone', 'J. K. Rowling', 'Fantasy', 'Y', '')")
    conn.execute("INSERT INTO book_master VALUES ('PJ19903', '0764598344213', 'Percy Jackson and the Lightning Thief', 'Rick Riordan', 'Greek mythology', 'Y', '')")
    conn.execute("INSERT INTO member_master VALUES ('NK00001', 'John', ' Doe', 'john@email.com')")
    conn.execute("INSERT INTO member_master VALUES ('NK00002', 'Jane', 'Doe', 'jane@email.com')")
    conn.execute("INSERT INTO member_master VALUES ('NK00003', 'Jerry', 'Doe', 'jerry@email.com')")

    conn.commit()

def Delete_Data(conn):
    #delete contents of book trans
    conn.execute("DELETE FROM BOOK_TRANS")
    #delete contents of book master
    conn.execute("DELETE FROM book_master ")
    #delete contents of member master
    conn.execute("DELETE FROM member_master ")
    conn.commit()

def Print_All_Data(cur):
    #execute and print book master table
    cur.execute("SELECT * FROM book_master ")
    print("BOOK MASTER:")
    for row in cur.fetchall():
        print(row)
    print("\n")
    #execute and print member master table
    cur.execute("SELECT * FROM member_master ")
    print("MEMBER MASTER:")
    for row in cur.fetchall():
        print(row)
    print("\n")
    #execute book trans
    cur.execute("SELECT * FROM BOOK_TRANS")
    print("BOOK TRANSACTION:")
    for row in cur.fetchall():
        print(row)
    print("\n")
    print(cur.fetchall())

def Create_DB_Main():
    conn = sqlite3.connect('library.db')
    cur = conn.cursor()
    while True:
        Choice = input("Would you like to\n1. Create Database and Tables\n2. Upload Sample Data\n3. Delete Data\n4. Print Tables\n5. Go Back To Menu\nEnter in Option: ")
        if Choice == "1":
            Answer = input("WARNING. IF YOU USE THIS OPTION ALL THE DATA FROM THE TABLES WILL BE LOST. DO YOU WANT TO CONTINUE? PRESS 'Y' ELSE PRESS 'N':  ")
            Answer = Answer.upper()
            if Answer == "Y":
                Create_Lib_Tables(conn)
                print("Tables Created")
        elif Choice == "2":
            Delete_Data(conn)
            Upload_Sample_Data(conn)
            print("Sample Data Uploaded")
        elif Choice == "3":
            Delete_Data(conn)
            print("Data Deleted")
        elif Choice == "4":
            Print_All_Data(cur)
        elif Choice == "5":
            break
        else:
            print("ERROR! INCORRECT OPTION ENTERED")

    conn.close()


def get(self, url, params={}):
        """
        Issues a GET request against the API, properly formatting the params

        :param url: a string, the url you are requesting
        :param params: a dict, the key-value of all the paramaters needed
                       in the request
        :returns: a dict parsed of the JSON response
        """

        params.update({'api_key': self.api_key})
        try:
            response = requests.get(self.host + url, params=params)
        except RequestException as e:
            response = e.args

        return self.json_parse(response.content)


import math
import os
import discord
from discord.ext import commands

client = commands.Bot(command_prefix = "dpy!")

def round_to_place(number, decimal = 1):
    factor = 10.0**decimal
    return math.trunc(number*factor)/factor

@client.event
async def on_ready():
    await client.change_presence(
        activity = discord.Activity(
            type = discord.ActivityType.listening,
            name = "corridones"
        )
    )

@client.command()
async def pato(ctx):
	new_embed = discord.Embed(
		title = "Un pato",
		description = f"Cuá cuá, {ctx.message.author.mention}, ¡aquí está tu pato!",
		colour = discord.Colour.blue()
	)

	new_embed.set_author(name = "DesarrollaBot.py")
	new_embed.set_image(url = "https://www.random-d.uk/api/randomimg")
	await ctx.send(embed = new_embed)

@client.command()
@commands.has_permissions(administrator = True)
async def repetir(ctx, message):
	await ctx.message.delete()
	await ctx.send(message)

@client.command()
async def redes(ctx):
    await ctx.send(
        """
Twitter: <https://twitter.com/dh_rblx>
Sitio web: <https://desarrolladoreshispanos.com>
GitHub: <https://github.com/Desarrolladores-Hispanos>
Grupo de Roblox: <https://www.roblox.com/groups/9369198/Desarrolladores-Hispanos>
Discord: pues ya estás en el servidor xd
        """
    )

@client.command()
@commands.has_permissions(manage_messages = True)
async def eliminar(ctx, cantidad):
    await ctx.channel.purge(limit = int(cantidad) + 1)

@client.command()
async def convertir_a_robux(ctx, usd):
    usd = float("%.2f" % float(usd)) # definitivamente hay una mejor manera de hacer esto xd
    await ctx.send("USD %s son <:robux:806422545359306752> %s" % (f"{usd:,}", f"{round_to_place(usd/0.0035, 2):,}"))

@client.command()
async def convertir_a_usd(ctx, robux):
    robux = int("%.2f" % int(robux)) # definitivamente hay una mejor manera de hacer esto xd
    await ctx.send("<:robux:806422545359306752> %s son USD $%s" % (f"{robux:,}", f"{round_to_place(robux*0.0035, 2):,}"))

client.run(os.getenv("BOT_TOKEN"))


from scipy.integrate import odeint  # 导入 scipy.integrate 模块
import numpy as np  # 导入 numpy包
import matplotlib.pyplot as plt  # 导入 matplotlib包

def dySIR(y, t, lamda, mu):  # SIR 模型，导数函数
    i, s = y
    di_dt = lamda*s*i - mu*i  # di/dt = lamda*s*i-mu*i
    ds_dt = -lamda*s*i  # ds/dt = -lamda*s*i
    return np.array([di_dt,ds_dt])

# 设置模型参数
number = 1e7  # 总人数
lamda = 0.2  # 日接触率, 患病者每天有效接触的易感者的平均人数
sigma = 2.5  # 传染期接触数
mu = lamda/sigma  # 日治愈率, 每天被治愈的患病者人数占患病者总数的比例
fsig = 1-1/sigma
tEnd = 200  # 预测日期长度
t = np.arange(0.0,tEnd,1)  # (start,stop,step)
i0 = 1e-6  # 患病者比例的初值
s0 = 1-i0  # 易感者比例的初值
Y0 = (i0, s0)  # 微分方程组的初值

print("lamda={}\tmu={}\tsigma={}\t(1-1/sig)={}".format(lamda,mu,sigma,fsig))

ySIR = odeint(dySIR, Y0, t, args=(lamda,mu))  # SIR 模型

# 绘图
#plt.title("Comparison among SI, SIS and SIR models")
plt.xlabel('t')
plt.axis([0, tEnd, -0.1, 1.1])
plt.axhline(y=0,ls="--",c='c')  # 添加水平直线
plt.plot(t, ySIR[:,0], '-r', label='i(t)-SIR')
plt.plot(t, ySIR[:,1], '-b', label='s(t)-SIR')
plt.plot(t, 1-ySIR[:,0]-ySIR[:,1], '-m', label='r(t)-SIR')
plt.legend(loc='best')  # youcans
plt.show()


# %%
from typing import Any, Dict
import lightning.pytorch as pl
import torch
from peft import get_peft_config, get_peft_model, LoraConfig, TaskType, PeftModel
from dataclasses import dataclass
from transformers import AutoConfig, AutoTokenizer
import thesis.src.utils.metrics as metrics
from thesis.src.models.mt5.mt5_module import MT5
from thesis.src.models.xlmr.xlmr_module import XLMRModule
from thesis.src.models.mt5.mt5_module_pcgrad import MT5PCGrad
from thesis.src.models.mbert.mbert_module import MBERTModule
from thesis.src.utils.constants import MODELS
import os

# def int_cast(word):
#     if len(word) > 1:
#         word = word[0]
#     return float(word) if word.isdigit() else -1.0

# def decode_sequences(tokenizer, sequences):
#     sequences[sequences == -100] = tokenizer.pad_token_id
#     return tokenizer.batch_decode(sequences, skip_special_tokens=True, clean_up_tokenization_spaces=True)
# self.__post_init__().

class ModelModule(pl.LightningModule):
    def __init__(
        self,
        model_name,
        tasks,
        batch_name_map_eval,
        batch_name_map_test,
        label2id,
        peft=False,
        peft_checkpoint=False,
        num_accum_batches=1,
        r3f=False,
        r4f=False,
        scale_loss=False,
        retain_grads=False,
    ):
        super().__init__()
        self.save_hyperparameters()

        if model_name == "mt5_pcgrad":
            self.automatic_optimization = False
        path = MODELS[model_name]["model_name"]
        child_module = MODELS[model_name]["module"]

        self.tokenizer = AutoTokenizer.from_pretrained(path)
        self.peft = peft
        self.peft_checkpoint = peft_checkpoint
        self.config = AutoConfig.from_pretrained(path)
        self.config.label2id = label2id
        self.config.tasks = tasks
        self.config.num_accum_batches = num_accum_batches
        self.config.r4f = r4f
        self.config.r3f = r3f
        self.config.scale_loss = scale_loss
        self.config.retain_gradients = retain_grads

        self.tasks = dict(tasks)
        # child_model = child_module(self.config)
        child = child_module(self.config)
        self.child = self.wrap_with_peft(child) if peft_checkpoint else child

        self.batch_name_map_eval = batch_name_map_eval
        self.batch_name_map_test = batch_name_map_test
        self.eval_metrics = metrics.init_metrics(self.batch_name_map_eval)
        self.test_metrics = metrics.init_metrics(self.batch_name_map_test)

        self.validation_step_outputs = []
        self.test_step_outputs = []
        self.predictions = {}

    def forward(self, input_ids, attention_mask=None, labels=None, *args, **kwargs):
        return self.child(
            input_ids=input_ids, attention_mask=attention_mask, labels=labels
        )

    def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None:
        super().on_save_checkpoint(checkpoint)
        if (
            self.peft
            and self.trainer.checkpoint_callback.current_score
            >= self.trainer.checkpoint_callback.best_model_score
            and (self.trainer.local_rank == 0 or self.trainer.local_rank == -1)
        ):
            self.child.model.save_pretrained(
                self.trainer.checkpoint_callback.dirpath + "/lora_ckpt"
            )

    def training_step(self, batch, batch_idx):
        loss = self.child.training_step(batch, batch_idx)

        if loss is not None:
            self.log(
                "train_loss",
                loss,
                on_step=True,
                prog_bar=True,
                logger=True,
                sync_dist=True,
            )
        return loss

    def validation_step(self, batch, batch_idx, dataloader_idx=0):
        predictions, references = self.child.validation_step(
            batch, batch_idx, dataloader_idx
        )
        self.validation_step_outputs.append(
            {
                "dataloader_idx": dataloader_idx,
                "predictions": predictions,
                "references": references,
                "guids": batch["guids"] if batch["task_name"] == "wpr" else [],
            }
        )

    def on_validation_epoch_end(self):
        self.custom_accumulated_log(
            self.validation_step_outputs, self.batch_name_map_eval, self.eval_metrics
        )
        self.validation_step_outputs.clear()

    def test_step(self, batch, batch_idx, dataloader_idx=0):
        predictions, references = self.child.test_step(batch, batch_idx, dataloader_idx)
        self.test_step_outputs.append(
            {
                "dataloader_idx": dataloader_idx,
                "predictions": predictions,
                "references": references,
                "guids": batch["guids"] if batch["task_name"] == "wpr" else [],
            }
        )

    def on_test_epoch_end(self):
        self.custom_accumulated_log(
            self.test_step_outputs,
            self.batch_name_map_test,
            self.test_metrics,
            name_split="test/",
        )
        self.test_step_outputs.clear()

    def predict_step(self, batch, batch_idx, dataloader_idx=0):
        predictions, references = self.child.predict_step(
            batch, batch_idx, dataloader_idx
        )
        if self.batch_name_map_test[dataloader_idx] not in self.predictions.keys():
            self.predictions[self.batch_name_map_test[dataloader_idx]] = {
                "predictions": [],
                "references": [],
                "guids": [],
            }
        self.predictions[self.batch_name_map_test[dataloader_idx]][
            "predictions"
        ] += predictions
        self.predictions[self.batch_name_map_test[dataloader_idx]][
            "references"
        ] += references
        self.predictions[self.batch_name_map_test[dataloader_idx]]["guids"] += (
            batch["guids"] if batch["task_name"] == "wpr" else []
        )

        return predictions

    def configure_optimizers(self):
        if self.peft:
            if not self.peft_checkpoint:
                self.child = self.wrap_with_peft(self.child)

            return torch.optim.Adam(self.parameters(), lr=5e-4)
        return self.child.configure_optimizers()

    def wrap_with_peft(self, child):
        peft_config = LoraConfig(
            task_type=TaskType.SEQ_2_SEQ_LM,
            inference_mode=False,
            r=32,
            lora_alpha=32,
            lora_dropout=0.1,
        )
        device = child.model.device
        child.model = get_peft_model(child.model, peft_config)
        child.model.to(device)
        child.model.print_trainable_parameters()
        return child

    def log_value(self, name, value, sync_dist=True):
        self.log(
            name,
            value,
            on_epoch=True,
            logger=True,
            sync_dist=sync_dist,
            add_dataloader_idx=False,
        )

    def custom_accumulated_log(
        self, step_outputs, batch_name_map, data_metrics, name_split="eval/"
    ):
        outs = {}
        for entry in step_outputs:
            key = entry["dataloader_idx"]
            if key not in outs.keys():
                outs[key] = {
                    "predictions": [],
                    "references": [],
                    "guids": [],
                }
            outs[key]["predictions"] += entry["predictions"]
            outs[key]["references"] += entry["references"]
            outs[key]["guids"] += entry["guids"]

        for dataloader_idx, data in outs.items():
            predictions = torch.tensor(data["predictions"])
            references = torch.tensor(data["references"])
            guids = torch.tensor(data["guids"])
            name = name_split + batch_name_map[dataloader_idx]
            if "wpr" in self.batch_name_map_eval[dataloader_idx]:
                data_metrics[dataloader_idx](
                    predictions.float(), references.long(), guids.cpu().long()
                )
                self.log_value(name + "_ncdg", data_metrics[dataloader_idx])
            elif "ctkfacts" in self.batch_name_map_eval[dataloader_idx]:
                data_metrics[dataloader_idx](predictions, references)
                self.log_value(name + "_f1", data_metrics[dataloader_idx])
            else:
                data_metrics[dataloader_idx](predictions, references)
                self.log_value(name + "_accuracy", data_metrics[dataloader_idx])

        avg = sum([metric.compute() for metric in data_metrics]) / len(batch_name_map)
        self.log_value(name_split + "accumulate", avg, sync_dist=False)


# Definition for singly-linked list.

class ListNode(object):
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution(object):
    def addTwoNumbers(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        def getnum(l):
            num = 0
            p = l
            while p is not None:
                num = num*10 + p.val
                p = p.next
            return num

        num1 = getnum(l1)
        num2 = getnum(l2)
        num3 = num1 + num2
        p = head = None

        for c in str(num3):
            if head is None:
                head = ListNode(int(c))
                p = head
            else:
                p.next = ListNode(int(c))
                p = p.next
        return head


"""

"""

from typing import Iterable, Optional, Any, Tuple
from itertools import tee
import logging

from fennel.core.program import Program
from fennel.tasks.start import StartTask
from fennel.tasks.proxy import ProxyTask
from fennel.tasks.put import PutTask
from fennel.tasks.compute import ComputeTask

def _pairwise(iterable: Iterable[Any]) -> Iterable[Tuple[Any, Any]]:
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    primary, secondary = tee(iterable)
    next(secondary, None)
    return zip(primary, secondary)

def _prime_factorization(num: int) -> Iterable[int]:
    """
    Find prime factorization of non-zero positive number.
    """

    assert num > 0

    step = 2
    factors = []

    while step * step <= num:
        if num % step:
            step += 1
        else:
            num //= step
            factors.append(step)

    if num > 1:
        factors.append(num)

    return sorted(factors)

def _aggregate_factors(factors: Iterable[int],
                       threshold: int
                       ) -> Iterable[int]:
    """
    """

    if len(factors) <= 1:
        return factors

    factors = sorted(factors)

    prod = factors[0] * factors[1]
    if prod > threshold:
        return factors

    return _aggregate_factors([prod] + factors[2:], threshold)

def _target(base: int, mask: int, process: int, index: int) -> int:
    """
    Calculate target in recursive multiplying group.
    """

    sub_mask = (index + 1) * mask
    block = process // base
    offset = (process + sub_mask) % base

    return block + offset

def generate_recursive_multiplying(processes: int,
                                   message_size: int,
                                   aggregate: Optional[int] = None
                                   ) -> Program:
    """
    Generate a factored recursive multiplying schedule.
    """

    assert processes >= 1
    assert _prime_factorization(processes)
    assert message_size >= 0

    program = Program()

    # generate start nodes + proxies
    for process in range(processes):
        program.add_node(f's{process}',
                         StartTask(f's{process}', process))

        program.add_node(f'x_{process}_0',
                         ProxyTask(f'x_{process}_0', process))

        program.add_edge(f's{process}', f'x_{process}_0')

    factorization = _prime_factorization(processes)
    logging.debug('processes %i prime factorization %i',
                  processes, factorization)

    # aggregate schedule to target
    if aggregate:
        factorization = _aggregate_factors(factorization, aggregate)

    mask = 1
    for ridx, factor in enumerate(factorization):
        base = factor * mask

        for process in range(processes):
            # generate proxy
            proxy_name = f'x_{process}_{ridx+1}'
            program.add_node(proxy_name, ProxyTask(proxy_name, process))

            # multicast send
            for index in range(factor - 1):
                # generate put
                put_name = f'p_{process}_{ridx}_{index}'

                target = _target(base, mask, process, index)

                program.add_node(put_name,
                                 PutTask(put_name, process,
                                         target, message_size))

                program.add_edge(put_name, f'c_{target}_{ridx}')
                program.add_edge(f'x_{process}_{ridx}', put_name)

            # generate compute
            compute_name = f'c_{process}_{ridx}'
            program.add_node(compute_name,
                             ComputeTask(compute_name, process, message_size))

            program.add_edge(f'x_{process}_{ridx}', compute_name)
            program.add_edge(compute_name, f'x_{process}_{ridx+1}')

        mask *= factor

    return program


import gym
import torch.optim as optim
import pickle
import matplotlib.pyplot as plt
import random

from dqn_model import DQN
from dqn_learn import OptimizerSpec, dqn_learing
from utils.gym import get_env, get_wrapper_by_name
from utils.schedule import LinearSchedule, ConstantSchedule

BATCH_SIZE = 32
GAMMA = 0.99
REPLAY_BUFFER_SIZE = 1000000
LEARNING_STARTS = 50000
LEARNING_FREQ = 4
FRAME_HISTORY_LEN = 4
TARGER_UPDATE_FREQ = 10000
LEARNING_RATE = 0.00025
ALPHA = 0.95
EPS = 0.01

"""
# deepmind's hyperparameters:
    exploration_schedule = LinearSchedule(1000000, 0.1)
    RMSProp algorithm with minibatches of size 32.
"""

def my_load(path):
    with open(path, 'rb') as f:
        saved_state = pickle.load(f)
    return saved_state
    # load previous data

"""
    if os.path.isfile(save_path):
        saved_state = my_load(save_path + feature_tested)
        Q.load_state_dict(saved_state.state_dict)
        Q_target.load_state_dict(saved_state.state_dict)
        start = saved_state.timestep
        Statistic = saved_state.stats
        mean_episode_reward = Statistic["mean_episode_rewards"][-1][1]
        best_mean_episode_reward = Statistic["best_mean_episode_rewards"][-1][1]
"""

def main(env, num_timesteps):
    # Change the index to select a different game.
    task = benchmark.tasks[3]

    # Run training
    seed = random.randint(0,100)  # Use a seed of zero (you may want to randomize the seed!)
    env = get_env(task, seed)

    def stopping_criterion(env):
        # notice that here t is the number of steps of the wrapped env,
        # which is different from the number of steps in the underlying env
        return get_wrapper_by_name(env, "Monitor").get_total_steps() >= num_timesteps

    optimizer_spec = OptimizerSpec(
        constructor=optim.RMSprop,
        kwargs=dict(lr=LEARNING_RATE, alpha=ALPHA, eps=EPS),
    )

    exploration_schedule = LinearSchedule(1000000, 0.1)

    # empty dict to hold all results
    Stats = {}

    new_lr = 0.001
    new_gamma = 0.999
    exploration_sches = [LinearSchedule(1000000, 0.1), ConstantSchedule(0.05),
                         ConstantSchedule(0.15), LinearSchedule(500000, 0.05)]

    optimizer_spec = OptimizerSpec(
        constructor=optim.RMSprop,
        kwargs=dict(lr=new_lr, alpha=ALPHA, eps=EPS),
    )

    env = get_env(task, seed)
    Stats["lr=0.001, gamma=0.999"] = dqn_learing(
        env=env,
        q_func=DQN,
        optimizer_spec=optimizer_spec,
        exploration=exploration_schedule,
        stopping_criterion=stopping_criterion,
        replay_buffer_size=REPLAY_BUFFER_SIZE,
        batch_size=BATCH_SIZE,
        gamma=new_gamma,
        learning_starts=LEARNING_STARTS,
        learning_freq=LEARNING_FREQ,
        frame_history_len=FRAME_HISTORY_LEN,
        target_update_freq=TARGER_UPDATE_FREQ,
        feature_tested="lr=0.001, gamma=0.999"
    )

    optimizer_spec = OptimizerSpec(
        constructor=optim.RMSprop,
        kwargs=dict(lr=LEARNING_RATE, alpha=ALPHA, eps=EPS),
    )

    env = get_env(task, seed)
    Stats["Default"] = dqn_learing(
        env=env,
        q_func=DQN,
        optimizer_spec=optimizer_spec,
        exploration=exploration_schedule,
        stopping_criterion=stopping_criterion,
        replay_buffer_size=REPLAY_BUFFER_SIZE,
        batch_size=BATCH_SIZE,
        gamma=GAMMA,
        learning_starts=LEARNING_STARTS,
        learning_freq=LEARNING_FREQ,
        frame_history_len=FRAME_HISTORY_LEN,
        target_update_freq=TARGER_UPDATE_FREQ,
        feature_tested=""
    )

    plt.clf()
    plt.xlabel('Timesteps')
    plt.ylabel('Mean Reward (past 100 episodes)')
    num_items = len(Stats["lr=0.001, gamma=0.999"]["mean_episode_rewards"])
    plt.plot(range(num_items), Stats["lr=0.001, gamma=0.999"]["mean_episode_rewards"], label="lr=0.001, gamma=0.999")
    num_items = len(Stats["Default"]["mean_episode_rewards"])
    plt.plot(range(num_items), Stats["Default"]["mean_episode_rewards"], label="Default")
    plt.legend()
    plt.title("Performance")
    plt.savefig('Final-Performance.png')

if __name__ == '__main__':
    # Get Atari games.
    benchmark = gym.benchmark_spec('Atari40M')

    # Change the index to select a different game.
    task = benchmark.tasks[3]

    # Run training
    seed = 0  # datetime.now()  # Use a seed of zero (you may want to randomize the seed!)
    env = get_env(task, seed)

    main(env, task.max_timesteps)


from datetime import datetime
from turtle import distance, position
from helper_methods import *
import pandas as pd
import os
from math import sqrt

db_con = connect_to_db()

# The max distance in a 10x10km region is 10000m * sqrt(2)
#max_distance = 10000 * sqrt(2)

max_distance =   sqrt(145000^2 + 200000^2)*1000 #140x194 km rectangle

# in_data_size    => Tasks input data size
# out_data_size   => Tasks output data size
# mi              => Tasks mi
# position        => Target fog nodes position
# up_bandwidth    => Target fog node upload bandwidth
# down_bandwidth  => Target fog node download bandwidth
# mips            => Target fog node available mips at time t
# sample_point    => Position of the point to test for

@njit
def calc_response_time(in_data_size, out_data_size, mi, position, up_bandwidth, down_bandwidth, mips, sample_point):

    #position = numpy.array([numpy.float64(position[0]), numpy.float64(position[1])])
    distance = calc_dist_njit(position, sample_point)
    distance_factor = 1 - (distance / max_distance)
    up_transfere_time = in_data_size / (up_bandwidth * distance_factor)
    calculation_time = mi / mips
    down_transfere_time = out_data_size / (down_bandwidth * distance_factor)
    return up_transfere_time + calculation_time + down_transfere_time

def calc_strategy_fastest(path_data, locations:list):

    #path_data [path_id, compromised_fog_nodes, events, fog_device_infos, device_stats]
    #fog_device_infos [fog_device_id, downlink_bandwidth, uplink_bandwidth, uplink_latency]
    #event [fog_device_id, event_name, event_type, event_id, timestamp, availableMips, taskID, dataSize, mi, maxMips]
    #device_stats []
    path_id = path_data[0]

    path_coords = get_path_coordinates_from_db(db_con, path_id)

    compromised_fog_nodes = path_data[1]

    events = path_data[2]  #event{ fog_device_id, event_name, event_type, event_id, timestamp, consideredFogNodes, consideredField, ...}

    fog_device_infos = path_data[3]

    device_stats = path_data[4]

    fog_device_positions = select_all_node_positions(db_con)

    #print(fog_device_positions[1792])
    #print(fog_device_positions[23552])
    #print(fog_device_positions[25856])
    #print(fog_device_positions[25857])
    #print(fog_device_positions[16642])
    #print(fog_device_positions[25858])

    #exit()

    total_correctness = 0
    avg_corr = 0

    #for each location check which fog node is the fastest to respond
    current_min = 1000000000 #some high nr

    add_event = None
    remove_event = None

    counted_events = 0

    for event in events:
        fog_device_id = event['fog_device_id']
        selected_fog_node_position = fog_device_positions[fog_device_id]

        if fog_device_id not in compromised_fog_nodes:
            continue

        event_name = event['event_name']
        if (event_name == "add"):
            add_event = event

        if (event_name == "remove"):
            remove_event= event

        if remove_event is None or add_event is None:
            continue

        if add_event['taskId'] != remove_event['taskId']:
            continue

        timestamp = event['timestamp']

        possible_locations = []

        actual_position = get_position_for_timestamp(path_coords, timestamp)

        counted_events += 1

        for location in locations:
            if (calc_dist_in_m(selected_fog_node_position, location) > 10000 * sqrt(2)):
                continue

            chosen_device = get_fastest_comp_fog_node(location, add_event, remove_event, fog_device_infos, device_stats, fog_device_positions, selected_fog_node_position)
            #print(datetime.now()- time2)
            #exit()
            #print("loc: " , location, "     node_pos: ",selected_fog_node_position, "   chosen device: ", chosen_device , "    selected_device: ", fog_device_id )
            if chosen_device == None:
                continue
            if chosen_device == fog_device_id:
                possible_locations.append(location)

        print(possible_locations)
        exit()

        probability = 1/len(possible_locations)

        for poss_location in possible_locations:
            continue

    avg_corr = total_correctness/counted_events

    return total_correctness, avg_corr

def get_fastest_comp_fog_node(location, add_event, remove_event, fog_device_infos, device_stats, fog_device_positions, selected_fog_node_position):
    in_data_size = add_event['dataSize']
    out_data_size = remove_event['dataSize']
    mi = add_event['mi']
    sample_point = numpy.array([numpy.float64(location[0]), numpy.float64(location[1])])
    #base_mips = add_event['maxMips']

    task_id = add_event['taskId']

    base_mips = max(device_stats[task_id].values())
    min_mips = min(device_stats[task_id].values())
    print(base_mips)
    print(min_mips)
    #exit()

    device_stats_keys = device_stats[task_id].keys()

    current_min_rt = 1000000000000
    fastest_node = None

    for i in range(len(fog_device_positions)): #iterate all fog nodes

        mips = base_mips
        position = fog_device_positions[i]
        #position = numpy.array([numpy.float64(position[0]), numpy.float64(position[1])])
        device = fog_device_infos["fog_device_id" == i]
        up_bandwidth = device['uplink_bandwidth']
        down_bandwidth = device['downlink_bandwidth']

        if i in device_stats_keys:
            mips = device_stats[i]

        response_time = calc_response_time(in_data_size, out_data_size, mi, position, up_bandwidth, down_bandwidth, mips, sample_point)

        if response_time < current_min_rt:
            current_min_rt = response_time
            fastest_node = i

    return fastest_node

def main():
    result_file_path = "results/fastest.csv"

    input_json_dir = "input/Strategie_2"

    locations= retrieve_list_from_json("json/locations_points.json")#[nodeid, node_position,locations]

     #clear result file
    f =  open(result_file_path, 'w+')
    f.close

    df = pd.DataFrame(columns=['strategy','rate','iteration','total_correctness','avg_correctness'])

     #iterate input files
    for dirpath, dirs, files in os.walk(input_json_dir):
        for file in files:
            print(dirpath, "     - file: ",file)
            input_file = os.path.join(dirpath,file)
            total_correctness,avg_corr= calc_strategy_fastest(retrieve_data_from_json(input_file),locations)
            file_split = (str(file)).split('_') # e.g. ['output', '3', '100', '1.json']
            strat = file_split[1]
            rate = file_split[2]
            iteration = file_split[3].split('.')[0]

            df = df.append({'strategy':strat, 'rate':rate, 'iteration':iteration, 'total_correctness':total_correctness,'avg_correctness':avg_corr}, ignore_index=True)

    return

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 24 20:12:22 2022

@author: terramotu
exercice 4 du TP sur les signaux
"""
import os, time, sys
from signal import SIGKILL

N = 5
pid = os.fork()

#Boucle du processus fils
if pid==0:
    while True:
        time.sleep(1)
        print("Je suis le fils qui est en train de boucler")
    #Le sys exit est inutile la sortie se fait par un signal mais je trouve ça plus propre...
    sys.exit(0)

#Boucle du processus père
else:
    for i in range(N):
        time.sleep(1)
        print("Et moi je suis le père qui boucle N fois")
    #Envoit le signal "sigkill" au PID
    os.kill(pid, SIGKILL)


# 1436. 旅行终点站

List = list

class Solution:
    def destCity(self, paths: List[List[str]]) -> str:
        l = set()
        r = set()
        for path in paths:
            l.add(path[0])
            r.add(path[1])
        res = r - l
        return res.pop()

s = Solution()
print(s.destCity([["B","C"],["D","B"],["C","A"]]))


import pandas as pd
from patsy import dmatrices
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt

df = pd.read_excel('D:/desktop/content ML/bicycleDataSet.xlsx', header=0, infer_datetime_format=True, parse_dates=[0], index_col=[0])
#df = pd.read_excel(open('C:/Users/Admin/Desktop/content ML/bicycleDataSet.xlsx', 'rb'))

ds = df.index.to_series()
df['MONTH'] = ds.dt.month
df['DAY_OF_WEEK'] = ds.dt.dayofweek
df['DAY'] = ds.dt.day

mask = np.random.rand(len(df)) < 0.8
df_train = df[mask]
df_test = df[~mask]
print('Training data set length='+str(len(df_train)))
print('Testing data set length='+str(len(df_test)))

df_trainip=df_train.iloc[:,[2,3,5,6,7]]
df_trainop=df_train.iloc[:,4]

df_testip=df_test.iloc[:,[2,3,5,6,7]]
df_testop=df_test.iloc[:,4]
#df_trainip=np.array(df_trainip)
#df_trainop=np.array(df_trainop)
#df_trainop=df_trainop.astype('float')
#df_trainip=df_trainip.astype('float')

poisson_training_results = sm.GLM(df_trainop, df_trainip, family=sm.families.Poisson()).fit()

print(poisson_training_results.summary())

#more concern about lambda value
print(poisson_training_results.mu)
print(len(poisson_training_results.mu))

#will find the value fo alpha using OLS regression

import statsmodels.formula.api as smf

#add lambda value to dataset.
df_train['BB_LAMBDA'] = poisson_training_results.mu

#add a derived column AUX_OLS_DEP
df_train['AUX_OLS_DEP'] = df_train.apply(lambda x: ((x['Brooklyn Bridge'] - x['BB_LAMBDA'])**2 - x['Brooklyn Bridge']) / x['BB_LAMBDA'], axis=1)

ols_expr = """AUX_OLS_DEP ~ BB_LAMBDA - 1"""

aux_olsr_results = smf.ols(ols_expr, df_train).fit()

print(aux_olsr_results.params)

aux_olsr_results.tvalues

nb2_training_results = sm.GLM(df_trainop, df_trainip,family=sm.families.NegativeBinomial(alpha=aux_olsr_results.params[0])).fit()
print(nb2_training_results.summary())

nb2_predictions = nb2_training_results.get_prediction(df_testip)
predictions_summary_frame = nb2_predictions.summary_frame()
print(predictions_summary_frame)

predicted_counts=predictions_summary_frame['mean']
actual_counts = df_testop
fig = plt.figure()
fig.suptitle('Predicted versus actual bicyclist counts on the Brooklyn bridge')
predicted, = plt.plot(df_testip.index, predicted_counts, 'go-', label='Predicted counts')
actual, = plt.plot(df_testip.index, actual_counts, 'ro-', label='Actual counts')
plt.legend(handles=[predicted, actual])
plt.show()