import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn import preprocessing

class IrisDF:
    def __init__(self, normalize=True, test_size=0.3):
        iris = datasets.load_iris()
        X = None
        if normalize:
            X = preprocessing.normalize(iris.data)
        else:
            X = iris.data
        y = self.encode(iris.target)

        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
            X, y, test_size=test_size
        )

    def encode(self, data):
        # encodes the target label y so it's compatible with nn output layer
        y = []
        for i in data:
            if i == 0:
                y.append([0, 0, 1])
            elif i == 1:
                y.append([0, 1, 0])
            else:
                y.append([1, 0, 0])
        return np.array(y)

class IrisNN:
    def __init__(self, gen_randomly=False):
        # define Hiperparameters
        self.input_size = 4
        self.hidden_size = 6
        self.output_size = 3

        if gen_randomly:
            # init weights with random values
            self.W1 = np.random.rand(self.input_size, self.hidden_size)
            self.W2 = np.random.rand(self.hidden_size, self.output_size)
            # init biases with random values
            self.b1 = np.random.rand(self.hidden_size)
            self.b2 = np.random.rand(self.output_size)

    def sigmoid(self, z):
        return 1 / (1 + np.exp(-z))

    # https://stackoverflow.com/questions/34968722/how-to-implement-the-softmax-function-in-python
    def softmax(self, z):
        assert len(z.shape) == 2
        s = np.max(z, axis=1)
        s = s[:, np.newaxis]
        e_x = np.exp(z - s)
        div = np.sum(e_x, axis=1)
        div = div[:, np.newaxis]
        return e_x / div

    def forward(self, X):
        # Propagate inputs through the network
        z2 = np.dot(X, self.W1) + self.b1
        a2 = self.sigmoid(z2)
        z3 = np.dot(a2, self.W2) + self.b2
        self.y_hat = self.softmax(z3)

    def compute_mse(self, X_train, y_train):
        # mse_sum = 0
        # for i, j in zip(y, self.y_hat):
        #     mse_sum += sum((j - i)**2)
        # return mse_sum / float(len(y))
        self.forward(X_train)
        return sum(sum((y_train - self.y_hat) ** 2)) / len(y_train)

    def compute_accuracy(self, X_test, y_test):
        self.forward(X_test)
        num_correct = 0
        for i, j in zip(y_test, self.y_hat):
            if i.argmax() == j.argmax():
                num_correct += 1
        return float(num_correct) / len(y_test)

    def merge_params(self):
        # used in PSO as particle position
        return np.concatenate(
            [self.W1.flatten(), self.b1, self.W2.flatten(), self.b2], axis=0
        )

    def unpack_params(self, data):
        offset = self.input_size * self.hidden_size
        self.W1 = data[:offset].reshape(self.input_size, self.hidden_size)
        self.b1 = data[offset : offset + self.hidden_size]
        offset += self.hidden_size
        self.W2 = data[offset : offset + self.hidden_size * self.output_size].reshape(
            self.hidden_size, self.output_size
        )
        offset += self.hidden_size * self.output_size
        self.b2 = data[offset:]


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import subprocess
import os
import sys
import logging
from collections import defaultdict as dd

logging.basicConfig(filename='extracter.log', level=logging.DEBUG, format='%(asctime)s\t%(levelname)s\t%(message)s')

# todo: set aligner path and outputdir
aligner_tool = '/home/aiden/Downloads/align-all-170804E/align.sh'
align_out_dir = '/home/aiden/Documents/toyota/ttdc/alignout/'

def pair_off(filelist):
    filepairs = dd(list)
    for file in filelist:
        filepairs[file.header].append(file)

    for k, v in filepairs.items():
        if len(v) != 2:
            sys.stderr.write(k)
        else:
            if v[0].lang == 'ja':
                print(v[0].orig, v[0].fullpath)
                align = run_aligner(k, v[0], v[1])
            else:
                align = run_aligner(k, v[1], v[0])

            if align == b'':
                sys.stderr.write(k + '\n')
            else:
                logging.warning('Align failed with files header {0}'.format(k))
                continue

def run_aligner(header, jafile, enfile):
    align_output = subprocess.check_output([
        aligner_tool,
        'ja-en',
        jafile.fullpath,
        enfile.fullpath,
        align_out_dir + header + '.align.out',
        '--db'
    ])

    return align_output

def prepare_dir(filedir):
    filelist = os.listdir(filedir)

    pairlist = []
    unklist = []
    for f in filelist:
        if f.endswith('.txt'):
            if (f.startswith('w') or f.startswith('e')):
                pairlist.append(FileProp(filedir + f))
            else:
                unklist.append(f)

    if len(unklist) != 0:
        sys.stderr.write('Logging files with Unknown Headers')
        for unk in unklist:
            logging.debug('Unknown format file: {0}', unk)
    return pairlist

class FileProp:
    def __init__(self, file):
        fname, flang = self.get_name(file)

        self.orig = file
        self.fullpath = os.path.abspath(file)
        self.header = fname

        if flang == 'w':
            self.lang = 'ja'
        elif flang == 'e':
            self.lang = 'en'
        else:
            self.lang = 'unk'

    @staticmethod
    def get_name(filename):
        filepath = filename.split('.')
        fullname = '.'.join(filepath[:-1])
        nameparts = fullname.split('_')
        lang = nameparts[0]
        name = '_'.join(nameparts[1:])

        return name, lang

if __name__ == "__main__":
    location = sys.argv[1]
    filelist = prepare_dir(location)
    pair_off(filelist)


import grpc

from auth.src.app.services.user_service import UserService
from integration.messages import user_pb2_grpc, user_pb2
from app_grpc import app_grpc

class UserServices(user_pb2_grpc.UserServicer):
    def GetUser(self, request, context):
        if request.id:
            with app_grpc.app_context():
                roles = UserService().get_user_grpc(request.id)
                if roles:
                    return user_pb2.RoleList(
                        role=roles
                    )
                message = 'User ID not found'
                context.set_details(message)
                context.set_code(grpc.StatusCode.NOT_FOUND)
                return user_pb2.RoleList()
        msg = 'Must pass ID'
        context.set_details(msg)
        context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
        return user_pb2.RoleList()


from __future__ import absolute_import

import logging
from pandas.tseries.offsets import BDay
from itertools import chain
from apache_beam.io.gcp.internal.clients import bigquery

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from datetime import datetime, date
from  .marketstats_utils import is_above_52wk,get_prices,MarketBreadthCombineFn, get_all_stocks, is_below_52wk,\
                            combine_movers,get_prices2, get_vix, ParseNonManufacturingPMI, get_all_us_stocks2,\
                            get_all_prices_for_date, InnerJoinerFn, create_bigquery_ppln,\
                            ParseManufacturingPMI,get_economic_calendar, get_equity_putcall_ratio,\
                            get_cftc_spfutures, create_bigquery_ppln_cftc, get_market_momentum, \
                            get_senate_disclosures, create_bigquery_manufpmi_bq, create_bigquery_nonmanuf_pmi_bq,\
                            get_sector_rotation_indicator, get_latest_fed_fund_rates

from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail, Email, Personalization
from functools import reduce

class MarketStatsCombineFn(beam.CombineFn):
    def create_accumulator(self):
        return []

    def add_input(self, sum_count, input_data):
        holder = sum_count
        logging.info('Adding:{}'.format(input_data))
        holder.append(input_data)
        return holder

    def merge_accumulators(self, accumulators):
        return chain(*accumulators)

    def extract_output(self, sum_count):
        all_data = sum_count
        sorted_els = sorted(all_data, key=lambda t: t[0])
        mapped = list(map (lambda tpl: tpl[1], sorted_els))

        stringified = list(map(lambda x: '{}-{}-{}'.format(x['AS_OF_DATE'],
                                                           x['LABEL'],
                                                           x['VALUE']), mapped))

        logging.info('MAPPED IS :{}'.format(mapped))
        return stringified

class MarketStatsSinkCombineFn(beam.CombineFn):
    def create_accumulator(self):
        return []

    def add_input(self, sum_count, input_data):
        holder = sum_count
        holder.append(input_data)
        return holder

    def merge_accumulators(self, accumulators):
        return chain(*accumulators)

    def extract_output(self, all_data):
        return [i for i in all_data]

class EmailSender(beam.DoFn):
    def __init__(self, recipients, key):
        self.recipients = recipients.split(',')
        self.key = key

    def _build_personalization(self, recipients):
        personalizations = []
        for recipient in recipients:
            logging.info('Adding personalization for {}'.format(recipient))
            person1 = Personalization()
            person1.add_to(Email(recipient))
            personalizations.append(person1)
        return personalizations

    def process(self, element):
        logging.info('Attepmting to send emamil to:{}, using key:{}'.format(self.recipients, self.key))
        template = "{}"
        content = template.format(element)
        print('Sending \n {}'.format(content))
        message = Mail(
            from_email='gcp_cloud_mm@outlook.com',
            to_emails=self.recipients,
            subject='Market Stats',
            html_content=content)

        personalizations = self._build_personalization(self.recipients)
        for pers in personalizations:
            message.add_personalization(pers)

        sg = SendGridAPIClient(self.key)

        response = sg.send(message)
        print(response.status_code, response.body, response.headers)

class XyzOptions(PipelineOptions):

    @classmethod
    def _add_argparse_args(cls, parser):
        parser.add_argument('--key')
        parser.add_argument('--sendgridkey')
        parser.add_argument('--recipients', default='mmistroni@gmail.com')

def run_pmi(p):
    return (p | 'startstart' >> beam.Create(['20210101'])
                    | 'pmi' >>   beam.ParDo(ParseNonManufacturingPMI())
                    | 'remap  pmi' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'NON-MANUFACTURING-PMI', 'VALUE' : d['Last']})
            )

def run_putcall_ratio(p):
    return (p | 'start putcall ratio' >> beam.Create(['20210101'])
            | 'putcall' >> beam.Map(lambda d: get_equity_putcall_ratio())
            | 'remap pcratio' >> beam.Map(
                lambda d: {'AS_OF_DATE': date.today().strftime('%Y-%m-%d'), 'LABEL': 'EQUITY_PUTCALL_RATIO', 'VALUE': str(d)})
            )

def run_manufacturing_pmi(p):
    return (p | 'startstartnpmi' >> beam.Create(['20210101'])
                    | 'manifpmi' >>   beam.ParDo(ParseManufacturingPMI())
                    | 'manufremap  pmi' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'MANUFACTURING-PMI', 'VALUE' : d['Last']})
            )

def run_economic_calendar(p, key):
    return (p | 'startcal' >> beam.Create(['20210101'])
                    | 'econcalendar' >>   beam.FlatMap(lambda d: get_economic_calendar(key))
                    | 'reMapping' >> beam.Map(lambda d: {'AS_OF_DATE' : d['date'],
                                                         'LABEL' : d['event'],
                                                         'VALUE' : f"Previous:{d['previous']},Estimate:{d['estimate']},Actual:{d.get('actual') or ''}"
                                                         }
                                              )

            )

def run_vix(p, key):
    return (p | 'start run_vix' >> beam.Create(['20210101'])
                    | 'vix' >>   beam.Map(lambda d:  get_vix(key))
                    | 'remap vix' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'VIX', 'VALUE' : str(d)})
            )

def run_senate_disclosures(p, key):
    return (p | 'start run_sd' >> beam.Create(['20210101'])
              | 'run sendisclos' >> beam.FlatMap(lambda d : get_senate_disclosures(key))
            )

def run_fed_fund_rates(p):
    return (p | 'start run_ffr' >> beam.Create(['20210101'])
              | 'run ffrates' >> beam.Map(lambda d : get_latest_fed_fund_rates())
              | 'remap fr' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'FED_FUND_RATES', 'VALUE' : str(d)})
            )

def run_market_momentum(p, key):
    return (p | 'start run_mm' >> beam.Create(['20210101'])
                    | 'mm' >>   beam.Map(lambda d:  get_market_momentum(key))
                    | 'remap mm' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'MARKET_MOMENTUM', 'VALUE' : str(d)})
            )

def run_growth_vs_value(p, key):
    return (p | 'start run_gv' >> beam.Create(['20210101'])
                    | 'gv' >>   beam.Map(lambda d: get_sector_rotation_indicator (key))
                    | 'remap mmgv' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'SECTOR ROTATION(GROWTH/VALUE)', 'VALUE' : str(d)})
            )

def run_cftc_spfutures(p, key):
    return (p | 'start_cftc' >> beam.Create(['20210101'])
                    | 'sptufutres' >>   beam.Map(lambda d:  get_cftc_spfutures(key))
                    | 'remap cftcspfutures' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'), 'LABEL' : 'CFTC-SPFUTURES', 'VALUE' : str(d)})
            )

def run_exchange_pipeline(p, key, exchange):
    all_us_stocks = list(map(lambda t: (t, {}), get_all_us_stocks2(key, exchange)))
    asOfDate = (date.today() - BDay(1)).date()
    prevDate = (asOfDate - BDay(1)).date()

    dt = get_all_prices_for_date(key, asOfDate.strftime('%Y-%m-%d'))
    ydt = get_all_prices_for_date(key, prevDate.strftime('%Y-%m-%d'))

    filtered = [(d['symbol'], d)  for d in dt]
    y_filtered = [(d['symbol'], {'prevClose': d['close']}) for d in ydt]

    tmp = [tpl[0] for tpl in all_us_stocks]
    fallus = [tpl for tpl in filtered if tpl[0] in tmp]
    yfallus = [tpl for tpl in y_filtered if tpl[0] in tmp]

    pcoll1 = p | f'Create coll1={exchange}' >> beam.Create(all_us_stocks)
    pcoll2 = p | f'Create coll2={exchange}' >> beam.Create(fallus)
    pcoll3 = p | f'Crete ydaycoll={exchange}' >> beam.Create(yfallus)

    pcollStocks = pcoll2 | f'Joining y{exchange}' >> beam.ParDo(InnerJoinerFn(),
                                                     right_list=beam.pvalue.AsIter(pcoll3))

    return  (
                    pcoll1
                    | f'InnerJoiner: JoinValues {exchange}' >> beam.ParDo(InnerJoinerFn(),
                                                              right_list=beam.pvalue.AsIter(pcollStocks))
                    | f'Map to flat tpl {exchange}' >> beam.Map(lambda tpl: (tpl[0], tpl[1]['close'], tpl[1]['close'] - tpl[1]['prevClose']))
                    | f'Combine MarketBreadth Statistics {exchange}' >> beam.CombineGlobally(MarketBreadthCombineFn())
                    | f'mapping {exchange}' >> beam.Map(lambda d: {'AS_OF_DATE' : date.today().strftime('%Y-%m-%d'),
                                                        'LABEL' : '{}_{}'.format(exchange.upper(), d[0:d.find(':')]),
                                                       'VALUE' : d[d.rfind(':')+1:]})

            )

def run_prev_dates_statistics(p) :
    # Need to amend the query to order by asofdate sc
    nysebqp = ( create_bigquery_ppln(p, 'NEW YORK STOCK EXCHANGE_MARKET BREADTH')
               #| 'map to tpl2' >> beam.Map(lambda d: ( d['AS_OF_DATE'], d['LABEL'], d['VALUE'] ))
    )

    return nysebqp

def run_prev_dates_statistics_cftc(p):
    # Need to amend the query to order by asofdate sc
    cftc_ppln = (create_bigquery_ppln_cftc(p)
               )

    return cftc_ppln

def run_prev_dates_statistics_manuf_pmi(p):
    # Need to amend the query to order by asofdate sc
    pmi_ppln = (create_bigquery_manufpmi_bq(p)
               )

    return pmi_ppln

def run_prev_dates_statistics_non_manuf_pmi(p):
    # Need to amend the query to order by asofdate sc
    npmi_ppln = (create_bigquery_nonmanuf_pmi_bq(p)
               )

    return npmi_ppln

def run_prev_dates_statistics_cftc(p):
    # Need to amend the query to order by asofdate sc
    cftc_ppln = (create_bigquery_ppln_cftc(p)
               )

    return cftc_ppln

def write_all_to_sink(results_to_write, sink):
    to_tpl = tuple(results_to_write)
    return (
            to_tpl
            | 'FlattenCombine all sink' >> beam.Flatten()
            | 'write all to sink' >> sink

    )

def run(argv=None, save_main_session=True):
    """Main entry point; defines and runs the wordcount pipeline."""

    # We use the save_main_session option because one or more DoFn's in this
    # workflow rely on global context (e.g., a module imported at module level).

    # Check  this https://medium.datadriveninvestor.com/markets-is-a-correction-coming-aa609fba3e34

    pipeline_options = XyzOptions()
    pipeline_options.view_as(SetupOptions).save_main_session = save_main_session
    with beam.Pipeline(options=pipeline_options) as p:

        iexapi_key = pipeline_options.key
        logging.info(pipeline_options.get_all_options())
        current_dt = datetime.now().strftime('%Y%m%d-%H%M')

        destination = 'gs://mm_dataflow_bucket/outputs/shareloader/{}_run_{}.csv'

        logging.info('====== Destination is :{}'.format(destination))
        logging.info('SendgridKey=={}'.format(pipeline_options.sendgridkey))

        bq_sink = beam.io.WriteToBigQuery(
            bigquery.TableReference(
                projectId="datascience-projects",
                datasetId='gcp_shareloader',
                tableId='market_stats'),
            schema='AS_OF_DATE:STRING,LABEL:STRING,VALUE:STRING',
            write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
            create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED)

        run_weekday = date.today().weekday()

        logging.info('Run pmi')

        pmi_res = run_pmi(p)
        manuf_pmi_res = run_manufacturing_pmi(p)

        if date.today().weekday() == 2 and date.today().day() < 15:
            # We need to store it only once a month
            logging.info(*'Running Market Stats')
            pmi_res | 'WritinNG PMI TO SINK' >> bq_sink
            manuf_pmi_res | 'writing non manuf pmi to sink' >> bq_sink

        if run_weekday == 5:
            logging.info(f'Weekday for rundate is {run_weekday}')
            cftc = run_cftc_spfutures(p, iexapi_key)
            cftc | 'cftc to sink' >> bq_sink

        vix_res = run_vix(p, iexapi_key)

        mmomentum_res = run_market_momentum(p, iexapi_key)

        growth_vs_val_res = run_growth_vs_value(p, iexapi_key)

        senate_disc = run_senate_disclosures(p, iexapi_key)

        logging.info('Run NYSE..')
        nyse = run_exchange_pipeline(p, iexapi_key, "New York Stock Exchange")
        logging.info('Run Nasdaq..')
        nasdaq = run_exchange_pipeline(p, iexapi_key, "NASDAQ Global Select")

        equity_pcratio = run_putcall_ratio(p)

        fed_funds = run_fed_fund_rates(p)

        econ_calendar = run_economic_calendar(p, iexapi_key)

        staticStart = (p | 'Create static start' >> beam.Create(
            [dict(AS_OF_DATE='------- ', LABEL=' THIS WEEK ECONOMIC CALENDAR', VALUE='--------')])
                   )

        static1 = (p |'Create static1' >>  beam.Create([dict(AS_OF_DATE='------- ', LABEL=' TODAYS PERFORMANCE', VALUE='--------')])
                 )

        static = (p | 'Create static 2' >> beam.Create([dict(AS_OF_DATE='------- ', LABEL=' LAST 5 DAYS PERFORMANCE', VALUE='--------')])
                 )
        statistics = run_prev_dates_statistics(p)

        cftc_historical = run_prev_dates_statistics_cftc(p)

        pmi_hist = run_prev_dates_statistics_manuf_pmi(p)

        non_pmi_hist = run_prev_dates_statistics_non_manuf_pmi(p)

        staticStart_key = staticStart | 'Add -2' >> beam.Map(lambda d: (-2, d))
        econCalendarKey = econ_calendar | 'Add -1' >> beam.Map(lambda d: (-1, d))
        static1_key = static1 | 'Add 0' >> beam.Map(lambda d: (0, d))
        pmi_key = pmi_res | 'Add 1' >> beam.Map(lambda d: (1, d))
        manuf_pmi_key = manuf_pmi_res | 'Add 2' >> beam.Map(lambda d: (2, d))

        vix_key = vix_res | 'Add 3' >> beam.Map(lambda d: (3, d))
        nyse_key = nyse | 'Add 4' >> beam.Map(lambda d: (4, d))
        nasdaq_key = nasdaq | 'Add 5' >> beam.Map(lambda d: (5, d))
        epcratio_key = equity_pcratio | 'Add 6' >> beam.Map(lambda d: (6, d))
        mm_key = mmomentum_res | 'Add mm' >> beam.Map(lambda d: (7, d))
        sd_key = senate_disc | 'Add sd' >> beam.Map(lambda d: (8, d))
        growth_vs_val_key = growth_vs_val_res | 'Add 14' >> beam.Map(lambda d: (9, d))
        fed_funds_key = fed_funds | 'Add ff' >> beam.Map(lambda d: (10, d))

        static_key = static | 'Add 10' >> beam.Map(lambda d: (15, d))
        stats_key = statistics | 'Add 11' >> beam.Map(lambda d: (16, d))
        cftc_key = cftc_historical | 'Add 12' >> beam.Map(lambda d: (17, d))

        # we need a global combiner to write to sink

        pmi_hist_key = pmi_hist | 'Add 20' >> beam.Map(lambda d: (20, d))

        non_manuf_pmi_hist_key = non_pmi_hist | 'Add 30' >> beam.Map(lambda d: (30, d))

        final = (
                (staticStart_key, econCalendarKey, static1_key, pmi_key,
                    manuf_pmi_key, nyse_key, nasdaq_key,  epcratio_key, mm_key, cftc_key,  vix_key, sd_key, growth_vs_val_key,
                        fed_funds_key,
                        static_key, stats_key,
                        pmi_hist_key, non_manuf_pmi_hist_key,
                        )
                | 'FlattenCombine all' >> beam.Flatten()
                | ' do A PARDO combner:' >> beam.CombineGlobally(MarketStatsCombineFn())
                | ' FlatMapping' >> beam.FlatMap(lambda x: x)
                | 'Combine' >> beam.CombineGlobally(lambda x: '

'.join(x))
                | 'SendEmail' >> beam.ParDo(EmailSender(pipeline_options.recipients, pipeline_options.sendgridkey))

        )

        final_sink_results = [
                 vix_res, mmomentum_res, growth_vs_val_res, nyse,
                 nasdaq, equity_pcratio, fed_funds,
                ]

        # Writing everything to sink
        logging.info('Writing all to sink')

        destination = 'gs://mm_dataflow_bucket/outputs/marketstats_{}'.format(
            date.today().strftime('%Y-%m-%d %H:%M'))

        tmp_sink = beam.io.WriteToText(destination, num_shards=1)

        write_all_to_sink(final_sink_results, bq_sink)

if __name__ == '__main__':
    logging.getLogger().setLevel(logging.INFO)
    run()


from threading import Condition
import time
import uuid

import pytest

import rsb
from rsb import Event, EventId
import rsb.eventprocessing
from rsb.eventprocessing import FullyParallelEventReceivingStrategy
from rsb.filter import RecordingFalseFilter, RecordingTrueFilter

class TestScopeDispatcher:

    def test_sinks(self):
        dispatcher = rsb.eventprocessing.ScopeDispatcher()
        dispatcher.add_sink(rsb.Scope('/foo'), 1)
        dispatcher.add_sink(rsb.Scope('/foo'), 2)
        dispatcher.add_sink(rsb.Scope('/bar'), 3)
        assert set(dispatcher.sinks) == {1, 2, 3}

    def test_matching_sinks(self):
        dispatcher = rsb.eventprocessing.ScopeDispatcher()
        dispatcher.add_sink(rsb.Scope('/foo'), 1)
        dispatcher.add_sink(rsb.Scope('/foo'), 2)
        dispatcher.add_sink(rsb.Scope('/bar'), 3)

        def check(scope, expected):
            assert set(
                dispatcher.matching_sinks(rsb.Scope(scope))) == set(expected)
        check("/", ())
        check("/foo", (1, 2))
        check("/foo/baz", (1, 2))
        check("/bar", (3,))
        check("/bar/fez", (3,))

class TestParallelEventReceivingStrategy:

    def test_matching_process(self):
        ep = rsb.eventprocessing.ParallelEventReceivingStrategy(5)

        mc1_cond = Condition()
        matching_calls1 = []
        mc2_cond = Condition()
        matching_calls2 = []

        def matching_action1(event):
            with mc1_cond:
                matching_calls1.append(event)
                mc1_cond.notifyAll()

        def matching_action2(event):
            with mc2_cond:
                matching_calls2.append(event)
                mc2_cond.notifyAll()

        matching_recording_filter_1 = RecordingTrueFilter()
        matching_recording_filter_2 = RecordingTrueFilter()
        ep.add_filter(matching_recording_filter_1)
        ep.add_filter(matching_recording_filter_2)
        ep.add_handler(matching_action1, wait=True)
        ep.add_handler(matching_action2, wait=True)

        event1 = Event(EventId(uuid.uuid4(), 0))
        event2 = Event(EventId(uuid.uuid4(), 1))

        ep.handle(event1)
        ep.handle(event2)

        # both filters must have been called
        with matching_recording_filter_1.condition:
            while len(matching_recording_filter_1.events) < 4:
                matching_recording_filter_1.condition.wait()

            assert len(matching_recording_filter_1.events) == 4
            assert event1 in matching_recording_filter_1.events
            assert event2 in matching_recording_filter_1.events

        with matching_recording_filter_2.condition:
            while len(matching_recording_filter_2.events) < 4:
                matching_recording_filter_2.condition.wait()

            assert len(matching_recording_filter_2.events) == 4
            assert event1 in matching_recording_filter_2.events
            assert event2 in matching_recording_filter_2.events

        # both actions must have been called
        with mc1_cond:
            while len(matching_calls1) < 2:
                mc1_cond.wait()
            assert len(matching_calls1) == 2
            assert event1 in matching_calls1
            assert event2 in matching_calls1

        with mc2_cond:
            while len(matching_calls2) < 2:
                mc2_cond.wait()
            assert len(matching_calls2) == 2
            assert event1 in matching_calls2
            assert event2 in matching_calls2

        ep.remove_filter(matching_recording_filter_2)
        ep.remove_filter(matching_recording_filter_1)

    def test_not_matching_process(self):

        ep = rsb.eventprocessing.ParallelEventReceivingStrategy(5)

        no_matching_calls = []

        def no_matching_action(event):
            no_matching_calls.append(event)

        no_match_recording_filter = RecordingFalseFilter()
        ep.add_filter(no_match_recording_filter)
        ep.add_handler(no_matching_action, wait=True)

        event1 = Event(EventId(uuid.uuid4(), 0))
        event2 = Event(EventId(uuid.uuid4(), 1))
        event3 = Event(EventId(uuid.uuid4(), 2))

        ep.handle(event1)
        ep.handle(event2)
        ep.handle(event3)

        # no Match listener must not have been called
        with no_match_recording_filter.condition:
            while len(no_match_recording_filter.events) < 3:
                no_match_recording_filter.condition.wait()
            assert len(no_match_recording_filter.events) == 3
            assert event1 in no_match_recording_filter.events
            assert event2 in no_match_recording_filter.events
            assert event3 in no_match_recording_filter.events

        assert len(no_matching_calls) == 0

        ep.remove_filter(no_match_recording_filter)

    def test_add_remove(self):
        for size in range(2, 10):
            ep = rsb.eventprocessing.ParallelEventReceivingStrategy(size)

            def h1(e):
                return e

            def h2(e):
                return e

            ep.add_handler(h1, wait=True)
            ep.add_handler(h2, wait=True)
            ep.add_handler(h1, wait=True)

            ep.handle(Event(EventId(uuid.uuid4(), 0)))
            ep.handle(Event(EventId(uuid.uuid4(), 1)))
            ep.handle(Event(EventId(uuid.uuid4(), 2)))

            ep.remove_handler(h1, wait=True)
            ep.remove_handler(h2, wait=True)
            ep.remove_handler(h1, wait=True)

class MockConnector:
    def activate(self):
        pass

    def deactivate(self):
        pass

    def push(self, event):
        pass

    def filter_notify(self, filter_, action):
        pass

    def set_observer_action(self, action):
        pass

# TODO(jmoringe): could be useful in all tests for active objects
class ActivateCountingMockConnector(MockConnector):
    def __init__(self):
        self.activations = 0
        self.deactivations = 0

    def activate(self):
        self.activations += 1

    def deactivate(self):
        self.deactivations += 1

    def expect(self, activations, deactivations):
        assert activations == self.activations
        assert deactivations == self.deactivations

class TestOutRouteConfigurator:

    def test_activation(self):
        connector = ActivateCountingMockConnector()
        configurator = rsb.eventprocessing.OutRouteConfigurator(
            connectors=[connector])

        # Cannot deactivate inactive configurator
        with pytest.raises(RuntimeError):
            configurator.deactivate()
        connector.expect(0, 0)

        configurator.activate()
        connector.expect(1, 0)

        # Cannot activate already activated configurator
        with pytest.raises(RuntimeError):
            configurator.activate()
        connector.expect(1, 0)

        configurator.deactivate()
        connector.expect(1, 1)

        # Cannot deactivate twice
        with pytest.raises(RuntimeError):
            configurator.deactivate()
        connector.expect(1, 1)

    def test_publish(self):
        class RecordingOutConnector(MockConnector):
            last_event = None

            def handle(self, event):
                RecordingOutConnector.last_event = event

        configurator = rsb.eventprocessing.OutRouteConfigurator(
            connectors=[RecordingOutConnector()])

        event = 42

        # Cannot publish while inactive
        with pytest.raises(RuntimeError):
            configurator.handle(event)
        assert RecordingOutConnector.last_event is None

        configurator.activate()
        configurator.handle(event)
        assert RecordingOutConnector.last_event == event

        event = 34
        configurator.handle(event)
        assert RecordingOutConnector.last_event == event

        # Deactivate and check exception, again
        RecordingOutConnector.last_event = None
        configurator.deactivate()
        with pytest.raises(RuntimeError):
            configurator.handle(event)
        assert RecordingOutConnector.last_event is None

class TestInRouteConfigurator:

    def test_activation(self):
        connector = ActivateCountingMockConnector()
        configurator = rsb.eventprocessing.InRouteConfigurator(
            connectors=[connector])

        # Cannot deactivate inactive configurator
        with pytest.raises(RuntimeError):
            configurator.deactivate()
        connector.expect(0, 0)

        configurator.activate()
        connector.expect(1, 0)

        # Cannot activate already activated configurator
        with pytest.raises(RuntimeError):
            configurator.activate()
        connector.expect(1, 0)

        configurator.deactivate()
        connector.expect(1, 1)

        # Cannot deactivate twice
        with pytest.raises(RuntimeError):
            configurator.deactivate()
        connector.expect(1, 1)

    def test_notify_connector(self):
        class RecordingMockConnector(MockConnector):
            def __init__(self):
                self.calls = []

            def filter_notify(self, filter_, action):
                self.calls.append((filter_, action))

            def expect(self1, calls):  # noqa: N805
                assert len(self1.calls) == len(calls)
                for (exp_filter, exp_action), (filter_, action) in \
                        zip(calls, self1.calls):
                    assert exp_filter == filter_
                    if exp_action == 'add':
                        assert action == rsb.filter.FilterAction.ADD

        connector = RecordingMockConnector()
        configurator = rsb.eventprocessing.InRouteConfigurator(
            connectors=[connector])
        configurator.activate()
        connector.expect(())

        f1, f2, f3 = 12, 24, 36
        configurator.filter_added(f1)
        connector.expect(((f1, 'add'),))

        configurator.filter_added(f2)
        connector.expect(((f1, 'add'), (f2, 'add')))

        configurator.filter_added(f3)
        connector.expect(((f1, 'add'), (f2, 'add'), (f3, 'add')))

        configurator.filter_removed(f3)
        connector.expect(((f1, 'add'), (f2, 'add'), (f3, 'add'),
                          (f3, 'remove')))

class TestFullyParallelEventReceivingStrategy:

    class CollectingHandler:

        def __init__(self):
            self.condition = Condition()
            self.event = None

        def __call__(self, event):
            with self.condition:
                self.event = event
                self.condition.notifyAll()

    def test_smoke(self):

        strategy = FullyParallelEventReceivingStrategy()

        h1 = self.CollectingHandler()
        h2 = self.CollectingHandler()
        strategy.add_handler(h1, True)
        strategy.add_handler(h2, True)

        event = Event(event_id=42)
        strategy.handle(event)

        with h1.condition:
            while h1.event is None:
                h1.condition.wait()
            assert event == h1.event

        with h2.condition:
            while h2.event is None:
                h2.condition.wait()
            assert event == h2.event

    def test_filtering(self):

        strategy = FullyParallelEventReceivingStrategy()

        false_filter = RecordingFalseFilter()
        strategy.add_filter(false_filter)

        handler = self.CollectingHandler()
        strategy.add_handler(handler, True)

        event = Event(event_id=42)
        strategy.handle(event)

        with false_filter.condition:
            while len(false_filter.events) == 0:
                false_filter.condition.wait(timeout=5)
                if len(false_filter.events) == 0:
                    self.fail("Filter not called")

        time.sleep(1)

        with handler.condition:
            assert handler.event is None

        strategy.remove_filter(false_filter)

    def test_parallel_call_of_one_handler(self):

        class Counter:
            def __init__(self):
                self.value = 0
        max_parallel_calls = Counter()
        current_calls = []
        call_lock = Condition()

        class Receiver:

            def __init__(self, counter):
                self.counter = counter

            def __call__(self, message):
                with call_lock:
                    current_calls.append(message)
                    self.counter.value = max(self.counter.value,
                                             len(current_calls))
                    call_lock.notifyAll()
                time.sleep(2)
                with call_lock:
                    current_calls.remove(message)
                    call_lock.notifyAll()

        strategy = FullyParallelEventReceivingStrategy()
        strategy.add_handler(Receiver(max_parallel_calls), True)

        event = Event(event_id=42)
        strategy.handle(event)
        event = Event(event_id=43)
        strategy.handle(event)
        event = Event(event_id=44)
        strategy.handle(event)

        num_called = 0
        with call_lock:
            while max_parallel_calls.value < 3 and num_called < 5:
                num_called = num_called + 1
                call_lock.wait()
            if num_called == 5:
                self.fail("Impossible to be called in parallel again")
            else:
                assert max_parallel_calls.value == 3


import torch
import torch.nn as nn
import torch.optim as optim
import random

# It is more oop friendly when Enc, Dec, Attention are implemented separably.
class Encoder(nn.Module):
    def __init__(self, rnn_type: str, vocab_size: int, emb_dim: int, hid_dim: int, dropout: float, n_layers: int=1, bidirectional: bool=True):
        super(Encoder, self).__init__()

        assert rnn_type in ['LSTM', 'RNN', 'GRU'], 'RNN type is not supported. Please select one of [GRU, RNN, LST]'

        self.emb_dim = emb_dim
        self.hid_dim = hid_dim
        self.bidirectional = bidirectional
        self.rnn_type = rnn_type.upper()
        self.n_layers = n_layers
        self.dropout = nn.Dropout(dropout)
        self.embedding = nn.Embedding(vocab_size, emb_dim)
        self.num_dir = 2 if bidirectional else 1

        if rnn_type == "LSTM":
            self.encoder = nn.LSTM(emb_dim, hid_dim, num_layers=self.n_layers, bidirectional=self.bidirectional, dropout=dropout)
        elif rnn_type == 'RNN':
            self.encoder = nn.RNN(emb_dim, hid_dim, num_layers=self.n_layers, bidirectional=self.bidirectional, dropout=dropout)
        elif rnn_type == 'GRU':
            self.encoder = nn.GRU(emb_dim, hid_dim, num_layers=self.n_layers, bidirectional=self.bidirectional, dropout=dropout)

        # for feeding initial hidden state to the decoder
        self.fc_h = nn.Linear(self.num_dir * hid_dim, hid_dim)
        if self.rnn_type == "LSTM":
            self.fc_c = nn.Linear(self.num_dir * hid_dim, hid_dim)

    def forward(self, src: torch.Tensor, hidden=None):
        # src: [seq_len, Batch_size]
        batch_size = src.size(1)
        seq_len = src.size(0)

        emb = self.embedding(src) # -> emb: [seq_len, Batch_size, emb_dim]
        emb = self.dropout(emb)

        # if initial hidden state were not fed, zero vectors are provided automatcially
        # Use only last layer's hidden state of stacked RNN
        if self.rnn_type == 'LSTM':
            H, (h, c) = self.encoder(emb, hidden) # don't need cell state
            # c: [num_dir * n_layers, Batch, enc_hid]
            c = c.view(self.n_layers, self.num_dir, batch_size, self.hid_dim) # [n_layers, num_dir, Batch, enc_hid]

            if self.bidirectional:
                c = torch.cat([c[:, -1, :, :], c[:, 0, :, :]], dim=2) # [n_layers, Batch, 2 * enc_hid]
                c = c.permute(1, 0, 2) # [Batch, n_layers, 2 * enc_hid]
                c = torch.tanh(self.fc_c(c)) # [Batch, n_layers, dec_hid]
            else:
                c = c.squeeze(1) # [n_layers, Batch, 2 * enc_hid]
                c = c.permute(1, 0, 2) # [Batch, n_layers, 2 * enc_hid]
                c = torch.tanh(self.fc_c(c)) # [Batch, n_layers, dec_hid]
            # c: [Batch, n_layers, dec_hid]
            c = c.permute(1, 0, 2)
            # c: [n_layers, Batch, dec_hid]
        else:
            H, h = self.encoder(emb, hidden)
        # H: [Seq_len, Batch, num_dir * hid]
        # h: [num_dir * n_layers, Batch, enc_hid]

        h = h.view(self.n_layers, self.num_dir, batch_size, self.hid_dim) # [n_layers, num_dir, Batch, enc_hid]

        if self.bidirectional:
            h = torch.cat([h[:, -1, :, :], h[:, 0, :, :]], dim=2) # [n_layers, Batch, 2 * enc_hid]
            h = h.permute(1, 0, 2) # [Batch, n_layers, 2 * enc_hid]
            h = torch.tanh(self.fc_h(h)) # [Batch, n_layers, dec_hid]

            H = H[:, :, :self.hid_dim] + H[:, :, self.hid_dim:] # [n_layers, Batch, enc_hid]
        else:
            h = h.squeeze(1) # [n_layers, Batch, 2 * enc_hid]
            h = h.permute(1, 0, 2) # [Batch, n_layers, 2 * enc_hid]
            h = torch.tanh(self.fc_h(h)) # [Batch, n_layers, dec_hid]

        h = h.permute(1, 0, 2)
        # H: [Seq_len, Batch, num_dir * hid]
        # h: [n_layers, Batch, dec_hid]
        if self.rnn_type == 'LSTM':
            # c: [n_layers, Batch, dec_hid]
            return H.contiguous(), (h.contiguous(), c.contiguous())
        else:
            return H.contiguous(), h.contiguous()

class LuongDecoder(nn.Module):
    def __init__(self, rnn_type: str, attn_type: str, vocab_size: int, emb_dim: int, hid_dim: int, attention: nn.Module, dropout: float,
                num_dir: int, device: str, n_layers: int=1, D: int=10, Pt: str='monotonic'):
        super(LuongDecoder, self).__init__()
        self.rnn_type = rnn_type.upper()
        self.attn_type = attn_type.lower()
        self.Pt = Pt.lower()

        assert self.rnn_type in ['LSTM', 'RNN', 'GRU'], 'RNN type is not supported. Please select one of [GRU, RNN, LST]'
        assert self.attn_type in ['global', 'local'], 'attention type is not supported. Please select one of [global, local]'
        assert self.Pt in ['monotonic', 'predictive'], 'Pt is not supported. Please select one of [monotonic, predictive]'

        self.D = D
        self.emb_dim = emb_dim
        self.hid_dim = hid_dim
        self.dropout = nn.Dropout(dropout)

        self.vocab_size = vocab_size
        self.num_dir = num_dir
        self.n_layers = n_layers
        self.attention = attention

        self.embedding = nn.Embedding(self.vocab_size, emb_dim)

        if self.rnn_type == "LSTM":
            self.decoder = nn.LSTM(self.hid_dim + emb_dim, hid_dim, dropout=dropout, num_layers=self.n_layers)
        elif self.rnn_type == 'RNN':
            self.decoder = nn.RNN(self.hid_dim + emb_dim, hid_dim, dropout=dropout, num_layers=self.n_layers)
        elif self.rnn_type == 'GRU':
            self.decoder = nn.GRU(self.hid_dim + emb_dim, hid_dim, dropout=dropout, num_layers=self.n_layers)

        if self.Pt == 'predictive':
            self.attn_dim = hid_dim // 8
            self.local_v = nn.Parameter(torch.randn(self.attn_dim, 1))
            self.local_w = nn.Linear(hid_dim, self.attn_dim)

        self.out_layer = nn.Linear(self.hid_dim, self.vocab_size)

    def forward(self, t: int, trg: torch.Tensor, prev_s: torch.Tensor, H: torch.Tensor):
        # trg(y_i): [Seq_len(=1), Batch_size]
        # prev_s: (h, c)
            # h: [n_layers, Batch, dec_hid]
            # c: [n_layers, Batch, dec_hid]
        # H: [Seq_len(T_x), Batch, enc_hid]
        batch_size = H.size(1)
        trg = trg.unsqueeze(0)
        self.t = t

        emb = self.embedding(trg) # -> emb: [Seq_len(=1), Batch_size, emb_dim]
        emb = self.dropout(emb)

        # Input feeding
        if self.rnn_type == 'LSTM':
            prev_c = prev_s[1]
            prev_s = prev_s[0] # [n_layers, Batch, dec_hid]

            temp = prev_s[-1].unsqueeze(0)
            input_ = torch.cat([temp, emb], dim=2) # [1, Batch_size, dec_hid + emb_dim]
            dec_output, (s, c) = self.decoder(input_, (prev_s, prev_c))
        else:
            temp = prev_s[-1].unsqueeze(0)
            input_ = torch.cat([temp, emb], dim=2) # [1, Batch_size, dec_hid + emb_dim]
            dec_output, s = self.decoder(input_, prev_s)

        context = self._get_attn_weight(H, s[-1]) # [batch_size, 1, enc_hid]

        # then, concat [y_{i-1}, c_i] and feed to the decoder rnn
        context = context.permute(1, 0, 2) # [1, batch_size, enc_hid]

        # dec_output: [1, Batch, dec_hid]
        # s: [num_layer, Batch, dec_hid]
        # c: [num_layer, Batch, dec_hid]
        # Use only last layer's hidden state in decoder
        output = self.out_layer(s[-1, :, :].unsqueeze(0).permute(1, 0, 2)) # [Batch, 1, vocab_size]

        if self.rnn_type == 'LSTM':
            return output.squeeze(1), (s, c)
        else:
            return output.squeeze(1), s

    def _get_attn_weight(self, H, s):
        # s: [Batch, dec_hid]
        # H: [Seq_len(T_x), Batch, enc_hid]

        align_score  = self.attention(H, s) # [batch_size, 1, seq_len]
        batch_size = align_score.size(0)

        if self.attn_type == 'global':
            align_score = torch.softmax(align_score, dim=2) # [batch_size, 1, seq_len]
            context = torch.bmm(align_score, H.permute(1, 0, 2)) # [batch_size, 1, enc_hid]
        else:
            Tx = align_score.size(2)

            if self.Pt == 'monotonic':
                Pt = self.t # integer
                start = max(0, Pt - self.D)
                end = Pt + self.D + 1
                align_score = align_score[:, :, start:end] # [batch_size, 1, 2D + 1]
                H = H.permute(1, 0, 2)[:, start:end, :] # [batch_size, 1, 2D + 1]

                local = torch.tensor([x for x in range(1, align_score.size(2) + 1)]).to(device) # [2D + 1]
                local = torch.exp(- ((local -   Pt) ** 2)/((self.D ** 2) / 2)).view(1, 1, -1) # [2D + 1]
                local = local.repeat(batch_size, 1, 1) # [B, 1, 2D + 1]
                align_score = align_score * local  # [B, 1, 2D + 1]

                context = torch.bmm(align_score, H) # # [batch_size, 1, 2D + 1]

            elif self.Pt == 'predictive':
                # Pt - D and Pt + D are different for each batch
                temp = self.local_w(s.unsqueeze(1)) # [Batch, 1, attn_dim]
                Pt = Tx * torch.sigmoid(temp.bmm(self.local_v.repeat(batch_size, 1, 1))).view(-1, 1) # [Batch, 1]

                # Start/end represent Pt-D/Pt+D repectively
                start = torch.cat([Pt, torch.zeros_like(Pt)], dim=1)
                start = start.max(1)[0].long() # [batch_size]
                end =  start + self.D + 1 # [batch_size]

                # align_score = torch.cat([align_score[i, :, start[i]:end[i]] for i in range(batch_size)], dim=0).unsqueeze(1) # [batch_size, 1, 2D + 1]
                # list of batch_size len
                align_score = [align_score[i, :, start[i]:end[i]] for i in range(batch_size)] # [1, 2D + 1] for an element
                H = H.permute(1, 0, 2)
                # H = torch.cat([H[i, start[i]:end[i], :] for i in range(batch_size)], dim=0).view(batch_size, -1, self.hid_dim)
                # list of batch_size len
                H = [H[i, start[i]:end[i], :] for i in range(batch_size)] # [2D + 1, hid_dim] for an element

                # for j Pt - D, Pt + D
                end_batch = (start + torch.tensor([H[i].size(0) for i in range(len(H))]).to(device)).view(-1, 1) # [batch, 1]
                end = torch.cat([end.view(-1, 1), end_batch], dim=1) # [batch, 2]
                end = end.min(1)[0] # [batch]

                context = torch.zeros(batch_size, 1, self.hid_dim).to(device)

                for i in range(batch_size):
                    local = torch.tensor([j for j in range(start[i], end[i])]).to(device) # [2D + 1]: for j
                    batch_Pt = Pt[i]
                    local = torch.exp(-1 * ((local-batch_Pt) ** 2)/((self.D ** 2) / 2)) # [2D + 1]
                    score = align_score[i] * local  # [1, 2D + 1]
                    context[i] = score.mm(H[i])
        return context

class Attention(nn.Module):
    def __init__(self, rnn_type: str, align_fn: str, hid_dim: int, num_dir: int):
        super(Attention, self).__init__()
        self.align_fn = align_fn.lower()
        self.rnn_type = rnn_type.upper()

        assert self.align_fn in ['concat', 'general', 'dot'], 'Alignment model is not supported. Please select one of [concat, general, dot]'

        self.num_dir = num_dir
        self.hid_dim = hid_dim

        if self.align_fn == 'concat':
            self.attn_dim = hid_dim // 8
            self.v = nn.Parameter(torch.rand(1, self.attn_dim)) # [1, attn_dim]
            self.attn = nn.Linear(self.hid_dim + self.hid_dim, self.attn_dim)
        elif self.align_fn == 'general':
            self.attn = nn.Linear(self.hid_dim, self.hid_dim)

    def forward(self, H, prev_s):
        # H: [Seq_len, Batch, enc_hid]
        # prev_s: (h, c)
            # h: [Batch, dec_hid]
            # c: [Batch, dec_hid]

        batch_size = H.size(1)

        if self.align_fn == 'concat':
            # prev_s: [Batch, dec_hid]
            # vT tanh(Ws + Uh)
            prev_s = prev_s.unsqueeze(1) # [Batch, 1, dec_hid]
            prev_s = prev_s.repeat(1, H.size(0), 1) # [Batch, seq_len, dec_hid]
            H = H.permute(1, 0, 2) # [Batch, Seq_len, enc_hid]
            a = torch.cat([prev_s, H], dim=2) # [batch_size, seq_len, dec_hid + enc_hid]
            a = torch.tanh(self.attn(a)) # [batch_size, seq_len, attn_dim]
            a = a.permute(0, 2, 1)  # [batch_size, attn_dim, seq_len(=1)]
            v = self.v.unsqueeze(0).expand(batch_size, -1, -1) # [batch_size, 1, attn_dim]
            align_score = torch.bmm(v, a) # [batch_size, 1, seq_len]

            return align_score

        elif self.align_fn == 'general':
            H = H.permute(1, 0, 2) # [Batch, Seq_len, enc_hid]
            a = self.attn(H) # [Batch, Seq_len, enc_hid]

            align_score = torch.bmm(a, prev_s.unsqueeze(2)) # [batch_size, seq_len, 1]
            align_score = align_score.permute(0, 2, 1) # [batch_size, 1, seq_len]

            return align_score

        elif self.align_fn == 'dot':
            H = H.permute(1, 2, 0) # [Batch, enc_hid, Seq_len]
            align_score = torch.bmm(prev_s.unsqueeze(1), H) # [batch_size, 1, seq_len]

            return align_score

class Seq2Seq(nn.Module):
    def __init__(self, rnn_type: str, encoder: nn.Module, decoder: nn.Module, trg_vocab_size: int, device: str, num_dir: int,
                teacher_forcing: float=0.5, beam_search: int=1):
        super(Seq2Seq, self).__init__()

        self.beam_search = beam_search
        self.teacher_forcing = teacher_forcing
        self.device = device
        self.num_dir = num_dir
        self.trg_vocab_size = trg_vocab_size
        self.encoder = encoder
        self.decoder = decoder
        self.rnn_type = rnn_type.upper()

    def forward(self, sources, targets):
        batch_size = sources.size(1)
        max_len = targets.size(0) # only search sentences of length up to

        outputs = torch.zeros(max_len, batch_size, self.trg_vocab_size).to(self.device)
        if self.rnn_type == 'LSTM':
            H, (h, c) = self.encoder(sources)
            # c: [Batch, dec_hid]
        else:
            H, h = self.encoder(sources)
            # h: [Batch, dec_hid]

        # H: [Seq_len, Batch, num_dir * enc_hid]

        s = h
        y = targets[0, :]

        for t in range(1, max_len):
            if self.rnn_type == 'LSTM':
                y, (s, c) = self.decoder(t, y, (s, c), H)
            else:
                y, s = self.decoder(t, y, s, H)
            outputs[t] = y
            is_teacher_force = random.random() < self.teacher_forcing
            topk = y.max(1)[1] # get idx of the next word
            y = targets[t] if is_teacher_force else topk

        return outputs


from django.core.management import BaseCommand

class Command(BaseCommand):
    help = "Gives the user a fixed amount of coins if they have"
    "less than that amount"

    def handle(self, *args, **options):
        from userprofile.models import UserProfile
        from notification.models import Notification
        profile_list = UserProfile.objects.filter(coins__lte=9)
        for p in profile_list:
            self.stdout.write("Daily Reward!")
            p.coins = 10
            p.save()
            n = Notification(description='Daily Reward: '
                             'You have 10 coins now!', profile=p)
            n.save()


import os
from googleapiclient.discovery import build
from dotenv import load_dotenv

load_dotenv()
api_key = os.environ.get('YOUTUBE_API_KEY')
print(api_key)
youtube = build('youtube', 'v3', developerKey=api_key)

request = youtube.channels().list(
    part='statistics',
    forUsername='schafer5'
)

response = request.execute()

print(response)


import torch
from datetime import datetime
from pathlib import Path
import os
import argparse
from torch import nn
from torch.utils.data import DataLoader
from torch.distributed import all_gather
from torch.distributed import get_world_size
import torchaudio
from ae_model import CNNAutoencoder
from distributed import init_distributed_mode

from custom_audio_dataset import BirdAudioDataset
from ae_custom_distributed_sampler import CustomDistributedSampler

from torch.utils.tensorboard import SummaryWriter

import scipy.stats as stats

def get_arguments():
    parser = argparse.ArgumentParser(
        description="Pretrain a resnet model with VICReg", add_help=False
    )

    # Optim
    parser.add_argument("--epochs", type=int, default=500, help="Number of epochs")
    parser.add_argument(
        "--batch-size", type=int, default=2048, help="Effective batch size"
    )
    parser.add_argument("--lr", type=float, default=0.0001, help="Learning rate")
    parser.add_argument("--wd", type=float, default=1e-7, help="Weight decay")

    # Checkpoints
    parser.add_argument(
        "--exp-dir",
        type=Path,
        default="./exp",
        help="Path to the experiment folder, where all logs/checkpoints will be stored",
    )
    parser.add_argument(
        "--resume",
        type=bool,
        default=True,
        help="Whether or not to resume from a checkpoint",
    )

    # Running
    parser.add_argument("--num-workers", type=int, default=10)
    parser.add_argument(
        "--device", default="cuda", help="device to use for training / testing"
    )

    # Distributed
    parser.add_argument(
        "--world-size", default=1, type=int, help="number of distributed processes"
    )
    parser.add_argument("--local_rank", default=-1, type=int)
    parser.add_argument(
        "--dist-url", default="env://", help="url used to set up distributed training"
    )

    # Pruning
    parser.add_argument("--prune-anomalies", action='store_true')
    parser.add_argument("--prune-frequency", default=50, type=int)

    # Spectrogram Cropping
    parser.add_argument("--crop-mode", choices=['none', 'second_half', 'third_quarter'], default='third_quarter')

    # Audio Files
    parser.add_argument("--audio-path", type=Path, default="/grand/projects/BirdAudio/Morton_Arboretum/audio/set3/00004879")
    parser.add_argument("--single-file", type=Path)
    parser.add_argument("--audio-file-cap", type=int, default=15)

    # Sampling
    parser.add_argument("--num-samples", type=int, default=22050)
    parser.add_argument("--sample-rate", type=int, default=22050)

    # Logging
    parser.add_argument("--log-frequency", type=int, default=1)

    # Model
    parser.add_argument("--bottleneck-dim", type=int, default=10)

    return parser

def get_all_audio_files(audio_path, audio_file_cap):
    audio_files = []
    for root, dirs, files in os.walk(audio_path, topdown=False):
        for name in files:
            extension = os.path.splitext(name)[1]
            if extension == ".wav":
                audio_file = os.path.join(root, name)
                audio_files.append(audio_file)
                if len(audio_files) >= audio_file_cap:
                    return audio_files
    return audio_files

def train_one_epoch(args, model, data_loader, loss_fn, optimizer, device, epoch, writer, gpu):
    for i, (inputs, _) in enumerate(data_loader, start=epoch * len(data_loader)):
        inputs = inputs.cuda(gpu, non_blocking=True)
        recons, _ = model(inputs)
        loss = loss_fn(recons, inputs)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        writer.add_scalar("Loss", loss.item())
    if (epoch + 1) % args.log_frequency == 0:
        print("Loss: ", loss.item())
    if args.rank == 0:
        state = dict(
            epoch=epoch + 1,
            model=model.state_dict(),
            optimizer=optimizer.state_dict(),
        )
        torch.save(state, args.exp_dir / "model.pth")

def train(
    args,
    model,
    bad,
    data_loader,
    prune_data_loader,
    loss_fn,
    optimizer,
    device,
    epochs,
    start_epoch,
    writer,
    gpu,
):
    start_time = datetime.now()
    for epoch in range(start_epoch, epochs):
        print(f"Epoch:  {epoch + 1}/{epochs}, Time: {datetime.now() - start_time}")
        if args.prune_anomalies and (epoch + 1) % args.prune_frequency == 0:
            print("Pruning anomalies!")
            normal_indices = prune_anomalies(model, bad, prune_data_loader, loss_fn, gpu)
            data_loader, prune_data_loader = get_data_loaders(args, bad, normal_indices)
        train_one_epoch(
            args, model, data_loader, loss_fn, optimizer, device, epoch, writer, gpu
        )
    print("Training is done.")
    if args.rank == 0:
        torch.save(model.state_dict(), args.exp_dir / "conv_autoencoder.pth")

def prune_anomalies(model, bad, prune_data_loader, loss_fn, gpu):
    indices = []
    losses = []

    with torch.no_grad():
        for i, (inputs, _) in enumerate(prune_data_loader):
            inputs = inputs.cuda(gpu, non_blocking=True)
            recons, _ = model(inputs)
            loss = loss_fn(recons, inputs)
            indices.append(i)
            losses.append(loss)

    indices = torch.tensor(indices, dtype=torch.int32).cuda(gpu, non_blocking=True)
    losses = torch.tensor(losses, dtype=torch.float32).cuda(gpu, non_blocking=True)

    all_indices = [
        torch.zeros(len(prune_data_loader), dtype=torch.int32).cuda(gpu, non_blocking=True) for _ in range(get_world_size())
    ]
    all_losses = [
        torch.zeros(len(prune_data_loader), dtype=torch.float32).cuda(gpu, non_blocking=True) for _ in range(get_world_size())
    ]

    all_gather(all_indices, indices)
    all_gather(all_losses, losses)

    all_indices = torch.cat(all_indices).detach().cpu().numpy()
    all_losses = torch.cat(all_losses).detach().cpu().numpy()

    all_z_scores = stats.zscore(all_losses)

    normal_indices = all_indices[all_z_scores < 3]

    print(f"Pruned {len(all_indices) - len(normal_indices)} samples...")

    return normal_indices

def main(args):
    torch.backends.cudnn.benchmark = True
    init_distributed_mode(args)
    print(args)
    gpu = torch.device(args.device)

    if args.rank == 0:
        args.exp_dir.mkdir(parents=True, exist_ok=True)

    mel_spectrogram = torchaudio.transforms.MelSpectrogram(
        sample_rate=args.sample_rate,
        n_fft=1024,
        hop_length=512,
        n_mels=((256 if args.crop_mode == "third_quarter" else (128 if args.crop_mode == "second_half" else 64)))
    )

    if args.single_file is not None:
        audio_files = [args.single_file]
        print(f"Loading single audio file: ", args. single_file)

    else:
        audio_files = get_all_audio_files(args.audio_path, args.audio_file_cap)
        print(f"Loading {len(audio_files)} audio file(s):")
        for audio_file in audio_files:
            print(audio_file)

    bad = BirdAudioDataset(
        audio_files,
        mel_spectrogram,
        args.sample_rate,
        args.num_samples,
        gpu,
        crop_mode=args.crop_mode
    )

    train_data_loader, prune_data_loader = get_data_loaders(args, bad, range(len(bad)))

    model = CNNAutoencoder(bottleneck_dim=args.bottleneck_dim).cuda(gpu)
    if torch.cuda.device_count() > 1:
        print("Using", torch.cuda.device_count(), "GPUs")
    model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])

    loss_fn = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)

    if (args.exp_dir / "model.pth").is_file() and args.resume:
        if args.rank == 0:
            print("resuming from checkpoint")
        ckpt = torch.load(args.exp_dir / "model.pth", map_location="cpu")
        start_epoch = ckpt["epoch"]
        model.load_state_dict(ckpt["model"])
        optimizer.load_state_dict(ckpt["optimizer"])
    else:
        start_epoch = 0

    log_dir = f"theta_run_birds/lr{args.lr}_wd{args.wd}_bs{args.batch_size}_epochs{args.epochs}"
    writer = SummaryWriter(log_dir)
    print(f"Tensorboard logging at {log_dir}")

    train(
        args,
        model,
        bad,
        train_data_loader,
        prune_data_loader,
        loss_fn,
        optimizer,
        args.device,
        args.epochs,
        start_epoch,
        writer,
        gpu,
    )

def get_data_loaders(args, bad, normal_indices):
    assert args.batch_size % args.world_size == 0
    per_device_batch_size = args.batch_size // args.world_size

    sampler = CustomDistributedSampler(bad, normal_indices)
    print("Number of samples: ", sampler.total_size)

    train_data_loader = DataLoader(
        bad,
        batch_size=per_device_batch_size,
        num_workers=args.num_workers,
        pin_memory=False,
        sampler=sampler,
    )
    prune_data_loader = DataLoader(
        bad,
        batch_size=1,
        num_workers=args.num_workers,
        pin_memory=False,
        sampler=sampler
    )

    return train_data_loader, prune_data_loader

if __name__ == "__main__":
    parser = get_arguments()
    args = parser.parse_args()
    main(args)


"""
81. RNNによる予測
ID番号で表現された単語列x=(x1,x2,…,xT)がある．ただし，Tは単語列の長さ，xt∈ℝVは単語のID番号のone-hot表記である（Vは単語の総数である）．
再帰型ニューラルネットワーク（RNN: Recurrent Neural Network）を用い，単語列xからカテゴリyを予測するモデルとして，次式を実装せよ．
"""
import tensorflow as tf

from tf_models import RNNModel
from utils import load_data, preprocess, build_vocabulary, text2sequence, filter_embeddings, seed_everything

if __name__ == '__main__':

    # データのロード
    dfs = load_data()

    # 前処理
    dfs['train']['clean_title'] = dfs['train'][['title']].apply(preprocess)
    dfs['valid']['clean_title'] = dfs['valid'][['title']].apply(preprocess)
    dfs['test']['clean_title'] = dfs['test'][['title']].apply(preprocess)

    # ボキャブラリの生成
    # 「出現頻度が2回未満の単語のID番号はすべて0とせよ．」は無視しています.（時間があったらstopword除外を入れます）
    vocab = build_vocabulary(dfs['train']['clean_title'])
    # 単語IDを確認する
    result = {k: vocab.word_index[k] for k in list(vocab.word_index)[:15]}
    print(result)
    """
    >>
    {'<UNK>': 1, 'update': 2, 'us': 3, 'new': 4, 'says': 5, 'stocks': 6, 'china': 7, 'kardashian': 8, 'euro': 9,
    'kim': 10, 'first': 11, 'may': 12, 'ecb': 13, 'shares': 14, 'fed': 15}
    """

    # 単語IDの列を取得（学習データ）
    X_train = text2sequence(dfs['train']['clean_title'], vocab)
    X_valid = text2sequence(dfs['valid']['clean_title'], vocab)
    X_test = text2sequence(dfs['test']['clean_title'], vocab)
    print(X_train.shape)
    print(X_valid.shape)
    print(X_test.shape)

    # 目的変数の生成
    category_dict = {'b': 0, 't': 1, 'e': 2, 'm': 3}
    y_train = dfs['train']['category'].map(category_dict)
    y_valid = dfs['valid']['category'].map(category_dict)
    y_test = dfs['test']['category'].map(category_dict)

    # モデルの生成
    seed_everything(42)
    tf.keras.backend.clear_session()
    model = RNNModel(len(vocab.word_index)+1, len(y_train.unique())).build()
    model.summary()
    """
    >>
    Model: "model_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    input (InputLayer)           [(None, None)]            0
    _________________________________________________________________
    embedding (Embedding)        (None, None, 300)         3733200
    _________________________________________________________________
    rnn (SimpleRNN)              (None, 100)               40100
    _________________________________________________________________
    dense_2 (Dense)              (None, 4)                 404
    =================================================================
    Total params: 3,773,704
    Trainable params: 3,773,704
    Non-trainable params: 0
    _________________________________________________________________
    """

    print(model(X_train[:4]))
    """
    >>
    tf.Tensor(
    [[0.26211333 0.20876467 0.18698843 0.34213355]
    [0.2443882  0.23337469 0.29404563 0.22819154]
    [0.31824696 0.18265769 0.19796237 0.30113298]
    [0.20688418 0.2832237  0.2216745  0.28821763]], shape=(4, 4), dtype=float32)
    """


import json
import typing
import uuid
from hashlib import md5
from random import choice
from string import digits, ascii_letters
from typing import Dict

import requests

class BusinessApi:
    """
        При отправке запроса к API приложение должно выполнить ряд действий.

        К массиву параметров запроса добавляются параметры: nonce, app_id и token, где
        -- nonce - случайно генерируемая, уникальная для каждого запроса строка,
        -- app_id - уникальный идентификатор, выданный при подключении интеграции на аккаунте,
        -- token - полученный токен интеграции.
        Полученный массив параметров сортируется в алфавитном порядке и преобразуется в json строку - params_string.
        К запросу добавляется заголовок sign. Подпись запроса вычисляется по формуле: sign = MD5( params_string + secret ),
        где параметром функции MD5 служит конкатенация компонентов:
        -- params_string - url-кодированная строка параметров
        -- secret - секретный ключ, полученный при подключении интеграции на аккаунте.
    """

    def __init__(self, secret_key, app_id):
        """
        Init - метод конструктор собирает библиотеку для работы.
        :param secret_key: секретный ключ, полученный при подключении интеграции на аккаунте.
        :param app_id: уникальный идентификатор, выданный при подключении интеграции на аккаунте.
        :param nonce: случайно генерируемая, уникальная для каждого запроса строка.
        """
        self.secret_key = secret_key
        self.app_id = app_id
        self.nonce = "".join(choice(digits + ascii_letters) for _ in range(10))

    @staticmethod
    def get_sign(secret: str, param: dict) -> str:
        """
        Создание уникальной подписи для каждого запроса.
            Вернет сгенерированную подпись из параметров запроса.
        :param secret:
        :param param:
        :return:
        """
        params_string = json.dumps(param).replace(' ', '')
        string = params_string + secret
        return md5(string.encode("utf-8")).hexdigest()

    @staticmethod
    def get_unique_hashcat():
        hashcat_one = uuid.uuid4()
        hashcat_two = uuid.uuid4()
        type_to_string = str(hashcat_one) + str(hashcat_two)
        hashcat = md5(type_to_string.encode("utf-8")).hexdigest()
        return hashcat

    def get_token(self):
        """
        Получение и восстановление токена.
            Вернёт новый токен если его нет, если есть то текущий токен.
        """
        try:
            param: Dict = {"app_id": self.app_id, "nonce": self.get_unique_hashcat()}
            sign = self.get_sign(self.secret_key, param)
            headers = {'Content-type': 'application/json; charset=utf-8',
                       'Accept': 'application/json', 'sign': sign}
            response = requests.get(url="https://check.business.ru/open-api/v1/Token/", params=param, headers=headers)
            data = response.json()
            token = json.dumps(data["token"])
            return token.strip('""')
        except Exception as error:
            print(error)

    def get_user(self):
        """
        Работа с пользователем / аккаунтом
            Вернет информацию о текущем пользователе / аккаунте
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/User/", params=param, headers=headers)
        data = response.json()
        print(data)

    def get_command_id(self, **kwargs):
        """
        Получение данных о команде в ФР по ID пользователя и ID команды
            Вернёт информацию о команде ФР
                Подробнее можно узнать на:

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_check

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_shift

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_cash

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_report

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/JSON_correction

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Shifts

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Checks

                https://app.swaggerhub.com/apis/Business.Ru/check.business.ru/2.0.0#/Goods
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/Command/{}".format(kwargs), params=param,
                                headers=headers)
        data = response.json()
        print(data)

    def get_command(self, filter_date_create_from, filter_date_create_to, filter_date_update_from,
                    filter_date_update_to, filter_date_result_from, filter_date_result_to, page, c_num):
        """
        Получение данных о команде в ФР по ID пользователя и ID команды
            Вернёт информацию о команде ФР с учётом фильтров
        """
        param: Dict = {"app_id": self.app_id,
                       "filter_date_create_from": filter_date_create_from,
                       "filter_date_create_to": filter_date_create_to,
                       "filter_date_update_from": filter_date_update_from,
                       "filter_date_update_to": filter_date_update_to,
                       "filter_date_result_from": filter_date_result_from,
                       "filter_date_result_to": filter_date_result_to,
                       "page": page,
                       "c_num": c_num,
                       "nonce": self.nonce,
                       "token": self.get_token(),
                       }

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': self.get_unique_sign()}
        response = requests.get(url="https://check.business.ru/open-api/v1/Command", params=param,
                                headers=headers)
        data = response.json()
        print(data)

    def get_state_system(self):
        """
        Получение данных о состоянии системы.
            Вернёт информацию о текущем состоянии системы.
        """
        param: Dict = {"app_id": self.app_id, "nonce": self.nonce, "token": self.get_token()}
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/StateSystem/", params=param, headers=headers)
        data = response.json()
        return data

    def get_shift(self, filter_date_create_from, filter_date_create_to, filter_date_update_from,
                  filter_date_update_to, filter_date_result_from, filter_date_result_to, page, c_num):
        """
        Получение данных о сменах.
            Вернёт информацию о сменах с учётом фильтров.
        """
        param: Dict = {"app_id": self.app_id,
                       "filter_date_create_from": filter_date_create_from,
                       "filter_date_create_to": filter_date_create_to,
                       "filter_date_update_from": filter_date_update_from,
                       "filter_date_update_to": filter_date_update_to,
                       "filter_date_result_from": filter_date_result_from,
                       "filter_date_result_to": filter_date_result_to,
                       "page": page,
                       "c_num": c_num,
                       "nonce": self.nonce,
                       "token": self.get_token(),
                       }
        sign = self.get_sign(self.secret_key, param)

        headers = {'Content-type': 'application/json; charset=utf-8',
                   'Accept': 'application/json', 'sign': sign}
        response = requests.get(url="https://check.business.ru/open-api/v1/Shift/", params=param, headers=headers)
        data = response.json()
        return data


import numpy as np
import cv2
import math

# Structuring element for morphology
kernel = np.ones((12,12),np.uint8) # 12by12  square

# VideoCapture using the only available camera
cap = cv2.VideoCapture(0)

# Will wait until black is seen (Code Reset) and used the location to only focus on panel while detecting the code
focused = False
# Hold set of seen colors
res = []
# define range of black in HSV
black_upper_hsv = np.array([179, 60, 95], dtype = "uint8")
black_lower_hsv = np.array([0, 0, 0], dtype = "uint8")

# Set up the detector with default parameters.
# Setup SimpleBlobDetector parameters.
params = cv2.SimpleBlobDetector_Params()
# Filter by Area.
params.filterByArea = True
params.minArea = 100
# Create detector
detector = cv2.SimpleBlobDetector(params)

# define the list of boundaries
boundaries = [
    (['R'], [17, 15, 100], [50, 56, 200]),      # R
    (['B'], [110, 100, 140], [160, 185, 255]),  # B
    (['Y'], [25, 146, 190], [62, 174, 250]),    # Y
    (['G'], [103, 86, 65], [145, 133, 128])     # G
]

imgFile = cv2.imread('/Users/ianstankosh/Desktop/HeadShot.jpg')
cv2.imshow('Original Black', imgFile)

hsv = cv2.cvtColor(imgFile, cv2.COLOR_BGR2HSV)
cv2.imshow('HSV_BLK', hsv)

# Threshold the HSV image to get only input colors
blk = cv2.inRange(hsv, black_lower_hsv, black_upper_hsv)
#blk = cv2.bitwise_not(blk)
cv2.imshow('blk', blk)
closing = cv2.morphologyEx(blk, cv2.MORPH_CLOSE, kernel)
cv2.imshow('Closing_blk', closing)

# Detect blobs.
keypoints = detector.detect(closing)

#Draw detected blobs as red circles.
#cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
im_with_keypoints = cv2.drawKeypoints(closing, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
cv2.imshow('Panel', im_with_keypoints)

panel = closing

imgFile2 = cv2.imread('BluePic.jpg')
cv2.imshow('Original Blue', imgFile2)
# Mask the frame to only focus on light panel
masked_data = cv2.bitwise_and(imgFile2, imgFile2, mask=panel)
cv2.imshow('Masked Data', masked_data)

# Covert to HSV to detect color
hsv2 = cv2.cvtColor(masked_data, cv2.COLOR_BGR2HSV)
cv2.imshow('HSV_BLU', hsv2)

# cv2.imwrite('hue.png', hsv2[:,:,0])
# cv2.imwrite('sat.png', hsv2[:,:,1])
# cv2.imwrite('val.png', hsv2[:,:,2])

# loop over the boundaries
for (color, lower, upper) in boundaries:
    # create NumPy arrays from the boundaries
    lower = np.array(lower, dtype = "uint8")
    upper = np.array(upper, dtype = "uint8")

    # find the colors within the specified boundaries and apply
    # the mask
    mask = cv2.inRange(hsv2, lower, upper)
    cv2.imshow('mask', mask)
    closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
    cv2.imshow('Closing', closing)
    #output = cv2.bitwise_and(image, image, mask = mask)
    #closing = cv2.bitwise_not(closing)

    # Detect blobs.
    keypoints = detector.detect(closing)

    #Draw detected blobs as red circles.
    #cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
    im_with_keypoints = cv2.drawKeypoints(closing, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
    cv2.imshow('Found', im_with_keypoints)

    if keypoints:
        res.append(color)

print ("Colors seen: ", res)

cv2.waitKey(0)
cv2.destroyAllWindows()


# from django.utils import timezone
from django.shortcuts import render, redirect

# from django.urls import reverse
from django.http import Http404
from django.core.paginator import Paginator, EmptyPage
from django.views.generic import ListView, DetailView, View
from django_countries import countries
from . import models
from . import forms

class HomeView(ListView):

    """ HomeView Class Definition """

    model = models.Room
    paginate_by = 10
    paginate_orphans = 5
    ordering = "created"
    page_kwarg = "page"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # now = timezone.now()
        # context["now"] = now
        return context

class RoomDetail(DetailView):

    """ RoomDetail Definition """

    model = models.Room

class SearchView(View):

    """ SearchView Definition """

    def get(self, request):
        country = request.GET.get("country")
        if country:
            form = forms.SearchForm(request.GET)
            if form.is_valid():
                city = form.cleaned_data.get("city")
                country = form.cleaned_data.get("country")
                room_type = form.cleaned_data.get("room_type")
                price = form.cleaned_data.get("price")
                guests = form.cleaned_data.get("guests")
                bedrooms = form.cleaned_data.get("bedrooms")
                beds = form.cleaned_data.get("beds")
                baths = form.cleaned_data.get("baths")
                instant_book = form.cleaned_data.get("instant_book")
                super_host = form.cleaned_data.get("super_host")
                amenities = form.cleaned_data.get("amenities")
                facilities = form.cleaned_data.get("facilities")

                filter_args = {}
                if city != "Anywhere":
                    filter_args["city__startswith"] = city
                filter_args["country"] = country
                if room_type is not None:
                    filter_args["room_type"] = room_type
                if price is not None:
                    filter_args["price__lte"] = price
                if guests is not None:
                    filter_args["guests__gte"] = guests
                if bedrooms is not None:
                    filter_args["bedrooms__gte"] = bedrooms
                if beds is not None:
                    filter_args["beds__gte"] = beds
                if baths is not None:
                    filter_args["baths__gte"] = baths
                if instant_book:
                    filter_args["instant_book"] = True
                if super_host:
                    filter_args["host__superhost"] = True
                for amenity in amenities:
                    filter_args["amenities"] = amenity
                for facility in facilities:
                    filter_args["facilities"] = facility

                qs = models.Room.objects.filter(**filter_args).order_by("-created")
                paginator = Paginator(qs, 10, orphans=5)
                page = request.GET.get("page", 1)
                rooms = paginator.get_page(page)
                return render(
                    request,
                    "rooms/room_search.html",
                    context={"form": form, "rooms": rooms},
                )
        else:
            form = forms.SearchForm()
            return render(request, "rooms/room_search.html", context={"form": form})

# HomeView (Generic ListView)와 동일
def all_rooms(request):
    page = request.GET.get("page", 1)
    room_list = models.Room.objects.all()  # queryset
    paginator = Paginator(room_list, 10, orphans=5)
    try:
        rooms = paginator.page(int(page))
        return render(request, "rooms/home.html", {"pages": rooms})
    except EmptyPage:
        return redirect("/")

# RoomDetail (Generic DetailView)와 동일
def room_detail(request, pk):
    try:
        room = models.Room.objects.get(pk=pk)
        return render(request, "rooms/detail.html", context={"room": room})
    except models.Room.DoesNotExist:
        raise Http404()

# RoomSearch (Generic SearchView)와 동일
def search(request):
    city = request.GET.get("city", "Anywhere")
    city = str.capitalize(city)
    country = request.GET.get("country", "KR")
    room_type = int(request.GET.get("room_type", 1))
    price = int(request.GET.get("price", 0))
    guests = int(request.GET.get("guests", 0))
    bedrooms = int(request.GET.get("bedrooms", 0))
    beds = int(request.GET.get("beds", 0))
    baths = int(request.GET.get("baths", 0))
    instant = bool(request.GET.get("instant", False))
    super_host = bool(request.GET.get("super_host", False))
    s_amenities = request.GET.getlist("amenities")
    s_facilities = request.GET.getlist("facilities")

    form = {
        "city": city,
        "s_country": country,
        "s_room_type": room_type,
        "price": price,
        "guests": guests,
        "bedrooms": bedrooms,
        "beds": beds,
        "baths": baths,
        "instant": instant,
        "super_host": super_host,
        "s_amenities": s_amenities,
        "s_facilities": s_facilities,
    }

    room_types = models.RoomType.objects.all()
    amenities = models.Amenity.objects.all()
    facilities = models.Facility.objects.all()

    choices = {
        "countries": countries,
        "room_types": room_types,
        "amenities": amenities,
        "facilities": facilities,
    }

    filter_args = {}
    if city != "Anywhere":
        filter_args["city__startswith"] = city
    filter_args["country"] = country
    if room_type != 0:
        filter_args["room_type__pk"] = room_type
    if price != 0:
        filter_args["price__lte"] = price
    if guests != 0:
        filter_args["guests__gte"] = guests
    if bedrooms != 0:
        filter_args["bedrooms__gte"] = bedrooms
    if beds != 0:
        filter_args["beds__gte"] = beds
    if baths != 0:
        filter_args["baths__gte"] = baths
    if instant:
        filter_args["instant_book"] = True
    if super_host:
        filter_args["host__superhost"] = True
    if len(s_amenities) > 0:
        for s_amenity in s_amenities:
            filter_args["amenities__pk"] = int(s_amenity)
    if len(s_facilities) > 0:
        for f_facility in s_facilities:
            filter_args["facilities__pk"] = int(f_facility)

    rooms = models.Room.objects.filter(**filter_args)  # search result

    return render(
        request, "rooms/search.html", context={**form, **choices, "rooms": rooms},
    )

# Django Form API를 사용한 것
def search_room(request):
    country = request.GET.get("country")
    if country:
        form = forms.SearchForm(request.GET)
        if form.is_valid():
            city = form.cleaned_data.get("city")
            country = form.cleaned_data.get("country")
            room_type = form.cleaned_data.get("room_type")
            price = form.cleaned_data.get("price")
            guests = form.cleaned_data.get("guests")
            bedrooms = form.cleaned_data.get("bedrooms")
            beds = form.cleaned_data.get("beds")
            baths = form.cleaned_data.get("baths")
            instant_book = form.cleaned_data.get("instant_book")
            super_host = form.cleaned_data.get("super_host")
            amenities = form.cleaned_data.get("amenities")
            facilities = form.cleaned_data.get("facilities")

            filter_args = {}
            if city != "Anywhere":
                filter_args["city__startswith"] = city
            filter_args["country"] = country
            if room_type is not None:
                filter_args["room_type"] = room_type
            if price is not None:
                filter_args["price__lte"] = price
            if guests is not None:
                filter_args["guests__gte"] = guests
            if bedrooms is not None:
                filter_args["bedrooms__gte"] = bedrooms
            if beds is not None:
                filter_args["beds__gte"] = beds
            if baths is not None:
                filter_args["baths__gte"] = baths
            if instant_book:
                filter_args["instant_book"] = True
            if super_host:
                filter_args["host__superhost"] = True
            for amenity in amenities:
                filter_args["amenities"] = amenity
            for facility in facilities:
                filter_args["facilities"] = facility

            rooms = models.Room.objects.filter(**filter_args)
            return render(
                request,
                "rooms/search_room.html",
                context={"form": form, "rooms": rooms},
            )
    else:
        form = forms.SearchForm()
        return render(request, "rooms/search_room.html", context={"form": form})


"""add table order, orderitems

Revision ID: 18246b9db8a9
Revises: 9f1e22d12ec9
Create Date: 2023-03-09 16:29:11.896704

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '18246b9db8a9'
down_revision = '9f1e22d12ec9'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('orders',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('customer_email', sa.String(length=50), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('order_items',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('item_id', sa.Integer(), nullable=False),
    sa.Column('order_id', sa.Integer(), nullable=False),
    sa.Column('quantity', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['item_id'], ['items.id'], ),
    sa.ForeignKeyConstraint(['order_id'], ['orders.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('order_items')
    op.drop_table('orders')
    # ### end Alembic commands ###


from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from flask import Flask, request, render_template, redirect
import os
from Google import Create_Service
import base64

app = Flask(__name__)

usernames = ['Wesley'] # Triggers

# Send e-mail with GMAIL API
def send_email(username, service):

    # Get email to send from page request
    TO = request.form["TO"]

    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = request.form["Title"].format(username)
    msg['To'] = request.form["TO"]
    send = [TO]

    # Get HTML from page request
    html = request.form["html"].format(username)       # + str(request.files['file'].read(), 'utf-8')

    # Record the MIME types with html format
    html_code = MIMEText(html.encode('utf-8'), 'html', 'utf-8')
    msg.attach(html_code)

    # Record MSG message in an RFC 2822 formatted and base64url encoded string
    raw_string = base64.urlsafe_b64encode(msg.as_bytes()).decode()

    return service.users().messages().send(userId='me', body={'raw': raw_string}).execute()

# Get user e-mail from GMAIL API
def get_user_email(service):
    get_user_email = service.users().getProfile(userId='me').execute()
    user_email = get_user_email['emailAddress''']
    return user_email

# Auth GMAIL API with OAUTH2
def oauth2_gmail_login():
    CLIENT_SECRET_FILE = 'client_secret.json'
    API_NAME = 'gmail'
    API_VERSION = 'v1'
    SCOPES = ['https://www.googleapis.com/auth/gmail.send','https://www.googleapis.com/auth/gmail.readonly']

    service = Create_Service(CLIENT_SECRET_FILE, API_NAME, API_VERSION, SCOPES)

    return service

# Home route
@app.route('/', methods=["POST", "GET"])
def home():
    if request.method == "POST":
        html = request.form["html"]
        print(html)
        for username in usernames:
            send_email(username, service=oauth2_gmail_login())
        return render_template("home.html", user_email=get_user_email(service=oauth2_gmail_login()))
    elif request.method == "GET":
        return render_template("home.html", user_email=get_user_email(service=oauth2_gmail_login()))

#Login route
@app.route('/login', methods=["POST","GET"])
def login():
    if request.method == "GET":
        return render_template("login.html")
    elif request.method == "POST":
        oauth2_gmail_login()
        return redirect('http://127.0.0.1:5000/')

if __name__ == '__main__':
    app.run(debug=True)


import matplotlib.pyplot as plt
import matplotlib
import re
prev=[-1,-1]
f=open("3.txt","r")
x=[]
y=[]
for line in f:
    tmp=re.findall("\d+.\\d+",line)
    x.append(float(tmp[0]))
    y.append(float(tmp[1]))
plt.scatter(x,y)
plt.plot(x,y)
axes = plt.gca()
axes.set_xlim([0,30])
axes.set_ylim([0,30])
plt.ylim(max(plt.ylim()), min(plt.ylim()))
# for i in range(len(x)):
    # plt.scatter(x[i],y[i])
    # if prev[0]!=-1:
        # x_l=[prev[0],x[i]]
        # y_l=[prev[1],y[i]]
        # plt.plot(x_l,y_l)
    # prev[0]=x[i]
    # prev[1]=y[1]
    # print(x[i],y[i])

plt.show()