import os
import json
import requests
from src.config import get_config
from src.mssql_db import execute_sp, get_sp_first_result_set
from src.utils import format_date

config = get_config()

def _get_custom_attributes(custom_attributes):
    custom_attrs = json.dumps(custom_attributes)
    if custom_attrs == '{}':
        custom_attrs = None

    return custom_attrs

def fetch_api_rep_data():
    """ Return the Telecom API rep json data. """
    if config['IS_PRODUCTION']:
        return fetch_api_prod_rep_data()

    return fetch_api_test_rep_data()

def fetch_api_test_rep_data():
    """ Returns JSON payload containing the test data. """
    file_path = os.path.join(config['ROOT_DIR'], 'test_data.json')
    if os.path.exists(file_path) is False:
        raise FileExistsError(f"The test data file was not found.")

    with open(file_path) as fp:
        contents = fp.read()

    return json.loads(contents)

def fetch_api_prod_rep_data():
    """ Returns JSON payload containing the production data. """
    result = requests.get(config['TELECOM_REP_ENDPOINT'])
    if result.status_code != 200:
        result.raise_for_status()

    return result.json()

def start_process():
    """ Calls the stored procedure to start the process. """
    results = execute_sp(
        'UMA_TELECOM.PROCESS_API_DATA',
        {
            'MESSAGE': 'START'
        }
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result['return_value']

def stop_process(master_load_id):
    """ Calls the stored procedure to stop the process. """
    results = execute_sp(
        'UMA_TELECOM.PROCESS_API_DATA',
        {
            'MESSAGE': 'FINISHED',
            'LOAD_HIST_PKID_IN': master_load_id
        }
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result

def save_rep(rep, master_load_id):
    """ Saves the rep data. """
    results = execute_sp(
        'UMA_TELECOM.SAVE_D_REP',
        {
            'REP_ID': rep['$id'],
            'REP_userId': rep['userId'],
            'REP_homeSite': rep['homeSite'],
            'REP_firstName': rep['firstName'],
            'REP_lastName': rep['lastName'],
            'REP_displayName': rep['displayName'],
            'REP_ntDomainUser': rep['ntDomainUser'],
            'REP_extension': rep['extension'],
            'REP_outboundANI': rep['outboundANI'],
            'REP_id_LIST': rep['id'],
            'REP_customAttributes': _get_custom_attributes(rep['customAttributes']),
            'REP_dateAdded': format_date(parse_date(rep['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        },
        out_arg='rep_id'
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result['rep_id']

def save_rep_skill(rep_id, rep_skill, master_load_id):
    """ Saves the rep skill. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_SKILL',
        {
            'D_REP_ID': rep_id,
            'REP_SKILL_ID': rep_skill['$id'],
            'REP_SKILL_displayName': rep_skill['displayName'],
            'REP_SKILL_proficiency': rep_skill['proficiency'],
            'REP_SKILL_desireToUse': rep_skill['desireToUse'],
            'REP_SKILL_id_ALTERNATE': rep_skill['id'],
            'REP_SKILL_dateAdded': format_date(parse_date(rep_skill['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def save_rep_role(rep_id, rep_role, master_load_id):
    """ Saves the rep role. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_ROLE',
        {
            'D_REP_ID': rep_id,
            'REP_ROLE_ID': rep_role['$id'],
            'REP_ROLE_roleId': rep_role['roleId'],
            'REP_ROLE_name': rep_role['name'],
            'REP_ROLE_id_ALTERNATE': rep_role['id'],
            'REP_ROLE_dateAdded': format_date(parse_date(rep_role['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def save_rep_workgroup(rep_id, rep_workgroup, master_load_id):
    """ Saves the rep workgroup. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_WORKGROUP',
        {
            'D_REP_ID': rep_id,
            'REP_WORKGROUP_ID': rep_workgroup['$id'],
            'REP_WORKGROUP_Name': rep_workgroup['name'],
            'REP_WORKGROUP_customAttributes': _get_custom_attributes(rep_workgroup['customAttributes']),
            'REP_WORKGROUP_id_ALTERNATE': rep_workgroup['id'],
            'REP_WORKGROUP_dateAdded': format_date(parse_date(rep_workgroup['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def parse_date(value):
    if value:
        return value.split('.', 1)[0]

    return ''


import logging
from datetime import datetime, date, time, timedelta

from flask import current_app

from app.data_analysis.helpers import get_daily_values_dict
from app.default.helpers import get_jobs
from app.default.models import Machine

def get_machine_quality(machine: Machine, time_start: datetime, time_end: datetime) -> float:
    """ Calculate the quality of machine output for calculating OEE"""
    jobs = get_jobs(time_start, time_end, machine=machine)
    total_machine_quantity_produced = 0
    total_machine_rejects_produced = 0
    for job in jobs:
        # I've already considered what happens if this job ran outside the requested start/end, and it would be ok,
        # All we can do is assume the job has same performance all the way through, so the ratio would still be the same
        rejects_qty = job.get_total_reject_quantity()
        good_qty = job.get_total_good_quantity()
        total_machine_quantity_produced += (good_qty + rejects_qty)
        total_machine_rejects_produced += rejects_qty
    if total_machine_quantity_produced == 0:
        return 0
    quality = (total_machine_quantity_produced - total_machine_rejects_produced) / total_machine_quantity_produced
    if quality > 1:
        logging.warning(f"Quality of >1 calculated for machine {machine.name} on {time_start.date()}")
    return quality

def get_daily_quality_dict(requested_date: date = None, human_readable=False):
    """ Return a dictionary with every machine's performance on the given date """
    quality_dict = get_daily_values_dict(get_machine_quality, requested_date)
    if human_readable:
        for k, v in quality_dict.items():
            v = v * 100
            quality_dict[k] = f"{round(v, 1)}%"
    return quality_dict


import requests
from bs4 import BeautifulSoup
from time import sleep
import pandas as pd
from timeit import default_timer as timer
from tqdm import tqdm
import random

# baselink = 'https://www.klenmarket.ru'
start = timer()

data = []
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36'}

with open("urlsAll.txt", "r") as file:
    linkList = file.readlines()
    # sleep(1)

for link in tqdm(linkList):
    r = requests.get(link.strip(), headers=headers, allow_redirects=False)
    sleep(random.randint(1, 5))
    if r.status_code != 200:
        continue
    # sleep(1)
    soup = BeautifulSoup(r.text, 'lxml')

    try:
        category1 = soup.findAll('a', class_='breadcrumbs__link')[1].text.strip().split(',')
    except:
        category1 = 'None'.split(',')
    try:
        category2 = soup.findAll('a', class_='breadcrumbs__link')[2].text.strip().split(',')
    except:
        category2 = 'None'.split(',')
    try:
        category3 = soup.findAll('a', class_='breadcrumbs__link')[3].text.strip().split(',')
    except:
        category3 = 'None'.split(',')
    try:
        category4 = soup.findAll('a', class_='breadcrumbs__link')[4].text.strip().split(',')
    except:
        category4 = 'None'.split(',')
    try:
        name = soup.find('h1').text.strip().split(',')
    except:
        name = 'None'.split(',')
    try:
        price = soup.find('span', class_='price__current-value').get('content')
    except:
        price = 'None'
    # try:
    #     code = soup.find('span', class_='origin__article').find('span', class_='text-warning').text.strip()
    # except:
    #     code = 'None'
    try:
        manufacturer = soup.find('div', class_='origin').find('a').text.strip()
    except:
        manufacturer = 'None'
    try:
        country = soup.find('span', class_='origin__country').text.strip()
    except:
        country = 'None'
    # link = 'https://stopgame.ru' + game.find('div', class_='caption').find('a').get('href')
    # score = items.find('div', class_='value').text
    # genre = items.find('div', class_='game-specs').findAll('div', class_='game-spec')[1].find('a').text
    data.append([' '.join(category1),
                 ' '.join(category2),
                 ' '.join(category3),
                 ' '.join(category4),
                 ' '.join(name),
                 price,
                 manufacturer,
                 country,
                 link.strip()])

# print(data)

#записываем полученные данные в CSV
header = ['category1',
          'category2',
          'category3',
          'category4',
          'name',
          'price',
          'manufacturer',
          'country',
          'link']
df = pd.DataFrame(data, columns=header)
df.to_csv(f'C:\\Users\d.novikov\PycharmProjects\Scrapers\Klenmarket.csv',sep=';', encoding='cp1251', errors='ignore')

end = timer()
print(end - start)


import streamlit as st
from streamlit.script_runner import RerunException
import pandas as pd
import calendar
pd.set_option("display.max_columns", None)
pd.set_option("mode.chained_assignment", None)
import plotly.express as px
from datetime import datetime
from dateutil.relativedelta import relativedelta

st.set_page_config(page_title="Bridgestone tools", layout="wide", page_icon="./materials/bs-logo-small.png")

# 設定不要顯示"Made with Stramlit"

hide_streamlit_style = """
            <style>
            #MainMenu {visibility: hidden;}
            footer {visibility: hidden;}
            </style>
            """
st.markdown(hide_streamlit_style, unsafe_allow_html=True)

### GFK數據處理視覺化

df = pd.read_csv(
    r"D:\kc.hsu\OneDrive - Bridgestone\GFK data\202201\GFK_CUSTOMREPORT_TRUCKTIRE_TW_BRIDGESTONE_Jan22.csv")

date = datetime.today().strftime("%Y-%m-%d")
date = datetime.strptime(date, "%Y-%m-%d")
interval = date - relativedelta(months=14)
interval = interval.strftime("%Y-%m")
gfk = df.copy()
gfk["Period"] = pd.to_datetime(gfk["Period"], yearfirst=True)
# gfk["Period"] = gfk["Period"].dt.to_period("M")
gfk["Period"] = gfk["Period"].dt.strftime("%Y-%m")
filtered_data = gfk[gfk["Period"] >= interval]

## Top 10 list
top10_players = ["BRIDGESTONE", "MAXXIS", "MICHELIN", "LINGLONG", "JINYUTIRES", "PRESA", "DUNLOP", "GOODYEAR",
                 "YOKOHAMA"]
filtered_data["PLAYERS"] = filtered_data["BRAND"].map(lambda x: "The others" if x not in top10_players else x)

filtered_data = filtered_data.astype({"SALES UNITS": "int", "SALES <LC>": "int", "PRICE TWD/UN.": "int"})

bar_sales_units = filtered_data.groupby(["Period", "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().reset_index()
bar_sales_units = bar_sales_units.sort_values(by="SALES UNITS", ascending=False)
bar_sales_units["in %"] = 100 * (
            bar_sales_units["SALES UNITS"] / bar_sales_units.groupby("Period")["SALES UNITS"].transform("sum"))

ttl_table = filtered_data[filtered_data["BRIDGESTONE TYPE"] != "EXCLUDED TBR AND LSR2"].groupby(["Period", "BRIDGESTONE TYPE"])[
    ["SALES UNITS"]].sum().unstack("Period").reset_index()

df = df.astype({"SALES UNITS": "int", "SALES <LC>": "int", "PRICE TWD/UN.": "int"})
df["Period"] = pd.to_datetime(df["Period"], dayfirst=True)
qt_data = df.groupby([pd.Grouper(key="Period", freq="QS"), "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().reset_index()
qt_data["Year"] = qt_data["Period"].dt.year
qt_data["Quarter"] = qt_data["Period"].dt.quarter
qt_data = qt_data.astype({"Year": "str", "Quarter": "str"})
qt_data["Y+Q"] = qt_data["Year"] + "Q" + qt_data["Quarter"]
qt_data["in %"] = 100 * (qt_data["SALES UNITS"] / qt_data.groupby("Period")["SALES UNITS"].transform("sum"))

ttl_qt_table = qt_data[qt_data["BRIDGESTONE TYPE"] != "EXCLUDED TBR AND LSR2"].groupby(["Y+Q", "BRIDGESTONE TYPE"])[["SALES UNITS"]].sum().unstack("Y+Q").reset_index()

### GFK文字敘述
gfk_numbers = filtered_data.groupby(["Period"])[["SALES UNITS"]].sum().reset_index()
previous_mon = int(datetime.today().strftime("%m")) - 1
before_pre_mon = previous_mon - 1
month_names = [calendar.month_name[i] for i in range(1, 13)]
months = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
month_dict = dict(zip(months, month_names))
result_description = "a. " + month_dict[previous_mon] + " vs " + month_dict[before_pre_mon] + " " + "{:.0f}%".format(100 * gfk_numbers.iloc[-1, 1] / gfk_numbers.iloc[-2, 1]) + " , vs PY " + "{:.0f}%".format(100 * gfk_numbers.iloc[-1, 1] / gfk_numbers.iloc[0, 1])

current_year = datetime.today().strftime("%Y")
# 檢查目前今年的資料是幾個月的
cum_months = len(gfk_numbers[gfk_numbers["Period"].str.contains(current_year)])
# 今年累計數量
ytd_units = gfk_numbers[gfk_numbers["Period"].str.contains(current_year)]["SALES UNITS"].sum()
# 去年累計數量
py_units = gfk_numbers[gfk_numbers["Period"].str.contains(str(cum_months - 1))]["SALES UNITS"][0:cum_months].sum()
ytd_py = "b. YTD vs PY " + "{:.0f}%".format(100 * ytd_units / py_units)

logo, space, header = st.columns([0.4, 0.4, 1])
with logo:
    st.image("./materials/bs-logo.png")

with header:
    st.header("GFK report")

st.subheader("Monthly Total sales in unit")

st.write(result_description)
st.write(ytd_py)
chart1, chart2 = st.columns([1, 0.6])
with chart1:
    st.plotly_chart(px.bar(bar_sales_units, x="Period", y="SALES UNITS", color="BRIDGESTONE TYPE",
                           text=bar_sales_units["in %"].map(lambda x: "{:.0f}%".format(x)),
                           title="Monthly truck tires sales units"))
with chart2:
    st.plotly_chart(px.bar(qt_data, x="Y+Q", y="SALES UNITS", color="BRIDGESTONE TYPE", text=qt_data["in %"].map(lambda x: "{:.0f}%".format(x)), labels={"Y+Q": "Quarter"}, title="Quarterly truck tires sales units"))

table1, table2 = st.columns([1, 0.6])
with table1:
    st.write(ttl_table)
with table2:
    st.write(ttl_qt_table)


# Algorithm from https://www.cs.cornell.edu/courses/cs3110/2011sp/Lectures/lec26-type-inference/type-inference.htm#4

class Unification(object):

    def IsVar(self, term):
        """If `term` is a variable returns its id."""
        raise ValueError('not implemented')

    def IsPred(self, term):
        """If `term` is a predicate returns a symbol and list of arguments."""
        raise ValueError('not implemented')

    def ReplaceArgs(self, pred, args):
        raise ValueError('not implemented')

    def Occurs(self, v_id, term):
        term_var = self.IsVar(term)
        if term_var is not None:
            return v_id == term_var
        else:
            _, args = self.IsPred(term)
            return any(self.Occurs(v_id, arg) for arg in args)

    def Substitute(self, s, v_id, t):
        t_var = self.IsVar(t)
        if t_var is not None:
            if t_var == v_id:
                return s
            else:
                return t
        else:
            _, args = self.IsPred(t)
            new_args = [self.Substitute(s, v_id, arg) for arg in args]
            return self.ReplaceArgs(t, new_args)

    def Apply(self, substitutions, term):
        for v_id, s in substitutions:
            term = self.Substitute(s, v_id, term)
        return term

    def UnifyOne(self, a, b):
        a_var = self.IsVar(a)
        b_var = self.IsVar(b)
        if a_var is not None and b_var is not None:
            if a_var == b_var:
                return []
            else:
                return [(a_var, b)]
        if a_var is None and b_var is None:
            a_sym, a_args = self.IsPred(a)
            b_sym, b_args = self.IsPred(b)
            if a_sym != b_sym:
                raise ValueError('Cannot unify: conflicting symbols %s and %s' % (a_sym, b_sym))
            if len(a_args) != len(b_args):
                raise ValueError('Cannot unify: conflicting arities %s and %s for %s' % (len(a_args), len(b_args), a_sym))
            return self.Unify(list(zip(a_args, b_args)))

        if a_var is None:
            # a is predicate and b is variable.
            if self.Occurs(b_var, a):
                raise ValueError('Cannot unify: circularity %s = %s' % (a, b_var))
            return [(b_var, a)]

        # a is variable and b is predicate.
        if self.Occurs(a_var, b):
            raise ValueError('Cannot unify: circularity %s = %s' % (a_var, b))
        return [(a_var, b)]

    def Unify(self, equations):
        result = []
        for (a, b) in equations:
            subst = self.UnifyOne(self.Apply(result, a), self.Apply(result, b))
            result += subst
        return result


# -*- coding: utf-8 -*-

from zope.interface import implements
from collective.tablepage.interfaces import IFieldValidator
from ospfe.singole_procedure import _

class ValidatorCIG(object):
    """A string, max 10 chars"""
    implements(IFieldValidator)

    def __init__(self, field):
        self.field = field

    def validate(self, configuration, data=None):
        value = data or self.field.request.form.get(configuration['id'])
        if value and len(value)>10:
            return _('error_bad_cig', default='CIG must be a value of no more than 10 chars')

class ValidatorMax250Chars(object):
    """A string, max 250 chars"""
    implements(IFieldValidator)

    def __init__(self, field):
        self.field = field

    def validate(self, configuration, data=None):
        col_id = configuration['id']
        value = data or self.field.request.form.get(configuration['id'])
        if value and len(value)>250:
            return _('error_max_chars',
                     default='The value for "$name" must contain no more than 250 characters ($count provided).',
                     mapping={'name': configuration.get('label', col_id).decode('utf-8'),
                              'count': len(value)})


import numpy as np
import pickle
import seaborn as sns

epsilon = 1e-8
beta1 = 0.9
beta2 = 0.999

def ReLU(z):
    return z * (z > 0)

def ReLU_prime(z):
    return 1. * (z > 0)

def sigmoid(z):
    return 1/(1+np.exp(-z))

def sigmoid_prime(z):
     return sigmoid(z)*(1-sigmoid(z))

def linear(z):
    return z

def linear_prime(z):
    return z >= 0

def tanh(z):
    return (np.exp(z) - np.exp(-z)) / (np.exp(z) + np.exp(-z))

def tanh_prime(z):
    return 1 - (tanh(z))**2

def softplus(z):
    return np.log(1 + np.exp(z))

def softplus_prime(z):
    return sigmoid(z)

def softmax(z):
    e_z = np.exp(z - np.max(z, axis=0))
    return e_z / np.sum(e_z, axis=0, keepdims=True)

def cost_binary_cross_entropy(Y_hat, Y, weights=None):
    if weights is None:
        weights = np.ones((2,1))
    m = Y.shape[1]
    #print(str(weights[0]), str(weights[1]))
    return - 1/m * np.sum(Y*np.log(Y_hat + epsilon) * weights[0] + (1 - Y) * np.log(1 - Y_hat + epsilon) * weights[1])

def cost_binary_cross_entropy_derivative(AL, Y, weights=None):
    if weights is None:
        weights = np.ones((2,1))
    #print(str(weights[0]), str(weights[1]))
    return -Y/(AL + epsilon) * weights[0] + (1-Y)/(1 - AL + epsilon) * weights[1]

def non_binary_cross_entropy(Y_hat, Y):
    return - 1/Y_hat.shape[1] * np.sum(Y * np.log(Y_hat + epsilon))

def non_binary_cross_entropy_derivative(AL, Y):
    return AL - Y

def cost_mse(Y_hat, Y):
    m = Y.shape[1]
    return 1/(2*m) * np.sum(Y_hat - Y)**2

def cost_mse_derivatie(AL, Y):
    return AL - Y

# return activation function by given name
def get_activation_function_by_name(name):
    if name == 'sigmoid':
        return sigmoid
    if name == 'ReLU':
        return ReLU
    if name == 'linear':
        return linear
    if name == 'tanh':
        return tanh
    if name == 'softplus':
        return softplus
    if name == 'softmax':
        return softmax

def get_activation_derivative_function_by_name(name):
    if name == 'sigmoid':
        return sigmoid_prime
    if name == 'ReLU':
        return ReLU_prime
    if name == 'linear':
        return linear_prime
    if name == 'tanh':
        return tanh_prime
    if name == 'softplus':
        return softplus_prime

def cost_function_by_name(name, binary=False):
    if name == 'mse':
        return cost_mse
    if name == 'cross-entropy' and binary:
        return cost_binary_cross_entropy
    if name == 'cross-entropy' and not binary:
        return non_binary_cross_entropy

def cost_function_derivative_by_name(name, binary=False):
    if name == 'mse':
        return cost_mse_derivatie
    if name == 'cross-entropy' and binary:
        return cost_binary_cross_entropy_derivative
    if name == 'cross-entropy' and not binary:
        return non_binary_cross_entropy_derivative

def sgn(x):
    return x and (1, -1)[x < 0]

class NeuralNetwork:
    # input_units_count - size of one input vector (input_units_count, 1)
    def __init__(self, input_units_count):

        # the number of neurons in each layer
        self.layers_structure = []
        self.layers_structure.append(input_units_count)

        # next added layer will have l = 1
        self.current_number = 1

        # dictionary of activation functions
        self.activation_functions = {}
        self.activation_function_derivatives = {}

        # dictionary of weights and biases
        self.parameters = {}
        # dictionary of betas and gammas
        self.batch_norm_parameters = {}
        self.batch_norm_caches = {}
        # dictionary for mu and sigma, calculated by mini-batches
        self.batch_biases = {}

        # cost function initialize with applying model
        self.cost_function = None
        self.cost_function_derivative = None

        self.isSoftmax = False

        # Regularization
        self.L2 = 0
        self.L1 = 0

        np.random.seed(4)

    def add(self, count_of_units, name_of_activation, batch_normalization=False):
        self.layers_structure.append(count_of_units)

        # adds function to dictionary
        self.activation_functions['g' + str(self.current_number)] = get_activation_function_by_name(name_of_activation)
        self.activation_function_derivatives['dg' + str(self.current_number)] = \
            get_activation_derivative_function_by_name(name_of_activation)

        gamma = None
        beta = None
        # initialize parameters for batch normalization
        if batch_normalization:
            gamma = np.random.randn(count_of_units, 1)
            beta = np.zeros((count_of_units, 1))

        self.batch_norm_parameters['gamma' + str(self.current_number)] = gamma
        self.batch_norm_parameters['beta' + str(self.current_number)] = beta

        self.current_number += 1

    # user should call this method after adding layers. Weights and biases will be generated here
    def apply_model(self, cost_function_name):
        # L - count of hidden layers + output layer
        L = len(self.activation_functions)

        for i in range(1, L + 1):
            self.parameters['W' + str(i)] = np.random.randn(self.layers_structure[i], self.layers_structure[i-1]) #* 0.01
            self.parameters['b' + str(i)] = np.zeros((self.layers_structure[i], 1))

        self.cost_function = cost_function_by_name(cost_function_name, binary=self.activation_functions['g' + str(L)] == sigmoid)
        self.cost_function_derivative = cost_function_derivative_by_name(cost_function_name,
                                                                         binary=self.activation_functions['g' + str(L)] == sigmoid)

        if self.activation_functions['g' + str(L)] == softmax:
            self.isSoftmax = True

    # needed for cross-validation. reset all weights, biases, gammas and betas
    def __reset_model_parameters__(self):
        # L - count of hidden layers + output layer
        L = len(self.activation_functions)

        for i in range(1, L + 1):
            self.parameters['W' + str(i)] = np.random.randn(self.layers_structure[i], self.layers_structure[i-1]) #* 0.01
            self.parameters['b' + str(i)] = np.zeros((self.layers_structure[i], 1))

            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                self.batch_norm_parameters['gamma' + str(i)] = np.random.randn(self.layers_structure[i], 1)
                self.batch_norm_parameters['beta' + str(i)] = np.zeros((self.layers_structure[i], 1))

    # compute output of NN for given set X
    # if train_prop=False => mu and sigma^2 should be used by mini-batches
    def __forward_prop__(self, X, t, train_prop=True):
        L = len(self.parameters) // 2
        A = X
        zs = []
        activations = [A]
        for i in range(1, L+1):
            A_prev = A
            Wi = self.parameters['W' + str(i)]
            bi = self.parameters['b' + str(i)]

            Z = np.dot(Wi, A_prev) + bi

            # check for batch_normalization
            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                # if first mini-batch
                # if t == 1:
                #     self.batch_biases['mu' + str(i)] = np.zeros(bi.shape)
                #     self.batch_biases['sigma' + str(i)] = np.zeros(bi.shape)

                Z = self.__batch_norm_forward__(Z, i, train_prop, t)

            A = self.activation_functions['g' + str(i)](Z)

            zs.append(Z)
            activations.append(A)

        return activations[-1], zs, activations

    # batch-normalization helper for forward propagation
    # z - vector to normalize, l - number of layer
    def __batch_norm_forward__(self, z, l, train_prop, t):
        gamma = self.batch_norm_parameters['gamma' + str(l)]
        beta = self.batch_norm_parameters['beta' + str(l)]

        if train_prop:
            mu = 1 / z.shape[1] * np.sum(z, axis=1, keepdims=True)

            imean = (z - mu)

            sigma = 1 / z.shape[1] * np.sum(imean ** 2, axis=1, keepdims=True)

            ivar = 1 / (np.sqrt(sigma + epsilon))

            z_hat = imean * ivar

            self.batch_biases['mu' + str(l)] = mu
            self.batch_biases['ivar' + str(l)] = ivar
        else:
            mu = self.batch_biases['mu' + str(l)]
            ivar = self.batch_biases['ivar' + str(l)]
            imean = None

            z_hat = (z-mu) * ivar

        y = gamma * z_hat + beta

        cache = (imean, ivar, gamma, beta, z_hat)

        self.batch_norm_caches[str(l)] = cache

        return y

    # calculates cost function
    def __calculate_cost_function__(self, Y_hat, Y, weights):
        return self.cost_function(Y_hat, Y, weights)

    # implementation of back_propagation
    # returns dictionary with dW[1], db[1],...,dW[L], db[L]
    def __back_prop__(self, dAL, zs, activations):

        L = len(activations)
        dAL_copy = dAL
        m = dAL.shape[1]

        # result dictionary with gradients
        grads = {}
        for i in range(1, L):
            if self.activation_functions['g' + str(L - i)] != softmax:
                dZ = dAL_copy * self.activation_function_derivatives['dg' + str(L - i)](zs[- i])
            else:
                dZ = dAL_copy

            if self.batch_norm_parameters['gamma' + str(L-i)] is not None:
                dZ, dgamma, dbeta = self.__batch_norm_backward__(dZ, L-i)
                grads['dgamma' + str(L - i)] = dgamma
                grads['dbeta' + str(L - i)] = dbeta

            dWL = 1 / m * np.dot(dZ, activations[-i - 1].T)
            grads['dW' + str(L - i)] = dWL
            grads['db' + str(L - i)] = 1/m * np.sum(dZ, axis=1, keepdims=True)
            dAL_copy = np.dot(self.parameters['W' + str(L - i)].T, dZ)

            # Regularization adding to dW
            if(self.L2 > 0):
                #grads['dW' + str(L - i)] += (self.L2 / m * self.parameters['W' + str(L - i)])
                grads['dW' + str(L - i)] += (self.L2 * self.parameters['W' + str(L - i)])

            if(self.L1 > 0):
                for k in range(0, self.parameters['W' + str(L - i)].shape[0]):
                    for j in range(0, self.parameters['W' + str(L - i)].shape[1]):
                        #grads['dW' + str(L - i)] += self.L1/m * sgn(self.parameters['W' + str(L - i)][k][j])
                        grads['dW' + str(L - i)] += self.L1 * sgn(self.parameters['W' + str(L - i)][k][j])

        return grads

    # calculating dZ, dBeta, dGamma after batch_normalization
    def __batch_norm_backward__(self, dZ, l):
        m = dZ.shape[1]

        imean, ivar, gamma, beta, z_hat = self.batch_norm_caches[str(l)]

        dx_hat = dZ * gamma

        dsigma = np.sum(dx_hat * imean * (-0.5) * ivar ** 3, axis=1, keepdims=True)

        dmu = np.sum(dx_hat * (-1)*ivar) + dsigma * (-2)/m * np.sum(imean, axis=1, keepdims=True)

        dZ_result = dx_hat*ivar + dsigma * 2/m * imean + dmu/m

        dgamma = np.sum(dZ * z_hat, axis=1, keepdims=True)
        dbeta = np.sum(dZ, axis=1, keepdims=True)

        return dZ_result, dgamma, dbeta

    # update parameters (weights an biases) with given gradients and learning rate
    def __update_parameters__(self, grads, learning_rate):
        L = len(self.parameters) // 2
        for i in range(1, L + 1):
            self.parameters['W' + str(i)] = self.parameters['W' + str(i)] - learning_rate * grads['dW' + str(i)]
            self.parameters['b' + str(i)] = self.parameters['b' + str(i)] - learning_rate * grads['db' + str(i)]

            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                self.batch_norm_parameters['gamma' + str(i)] =  \
                    (self.batch_norm_parameters['gamma' + str(i)] - learning_rate * grads['dgamma' + str(i)])
                self.batch_norm_parameters['beta' + str(i)] = \
                    (self.batch_norm_parameters['beta' + str(i)] - learning_rate * grads['dbeta' + str(i)])

    def get_regularization_adding_to_cost(self, m):
        sum_of_reg = 0
        # L2
        if(self.L2 > 0):
            for i in range(1, len(self.parameters) // 2 + 1):
                W = self.parameters['W' + str(i)]
                sum_of_reg += (self.L2/(2*m) * np.sum(np.dot(W.T, W)))

        # L1
        if(self.L1 > 0):
            for i in range(1, len(self.parameters) // 2 + 1):
                W = self.parameters['W' + str(i)]

                sum_of_reg += (self.L1/m * np.sum(np.abs(W)))

        return sum_of_reg

    # Initialize of V and S parameters for adam optimizer
    def __init_adam_parameters__(self):
        L = len(self.parameters) // 2

        # Vdw and Vdb parameters dictionary for all layer
        v = {}
        # the same with Sdw and Sb
        s = {}

        for i in range(1, L + 1):
            v['dW' + str(i)] = np.zeros(self.parameters['W' + str(i)].shape)
            v['db' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)

            s['dW' + str(i)] = np.zeros(self.parameters['W' + str(i)].shape)
            s['db' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)

            # initializing batch_norm parameters for all neuron in current layer if needed
            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                v['dgamma' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)
                v['dbeta' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)

                s['dgamma' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)
                s['dbeta' + str(i)] = np.zeros(self.parameters['b' + str(i)].shape)

        # returns dictionaries with initialized by zeros parameters
        return v, s

    # train model with
    # X_train - train features, Y_train - answers for train set
    # L1 - L1 regularization, L2 - L2 regularization
    # X_test, Y_test - test set for monitoring overfitting
    def train(self, X_train, Y_train, learning_rate, epochs, L1=0, L2=0, X_test=None, Y_test=None,
              output=True, mini_batch_size=None, optimizer='sgd', lr_decay=1, weights=None):
        count_epochs = []
        train_costs = []
        test_costs = []
        # only for non-binary classifictaion
        if (Y_test is not None) and (self.isSoftmax):
            Y_test_args = np.argmax(Y_test, axis=0)
        else:
            Y_test_args = Y_test

        v = None
        s = None

        if optimizer == 'adam':
            v, s = self.__init_adam_parameters__()

        # Regularization
        self.L2 = L2
        self.L1 = L1

        max_right_answ = 0
        t = 0

        # if batch-size is not defined, then all train data is mini-batch
        if mini_batch_size is None:
            mini_batch_size = X_train.shape[1]

        t = 0

        # reducing of learning rate
        current_learning_coef = lr_decay
        corrected_lr = learning_rate
        for i in range(1, epochs + 1):
            t += 1
            corrected_lr *= current_learning_coef
            # preparing mini-batch
            #stacked_data = np.row_stack((X_train, Y_train)).T
            #np.random.shuffle(stacked_data)
           # mini_batches = [
            #    stacked_data[k: k + mini_batch_size]
            #    for k in range(0, X_train.shape[1], mini_batch_size)]
            # number of current mini-batch

            #for current_mini_batch in mini_batches:
                #current_mini_batch = current_mini_batch.T

                #X_batch = np.array(current_mini_batch[0: X_train.shape[0]])
                #Y_batch = np.array(current_mini_batch[X_train.shape[0]: X_train.shape[0]+Y_train.shape[0]])

                # AL - predictions on train set
                # zs, activations - cache for easier computing of backprop
            AL, zs, activations = self.__forward_prop__(X_train, t)

            dAL = self.cost_function_derivative(AL, Y_train, weights)

            # select suitable optimizer to update weights and biases
            if optimizer == 'sgd':
                self.__sgd_optimize__(dAL=dAL, zs=zs, activations=activations, learning_rate=corrected_lr)
            elif optimizer == 'adam':
                v, s = self.__adam_optimize__(dAL=dAL, zs=zs,activations=activations,learning_rate=corrected_lr,
                                   t=t, v=v, s=s)

            AL, zs, activations = self.__forward_prop__(X_train, -1, False)
            cost_train = self.__calculate_cost_function__(AL, Y_train, weights)

            # Adding regularization to cost function (also should be added to gradient)
            regularization = self.get_regularization_adding_to_cost(AL.shape[1])
            #cost_train += regularization

            count_epochs.append(i)
            train_costs.append(cost_train)

            if output:
                if X_test is not None:
                    AL_test, zs_test, act_test = self.__forward_prop__(X_test, -1, False)
                    cost_test = self.__calculate_cost_function__(AL_test, Y_test, weights)
                    #cost_test += regularization

                    predicted = self.predict(X_test)
                    right_answers = 0
                    if not self.isSoftmax:
                        for j in range(0, predicted.shape[1]):
                            right_answers += predicted[0][j] == Y_test_args[0][j]
                    else:
                        for j in range(0, predicted.shape[0]):
                            right_answers += predicted[j] == Y_test_args[j]

                    if right_answers > max_right_answ:
                        max_right_answ = right_answers

                    test_costs.append(cost_test)
                    if self.isSoftmax:
                        predicted_all_count = predicted.shape[0]
                    else:
                        predicted_all_count = predicted.shape[1]
                    print("Epoch %i: Train cost: %f  Test cost: %f, right answers: %i/%i"
                          %(i, cost_train, cost_test, right_answers, predicted_all_count))
                else:
                    print("Epoch %i: Train cost: %f " % (i, cost_train))

        return count_epochs, train_costs, test_costs, max_right_answ

    # SGD optimizer
    def __sgd_optimize__(self, dAL, zs, activations, learning_rate):
        grads = self.__back_prop__(dAL=dAL, zs=zs, activations=activations)
        self.__update_parameters__(grads=grads, learning_rate=learning_rate)

    # adam optimizer returns new values of v and s
    def __adam_optimize__(self, dAL, zs, activations, learning_rate, t, v, s):
        grads = self.__back_prop__(dAL=dAL, zs=zs, activations=activations)

        L = len(self.parameters) // 2

        # bias corrected dictionary of v and s
        v_cr = {}
        s_cr = {}

        for i in range(1, L + 1):
            # calculating of exponentially weight averages for V
            v['dW' + str(i)] = beta1 * v['dW' + str(i)] + (1 - beta1) * grads['dW' + str(i)]
            v['db' + str(i)] = beta1 * v['db' + str(i)] + (1 - beta1) * grads['db' + str(i)]
            # The same with S
            s['dW' + str(i)] = beta2 * s['dW' + str(i)] + (1 - beta1) * (grads['dW' + str(i)]**2)
            s['db' + str(i)] = beta2 * s['db' + str(i)] + (1 - beta1) * (grads['db' + str(i)]**2)

            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                v['dgamma' + str(i)] = beta1 * v['dgamma' + str(i)] + (1 - beta1) * grads['dgamma' + str(i)]
                v['dbeta' + str(i)] = beta1 * v['dbeta' + str(i)] + (1 - beta1) * grads['dbeta' + str(i)]

                s['dgamma' + str(i)] = beta2 * s['dgamma' + str(i)] + (1 - beta2) * (grads['dgamma' + str(i)] ** 2)
                s['dbeta' + str(i)] = beta2 * s['dbeta' + str(i)] + (1 - beta2) * (grads['dbeta' + str(i)] ** 2)

            # calculation of corrected values
            v_cr['dW' + str(i)] = v['dW' + str(i)] / (1 - beta1 ** t)
            v_cr['db' + str(i)] = v['db' + str(i)] / (1 - beta1 ** t)

            s_cr['dW' + str(i)] = s['dW' + str(i)] / (1 - beta2 ** t)
            s_cr['db' + str(i)] = s['db' + str(i)] / (1 - beta2 ** t)

            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                v_cr['dgamma' + str(i)] = v['dgamma' + str(i)] / (1 - beta1 ** t)
                v_cr['dbeta' + str(i)] = v['dbeta' + str(i)] / (1 - beta1 ** t)

                s_cr['dgamma' + str(i)] = s['dgamma' + str(i)] / (1 - beta2 ** t)
                s_cr['dbeta' + str(i)] = s['dbeta' + str(i)] / (1 - beta2 ** t)

            # updating of weights
            self.parameters['W' + str(i)] = self.parameters['W' + str(i)] - learning_rate * \
                                            (v_cr['dW' + str(i)] / (np.sqrt(s_cr['dW' + str(i)]) + epsilon))
            self.parameters['b' + str(i)] = self.parameters['b' + str(i)] - learning_rate * \
                                            (v_cr['db' + str(i)] / (np.sqrt(s_cr['db' + str(i)]) + epsilon))

            if self.batch_norm_parameters['gamma' + str(i)] is not None:
                self.batch_norm_parameters['gamma' + str(i)] = self.batch_norm_parameters['gamma' + str(i)] - \
                    learning_rate * (v_cr['dgamma' + str(i)] / (np.sqrt(s_cr['dgamma' + str(i)]) + epsilon))
                self.batch_norm_parameters['beta' + str(i)] = self.batch_norm_parameters['beta' + str(i)] - \
                    learning_rate * (v_cr['dbeta' + str(i)] / (np.sqrt(s_cr['dbeta' + str(i)]) + epsilon))

        return v, s

    # make prediction for given X
    def predict(self, X):
        AL, zs, act = self.__forward_prop__(X, t=-1, train_prop=False)
        L = len(self.parameters) // 2
        # if net has sigmoid activation at Lth layer => binary classification
        if self.activation_functions['g' + str(L)] == sigmoid:
            return AL >= 0.5
        # regression
        elif self.activation_functions['g' + str(L)] == ReLU or self.activation_functions['g' + str(L)] == linear:
            return AL
        else:
            return np.argmax(AL, axis=0)

    # save current model to the path /saved models/<file_name>
    def save_to_file(self, file_name):
        serialized_model = pickle.dumps(self)
        file = open('saved models/' + file_name, 'wb')
        file.write(serialized_model)
        file.close()
        # loaded_nn = pickle.loads(c)

    # reads model from file, deserialize it and returns object
    @staticmethod
    def read_model_from_file(file_name):
        file = open('saved models/' + file_name, 'rb')
        serialized_model = file.read()
        file.close()
        deserialized_model = pickle.loads(serialized_model)
        return deserialized_model

    # show heatmap with weight matrix for given layer
    def draw_heatmap(self, layer_to_vizualize=1):
        sns.heatmap(self.parameters['W' + str(layer_to_vizualize)])

    def cross_validation(self, X, Y, learning_rate, epochs, L1=0, L2=0,
              output=True, mini_batch_size=None, optimizer='sgd', count_range=5, weights=None):
        test_begin = 0
        test_end = X.shape[1] // count_range

        test_cost_avg = 0
        for i in range(count_range):
            if test_end > X.shape[1]:
                test_end = X.shape[1]

            self.__reset_model_parameters__()
            # copy X and Y (test range will drop)
            X_train = np.copy(X)
            Y_train = np.copy(Y)

            test_range = np.arange(test_begin, test_end)
            X_test = np.take(X, test_range, axis=1)
            Y_test = np.take(Y, test_range, axis=1)

            print("sum in test set: " + str(np.sum(Y_test)))

            X_train = np.delete(X_train, test_range, axis=1)
            Y_train = np.delete(Y_train, test_range, axis=1)

            epoch, train_cost, test_cost, max_right = \
                self.train(X_train, Y_train, learning_rate, epochs, L1, L2, X_test,
                           Y_test, output, mini_batch_size, optimizer,weights=weights)

            test_cost_avg += test_cost[-1]

            test_begin = test_end
            test_end += len(test_range)

        return test_cost_avg/count_range

    def get_parameters(self):
        return self.parameters


#import tensorflow as tf
import pandas as pd
from . import data_load_100hz
import numpy as np
from sklearn.preprocessing import LabelEncoder, OneHotEncoder

class Preprocess:
    def __init__(self):
        pass

    def delete_nan_from_data(self, data_100hz, label):
        indexes = []
        for index, data in enumerate(data_100hz):
            if 'Snow' not in data and 'Asphalt' not in data and 'Ice' not in data and 'Mu Change' not in data and 'Left = Asphalt, Right = Ice' not in data and 'Left = Ice, Right = Asphalt' not in data:
                indexes.append(index)
        data_100hz = np.array(data_100hz)

        deleted_100hz = np.delete(data_100hz, indexes, axis=0)

        deleted_100hz = np.delete(deleted_100hz, -1, axis=1)

        deleted_label = np.delete(np.array(label), indexes, axis=0)
        return deleted_100hz, deleted_label

    def delete_nan_from_100hz(self, data_100hz, label):
        indexes = []
        for index, data in enumerate(data_100hz):
            if 'Snow' not in data and 'Asphalt' not in data and 'Ice' not in data and 'Left = Ice, Right = Asphalt' not in data:
                indexes.append(index)
        data_100hz = np.array(data_100hz)
        deleted_100hz = np.delete(data_100hz, indexes, axis=0)

        deleted_100hz = np.delete(deleted_100hz, -1, axis=1)

        deleted_label = np.delete(np.array(label), indexes, axis=0)
        return deleted_100hz, deleted_label

    def one_hot_encoding(self, labels):
        le = LabelEncoder()
        result = []
        for label in labels:
            for l in label:
                result.append(l)
        le.fit(result)
        label_encoded = le.transform(result)

        oe = OneHotEncoder(sparse = False)
        integer_encoded = label_encoded.reshape(len(label_encoded), 1)
        onehot_encoded = oe.fit_transform(integer_encoded)

        return label_encoded, onehot_encoded


import random
import sys
import time
from game import constants
from game.action import Action
from game.point import Point
from asciimatics.event import KeyboardEvent

class HandleCollisionsAction(Action):
    """A code template for handling collisions. The responsibility of this
    class of objects is to update the game state when actors collide.

    Stereotype:
        Controller

    Attributes:
        game_state (Game_State): an instance of the game_state object
    """
    def __init__(self,game_state):
        self.game_state = game_state

    def execute(self, cast):
        """Executes the action using the given actors.

        Args:
            cast (dict): The game actors {key: tag, value: list}.
        """

        if self.game_state.get_game_state():
            cast["mid game label"][0].set_text("")

        paddle_coords = []
        paddle_x = cast["paddle"][0].get_position().get_x()
        for x in range(paddle_x,paddle_x + 12):
            paddle_coords.append(x)

        #score label = cast["score label"][0] # there's only one
        paddle = cast["paddle"][0] # there's only one
        ball = cast["ball"][0]
        ball_x = ball.get_position().get_x()
        ball_y = ball.get_position().get_y()

        paddle_position = paddle.get_position()
        paddle_x = paddle_position.get_x()
        paddle_y = paddle_position.get_y()

        #Paddle impact
        if ball_y == 18:
            if ball_x in paddle_coords:
                ball_velocity = ball.get_velocity()
                ball_x = ball_velocity.get_x()
                ball_y = ball_velocity.get_y()
                new_ball_y = ball_y - 2
                ball.set_velocity(Point(ball_x,new_ball_y))

        #Hitting the top wall
        if ball.get_position().get_y() == 1:

            ball_velocity = ball.get_velocity()
            ball_x = ball_velocity.get_x()
            ball_y = ball_velocity.get_y()
            new_ball_y = - ball_y
            ball.set_velocity(Point(ball_x,new_ball_y))

        #Hitting the bottom wall
        if ball.get_position().get_y() == constants.MAX_Y - 1:
            if cast["lives label"][0].get_lives() > 0:
                cast["lives label"][0].update_lives(-1)
                cast["mid game label"][0].set_text("Press the space bar to continue")
                ball.set_position(Point(40,18))
                cast["paddle"][0].set_position(Point(35,19))
                self.game_state.set_game_state(False)

            else:
                self.game_state.set_game_state("Game Over")

        #Hitting the left wall
        if ball.get_position().get_x() == 1:
            ball_velocity = ball.get_velocity()
            new_ball_x = ball_velocity.get_x() + 2
            ball_y = ball_velocity.get_y()

            ball.set_velocity(Point(new_ball_x,ball_y))

        #Hitting the right wall
        if ball.get_position().get_x() == constants.MAX_X - 1:
            ball_velocity = ball.get_velocity()
            new_ball_x = - ball_velocity.get_x()
            ball_y = ball_velocity.get_y()
            ball.set_velocity(Point(new_ball_x,ball_y))

        #Brick impact
        for brick in cast["brick"]:

            brick_x = brick.get_position().get_x()
            brick_y = brick.get_position().get_y()

            if ball.get_position().get_x() == brick_x:
                if ball.get_position().get_y() == brick_y:
                    ball_velocity = ball.get_velocity()
                    ball_x = ball_velocity.get_x()
                    ball_y = ball_velocity.get_y()
                    new_ball_y = -ball_y
                    ball.set_velocity(Point(ball_x,new_ball_y))

                    cast["brick"].pop(cast["brick"].index(brick))
                    cast["score label"][0].update_score(1)


class Solution:
    def twoSum(self, numbers,target):
        left,right = 0,len(numbers)-1
        while (left<right):
            currentSum = numbers[left] + numbers[right]
            if currentSum == target:
                return left+1,right+1
            elif currentSum > target:
                right -= 1
            else:
                left += 1


# coding=utf-8

import sys
from pandas import DataFrame
sys.path.append('../BaseModule')
sys.path.append("../../Base/DataReceiver")
sys.path.append("../../Base")
# import PreProcessing as pp
import LocalReceiver as lr
# import DataSampling as ds
# import traceback
from read_cnf import get_conf_info
from pandas import DataFrame
import numpy as np
from collections import Counter

if __name__ == '__main__':

    conf_info = get_conf_info()
    raw_data = lr.load_local_data(conf_info["raw_data"])  # get original data
    print(DataFrame(raw_data))

    percentile = []
    for idx in range(raw_data.shape[0]):
        column = raw_data[idx, :]
        percentile.append(1-sum(np.isnan(column))/float(len(column)))

    # print(percentile)

    cnt_dict = {}
    thresholds = [x/100 for x in range(101)]
    for item in percentile:
        for i in range(100):
            if thresholds[i] < item <= thresholds[i+1]:
                cnt_dict[(thresholds[i], thresholds[i+1])] = cnt_dict.get((thresholds[i], thresholds[i+1]), 0) + 1
                break

    cnt_dict = sorted(cnt_dict.items(), key=lambda d: d[0][0])
    print(cnt_dict)


import os.path
import time
from PIL import Image
from threading import Thread
import cv2
import os
from flask import Flask,request,render_template
from datetime import date
from datetime import datetime
import numpy as np
import pandas as pd

#### Defining Flask App
app = Flask(__name__)

#### Saving Date today in 2 different formats
datetoday = date.today().strftime("%m_%d_%y")
datetoday2 = date.today().strftime("%d-%B-%Y")

#### Initializing VideoCapture object to access WebCam
face_detector = cv2.CascadeClassifier('static/haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)

#### If these directories don't exist, create them
if not os.path.isdir('Attendance'):
    os.makedirs('Attendance')
if not os.path.isdir('datasets'):
    os.makedirs('datasets')
if f'Attendance-{datetoday}.csv' not in os.listdir('Attendance'):
    with open(f'Attendance/Attendance-{datetoday}.csv','w') as f:
        f.write('Name,Roll,Time')

#### get a number of total registered users
def totalreg():
    return len(os.listdir('datasets'))

# -------------- image labesl ------------------------

def getImagesAndLabels(path):
    # create empth face list
    faces = []
    # create empty ID list
    Ids = []
    for folder in os.listdir(path):
        # get the path of all the files in the folder
        folder_path = os.path.join(path,folder)
        imagePaths = [os.path.join(folder_path, f) for f in os.listdir(folder_path)]
        # now looping through all the image paths and loading the Ids and the images
        for imagePath in imagePaths:
            # loading the image and converting it to gray scale
            pilImage = Image.open(imagePath).convert('L')
            # Now we are converting the PIL image into numpy array
            imageNp = np.array(pilImage, 'uint8')
            # getting the Id from the image
            Id = int(os.path.split(imagePath)[-1].split(".")[1])
            # extract the face from the training image sample
            faces.append(imageNp)
            Ids.append(Id)
    return faces, Ids

#### Extract info from today's attendance file in attendance folder
def extract_attendance():
    df = pd.read_csv(f'Attendance/Attendance-{datetoday}.csv')
    names = df['Name']
    rolls = df['Roll']
    times = df['Time']
    l = len(df)
    return names,rolls,times,l

#### Add Attendance of a specific user
def add_attendance(id):
    for name in os.listdir('datasets'):
        if name.split('_')[1] == id:
            username = name.split('_')[0]
            userid = name.split('_')[1]
            current_time = datetime.now().strftime("%H:%M:%S")

            df = pd.read_csv(f'Attendance/Attendance-{datetoday}.csv')
            if int(userid) not in list(df['Roll']):
                with open(f'Attendance/Attendance-{datetoday}.csv', 'a') as f:
                    f.write(f'\n{username},{userid},{current_time}')

### A function which trains the model on all the faces available in faces folder
def train_model():
    # ----------- train images function ---------------
    recognizer = cv2.face.LBPHFaceRecognizer_create()
    harcascadePath = "static/haarcascade_frontalface_default.xml"
    detector = cv2.CascadeClassifier(harcascadePath)
    faces, Id = getImagesAndLabels("datasets")
    Thread(target=recognizer.train(faces, np.array(Id))).start()
    # Below line is optional for a visual counter effect
    Thread(target=counter_img("datasets")).start()
    recognizer.save("static/Trainner.yml")
    print("All Images")

# Optional, adds a counter for images trained (You can remove it)
def counter_img(path):
    imgcounter = 1
    imagePaths = [os.path.join(path, f) for f in os.listdir(path)]
    for imagePath in imagePaths:
        print(str(imgcounter) + " Images Trained", end="\r")
        time.sleep(0.008)
        imgcounter += 1

def get_name_from_ID(id):
    for folder in os.listdir('datasets'):
        name, str_id = folder.split('_')
        if int(str_id) == id:
            return name
#-------------------------start recognition ---------------------------------
def face_recognition():  # generate frame by frame from camera
    def draw_boundary(img, classifier, scaleFactor, minNeighbors, color, clf):
        gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        features = classifier.detectMultiScale(gray_image, scaleFactor, minNeighbors)

        for (x, y, w, h) in features:
            cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)
            id, pred = clf.predict(gray_image[y:y + h, x:x + w])
            confidence = int(100 * (1 - pred / 300))
            add_attendance(str(id))
            s = " id: " + str(id)
            name = get_name_from_ID(id) + s
            if confidence > 70:
                cv2.putText(img, name, (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, color, 1, cv2.LINE_AA)
            else:
                cv2.putText(img, "UNKNOWN", (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 1, cv2.LINE_AA)

    def recognize(img, clf, faceCascade):
        draw_boundary(img, faceCascade, 1.1, 10, (255, 255, 0), clf)

    faceCascade = cv2.CascadeClassifier("static/haarcascade_frontalface_default.xml")
    clf = cv2.face.LBPHFaceRecognizer_create()
    clf.read("static/Trainner.yml")

    wCam, hCam = 500, 400

    cap = cv2.VideoCapture(0)
    cap.set(3, wCam)
    cap.set(4, hCam)

    while True:
        ret, img = cap.read()
        recognize(img, clf, faceCascade)
        cv2.imshow('Attendance',img)
        key = cv2.waitKey(1)
        if key == 27:
            break

################## ROUTING FUNCTIONS #########################

#### Our main page
@app.route('/')
def home():
    names,rolls,times,l = extract_attendance()
    return render_template('home.html',names=names,rolls=rolls,times=times,l=l,totalreg=totalreg(),datetoday2=datetoday2)

#### This function will run when we click on Take Attendance Button
@app.route('/start',methods=['GET'])
def start():
    if 'Trainner.yml' not in os.listdir('static'):
        return render_template('home.html',totalreg=totalreg(),datetoday2=datetoday2,mess='There is no trained model in the static folder. Please add a new face to continue.')

    face_recognition()
    cap.release()
    cv2.destroyAllWindows()
    names,rolls,times,l = extract_attendance()
    return render_template('home.html',names=names,rolls=rolls,times=times,l=l,totalreg=totalreg(),datetoday2=datetoday2)

@app.route('/add',methods=['GET','POST'])
def add():
    name = request.form['newusername']
    Id = request.form['newuserid']
    userimagefolder = 'datasets/' + name + '_' + str(Id)
    if not os.path.isdir(userimagefolder):
        os.makedirs(userimagefolder)
    cam = cv2.VideoCapture(0)
    harcascadePath = 'static/haarcascade_frontalface_default.xml'
    detector = cv2.CascadeClassifier(harcascadePath)
    sampleNum = 0
    maxSample = 50

    while (True):
        ret, img = cam.read()
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = detector.detectMultiScale(gray, 1.3, 5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x, y), (x + w, y + h), (10, 159, 255), 2)
            # incrementing sample number
            sampleNum = sampleNum + 1
            # saving the captured face in the dataset folder TrainingImage
            cv2.imwrite(userimagefolder + '/' + name + "." + str(Id) + '.' + str(sampleNum) + ".jpg", gray[y:y + h, x:x + w])
            # display the frame
            cv2.imshow('frame', img)
        # wait for 100 miliseconds
        if cv2.waitKey(100) & 0xFF == ord('q'):
            break
        # break if the sample number is more than 100
        elif sampleNum >= maxSample:
            break
    cam.release()
    cv2.destroyAllWindows()
    print('Training Model')
    train_model()
    names, rolls, times, l = extract_attendance()
    return render_template('home.html', names=names, rolls=rolls, times=times, l=l, totalreg=totalreg(), datetoday2=datetoday2)

if __name__ == '__main__':
    app.run(debug=True)


from pyquery import PyQuery as pq
from proxypool.schemas.proxy import Proxy
from proxypool.crawlers.base import BaseCrawler
from loguru import logger
# import SimpleHTTPServer;
# import SocketServer;

BASE_URL = 'http://127.0.0.1:7777/proxyscrape_1000_http_proxies.txt'
# PORT = 7777; #server port
# Handler = SimpleHTTPServer.SimpleHTTPRequestHandler;
# httpd = SocketServer.TCPServer(("127.0.0.1", PORT), Handler);

class ProxyScrapeCrawler(BaseCrawler):
    """
    ipidea crawler, http://tiqu.linksocket.com:81/abroad?num=20&type=2&lb=1&sb=0&flow=1&regions=kr&n=0
    """
    urls = [BASE_URL]
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36",
    }

    @logger.catch
    def crawl(self):
        """
        crawl main method
        """
        for url in self.urls:
            logger.info(f'fetching {url}')
            with open('/Users/anonymity/Desktop/COI/proxyscrape_1000_http_proxies.txt','r') as f:
                for proxy in self.parse(f):
                    logger.info(f'fetched proxy {proxy.string()} from {url}')
                    yield proxy
            f.close()

    def parse(self, file):
        """
        parse html file to get proxies
        :return:
        """
        try:
            proxies = file.readlines()
        except ValueError:
            print('cannot load the txt file from http://127.0.0.1:7777/proxyscrape_1000_http_proxies.txt')
            exit(-1)
        for proxy in proxies:
            print(proxy)
            strls = proxy.split(':')
            yield Proxy(host=strls[0].strip(), port=strls[1].strip())

if __name__ == '__main__':
    # httpd.serve_forever();
    crawler = ProxyScrapeCrawler()
    for proxy in crawler.crawl():
        print(proxy)


#!/usr/bin/env python
# coding=utf-8

"""Helper functions"""

import re
import bert
import jieba
import logging
import feather
import operator
import unicodedata
import numpy as np
import pandas as pd
import tensorflow as tf
import multiprocessing as mp
from functools import partial
from dbfread import DBF
from opencc import OpenCC
from bert import tokenization
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder

# disable warnings in jieba packages
jieba.setLogLevel(logging.CRITICAL)

def normalize_full_width(text):
    """
    a function to normalize full width characters
    """
    return unicodedata.normalize('NFKC', text)

def sep_chi_eng(text):
    """
    a function to separate adjacent Chinese and English words from each other with a space character
    e.g. sep_chi_eng("apple手機iphone") = "apple 手機 iphone"
    """
    text = re.sub("([a-z])([\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC])", "\\1 \\2", text)
    text = re.sub("([\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC])([a-z])", "\\1 \\2", text)
    return text

def segment_chi(text):
    """
    a function to segment Chinese text using jieba
    e.g. segment_chi("蘋果手機") = "蘋果 手機"
    """
    return ' '.join(jieba.cut(text, cut_all=False))

def case_lower(text):
    """
    a function to convert English letters to lower case
    """
    return text.lower()

def remove_punct(text):
    """
    a function to remove punctuation
    """
    return re.sub("[^\w\s]", " ", text)

def detect_chi(text):
    """
    a function to return True if text containing at least 1 Chinese character, False otherwise.
    """
    return bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]", text))

class ModifiedWordPieceTokenizer(object):
    """
    a text tokenizer for performing modified Word Piece segmentation. The tokenizer is modified based on BERT's.
    It preserves the segmented Chinese instead of converting some of them to the single and special [UKN] token.

    Package required: bert, re
    """

    def __init__(self, bert_vocab_file, do_lower_case):
        self.bert_vocab_file = bert_vocab_file
        self.do_lower_case = do_lower_case
        self.tokenizer = bert.tokenization.FullTokenizer(vocab_file=self.bert_vocab_file,
                                                         do_lower_case=self.do_lower_case)

    def tokenize(self, text):
        """
        tokenize the input text

        INPUT:
            text: string
        OUTPUT:
            string
        """
        wordPieces = self.tokenizer.tokenize(text)
        tokens = text.lower().split()

        i = 0  # index for tokens
        j = 0  # index for wordPieces
        string = ''  # for storing the concatenation of wordPieces
        output = []  # empty list to store the English wordPieces and concatenated Chinese vocabularies

        for wordPiece in wordPieces:
            wordPiece = re.sub('##', '',
                               wordPiece)  # substitute the special "##" symbol for concatenation of wordPieces later on
            if wordPiece == '[UNK]':
                wordPiece = '難'  # substitute the [UNK] character for Chinese to one single arbitray Chinese word 難
            string += wordPiece  # add the wordPiece to string

            # match base on comparing length of token and string but not at textual level due to presence of [UKN]
            # character/ the arbitray character is created such that the unknown Chinese word still remain a word
            # length of 1
            if len(string) == len(tokens[i]):
                if bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]|[UNK]", string)):
                    # append token instead of string due to possible presence of unknown
                    # Chinese character in string
                    output.append(tokens[i])
                else:
                    # append English wordPieces
                    output.append(wordPieces[j])
                i += 1  # after matching 1 token, increase the token index by 1
                string = ''  # reset the string for comparison in the next token
            elif not bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]", wordPiece)):
                # append English wordPieces
                output.append(wordPieces[j])

            j += 1  # next wordPieces

        return ' '.join(output)

def text_cleaning(text_array, word_tokenization, bert_vocab_file, do_lower_case, num_threads=1):
    """
    a function to clean text, including
    - Normalize full width characters
    - Convert all English letters to lower case
    - Translate Chinese simplified characters to traditional Chinese
    - Separate Chinese and English tokens from each other
    - Segment Chinese text
    - remove punctuation
    - perform the special Word Piece Segmentation if specified by the argument "word_tokenization"

    INPUT:
        text_array: list/numpy array/pandas Series
        word_tokenization: method of word segmentation, either split by "space" or "word_piece" tokenization
        bert_vocab_file: string --path to the .txt file of vocabularies for modified Word Piece Tokenizer
        do_lower_case: boolean --whether the modified Word Piece Tokenizer converts English letters to lower case or not
        num_threads: int --number of CPU processors for performing the text cleaning
    OUTPUT:
        text_array: a list of string (cleaned text)
    """

    with mp.Pool(processes=num_threads) as pool:

        print("   Normalizing full-width characters...")
        text_array = pool.map(normalize_full_width, text_array)

        print("   Converting English letters to lower case...")
        text_array = pool.map(case_lower, text_array)

        print("   Translating Simplified Chinese to Traditional...")
        bool_contain_chi = pool.map(detect_chi, text_array)
        text_array = np.array(text_array)
        cc = OpenCC('s2t')
        text_array[bool_contain_chi] = list(map(lambda x: cc.convert(x), text_array[bool_contain_chi]))

        print("   Separating Chinese amd English word tokens from each other...")
        text_array = pool.map(sep_chi_eng, text_array)

        print("   Segmenting Chinese vocabularies...")
        text_array = np.array(text_array)
        dtype_0 = str(text_array.dtype)
        dtype_len_0 = int(re.findall("[0-9]+", dtype_0)[0])

        if sum(bool_contain_chi):
            replacement = pool.map(segment_chi, text_array[bool_contain_chi])
            replacement = np.array(replacement)
            dtype_1 = str(replacement.dtype)
            dtype_len_1 = int(re.findall("[0-9]+", dtype_1)[0])

            # change the type of text_array to that of the replacement
            # in order to avoid some of the characters in the segmented text being trimmed
            if dtype_len_0 > dtype_len_1:
                text_array = text_array.astype(dtype_0)
            else:
                text_array = text_array.astype(dtype_1)

            text_array[bool_contain_chi] = replacement

        print("   Removing punctuation...")
        text_array = pool.map(remove_punct, text_array)

        if word_tokenization == "word_piece":
            print("   Word Piece Segmentation...")
            tokenizer = ModifiedWordPieceTokenizer(bert_vocab_file=bert_vocab_file,
                                                   do_lower_case=do_lower_case)
            text_array = pool.map(tokenizer.tokenize, text_array)
        elif word_tokenization == "space":
            return text_array

    return text_array

def tokens_to_index(token_list, vocab_dict):
    """
    convert word tokens to number index

    INPUT:
        token_list: a list of string tokens
        vocab_dict: a dictionary with tokens(key)-integer(value) pairs
    OUTPUT:
        index_list: a list of integer indices
    """
    index_list = []
    for x in token_list:
        try:
            index_list.append(vocab_dict[x])
        except:
            index_list.append(1)  # index for oov tokens
    return index_list

def padding(index_list, max_len):
    """
    pad 0 to the index_list

    INPUT:
        index_list: an integer list with length <= max_len
        max_len: integer --maximum number of entries in the list
    OUTPUT:
        an integer list with length == max_len
    """
    return [0] * (max_len - len(index_list)) + index_list

def truncate_text(token_list, max_len):
    """
    cutting tokens that exceeds the max_len for each input sentence

    INPUT:
        token_list: a list of string with variable length
        max_len: maximum number of tokens in the text
    OUTPUT:
        a list of string with length <= max_len
    """
    return token_list[:max_len]

def get_one_hot_encoding(label_lists, labels):
    """
    a function to return class labels in one hot encoding format
    INPUT:
        label_lists: list of integer labels for the samples
        labels: a list of all possible classification classes
    OUTPUT:
        y: 2-dimensional list, specifically a one-hot matrix -- shape of [N, len(labels)]
    """
    le = LabelEncoder()
    le = le.fit(labels)
    y = le.transform(label_lists)
    y = to_categorical(y, num_classes=len(labels))
    return y

def split_text(text):
    """a function to split sentence into a list of tokens based on space characters."""
    return text.split()

def is_valid_label(label, all_labels):
    """a function to return whether the input "label" exist in the all_labels, which is a list of string"""
    return label in all_labels

def label_validation(df, col_label, all_labels, num_threads=1):
    """
    a function to separate a data frame into two, one contains records with valid labels, one without

    INPUT:
        df: a pandas dataframe which the column "TEXT" and "LABEL" must exist
        col_label: name of label column in the dataframe
        labels: a list of all available labels of the classification task
        num_threads: number of CPU processors for performing the subtasks
    OUTPUT:
        df_valid: a pandas dataframe which all records have valid labels
        df_invalid: a pandas dataframe which all records have invalid labels
    """
    with mp.Pool(processes=num_threads) as pool:
        bool_valid_label = pool.map(partial(is_valid_label, all_labels=all_labels), df[col_label])
        bool_invalid_label = pool.map(operator.not_, bool_valid_label)

        df_valid = df.loc[bool_valid_label, :]
        df_invalid = df.loc[bool_invalid_label, :]

    return df_valid, df_invalid

def is_uninformative(index_list, max_len):
    """
    a boolean function to determine whether an input text is informative or not.
    Since the index 0 is used for padding and the index 1 is used for out-of-vocabulary tokens, when all token indices
    for an input text is either 0 or 1 only (equivalently small than or equal to 1), the text is
    unlikely to be reasonably predicted by the model.

    INPUT:
        index_list: a list of indices
        max_len:  maximum number of tokens in each text
    OUTPUT:
        True or False

    """
    return bool(sum([e <= 1 for e in index_list]) == max_len)

def get_wxy(des_lists, vocab_dict, max_len, labels, num_threads=1, label_lists=None, label_to_weight_map=None):
    """
    a function to get the x, y of input data frame, weight and boolean lists

    INPUT:
        des_lists: a list/numpy array/pandas Series of N texts
        vocab_dict: a dictionary with tokens(key)-integer(value) pairs
        max_len: maximum number of tokens in each text
        labels: a list of all possible classification classes
        num_threads: int --number of CPU processors for performing the subtasks
        label_lists: a list/numpy array/pandas Series of labels each input text belongs to
        label_weight_map: a map with label as the key and the corresponding weight as value

    OUTPUT:
        x_out_valid: 2-dimensional numpy array -- shape of [N-U, max_len], where U is the number of uninformative
                                               -- samples. The uninformative samples are excluded for training.
        y_out_valid: 2-dimensional numpy array, specifically a one-hot matrix -- shape of [N-U, len(labels)]
        weight_valid: an numpy array of weight for informative samples only, length of array is N-U
        bool_informative: a boolean list of length N, True means informative and vice versa
        bool_uninformative: a boolean list of length N, True means uninformative and vice versa
    """

    with mp.Pool(processes=num_threads) as pool:

        token_lists = pool.map(split_text, des_lists)
        truncated_token_lists = pool.map(partial(truncate_text, max_len=max_len), token_lists)
        truncated_index_lists = pool.map(partial(tokens_to_index, vocab_dict=vocab_dict), truncated_token_lists)
        x = pool.map(partial(padding, max_len=max_len), truncated_index_lists)
        x_out = np.array(x)

        bool_uninformative = pool.map(partial(is_uninformative, max_len=max_len), x_out)
        bool_informative = pool.map(operator.not_, bool_uninformative)

        x_out_valid = x_out[bool_informative]

        if label_lists is not None:
            y = get_one_hot_encoding(label_lists, labels)
            y_out = np.array(y)
            y_out_valid = y_out[bool_informative]

        else:
            y_out_valid = None

        if label_to_weight_map is not None:
            informative_labels = np.array(label_lists)[bool_informative]
            weight_valid = np.array(list(map(lambda x: label_to_weight_map[x], informative_labels)))
        else:
            weight_valid = None

    return x_out_valid, y_out_valid, weight_valid, bool_informative, bool_uninformative

def focal_loss(gamma=2., alpha=0.25):
    gamma = float(gamma)
    alpha = float(alpha)

    def focal_loss_fixed(y_true, y_pred):
        """Focal loss for multi-classification
        FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
        Notice: y_pred is probability after softmax
        gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
        d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
        Focal Loss for Dense Object Detection
        https://arxiv.org/abs/1708.02002

        Arguments:
            y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
            y_pred {tensor} -- model's output, shape of [batch_size, num_cls]

        INPUT:
            gamma {float} -- (default: {2.0})
            alpha {float} -- (default: {0.25})

        OUTPUT:
            [tensor] -- focal loss value.
        """
        epsilon = 1.e-9
        y_true = tf.convert_to_tensor(y_true, tf.float32)
        y_pred = tf.convert_to_tensor(y_pred, tf.float32)

        model_out = tf.add(y_pred, epsilon)
        ce = tf.multiply(y_true, -tf.log(model_out))
        weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
        fl = tf.multiply(alpha, tf.multiply(weight, ce))
        reduced_fl = tf.reduce_max(fl, axis=1)
        return tf.reduce_mean(reduced_fl)

    return focal_loss_fixed

def load_data_file(file_path, FLAGS):
    """
    a function load the feather file and perform basic validation checking

    INPUT:
        file_path: string --path to the data file
        FLAGS: model flags
    OUTPUT:
        df: a pandas data frame which the columns "TEXT" and "LABEL" must exist.
    """

    # import the data file as pandas dataframe
    if bool(re.search(".feather", file_path.lower())):
        df = feather.read_dataframe(file_path)
    elif bool(re.search(".csv", file_path.lower())):
        df = pd.read_csv(file_path, dtype={'TEXT': str, 'LABEL': str})
    elif bool(re.search(".xlsx", file_path.lower())):
        df = pd.read_excel(file_path, dtype={'TEXT': str, 'LABEL': str})
    elif bool(re.search(".dbf", file_path.lower())):
        text = DBF(file_path, char_decode_errors="ignore")  # set "ignore_missing_memofile=True" in absence of memo file
        df = pd.DataFrame(iter(text))
    else:
        raise ValueError("Fail to open the file {}, the acceptable data file format \
        include .feather, .csv, .txt, .xlsx and .dbf only".format(file_path))

    # basic validation checking
    if "TEXT" not in df.columns:
        raise ValueError("The column `TEXT` is missing from the input data file: {}.".format(file_path))
    # only in "do_train" or "do_eval" mode the column "LABEL" is required
    if FLAGS.do_train or FLAGS.do_eval:
        if "LABEL" not in df.columns:
            raise ValueError("The column `LABEL` is missing from the input data file: {}.".format(file_path))

    return df

def get_cr_distr(prediction, col_label, col_pred, col_probit):
    """
    get the distribution of classification rate

    INPUT:
        prediction: a pandas dataframe with at least three columns, one for the groud truth label of the prediction,
                    another for the predicted label as well as one for the predicted probability of the predicted label
        col_label: column name of the ground truth label
        col_pred:  column name of the predicted label
        col_predict: column name of the predicted probability of the predicted label
    OUTPUT:
        distr_table: a pandas dataframe that shows the distribution of the classification rate and the accuracy
    """

    bins = np.arange(0, 1.1, 0.1)
    prediction["Probability"] = pd.cut(prediction[col_probit], bins)
    prediction["True/False"] = prediction[col_label] == prediction[col_pred]
    prediction["Proportion (%)"] = 1

    distr_table = prediction.loc[:, ["Probability", 'True/False', 'Proportion (%)']].groupby(["Probability", 'True/False'])
    distr_table = (distr_table.count()/prediction.shape[0])*100
    distr_table = distr_table.unstack()
    distr_table["Proportion (%)", "Total"] = distr_table["Proportion (%)"][True] + distr_table["Proportion (%)"][False]
    distr_table["Classification rate (%)"] = 100 * (distr_table["Proportion (%)"][True] / \
                                             (distr_table["Proportion (%)"][True] + distr_table["Proportion (%)"][False]))
    return distr_table


#/usr/bin/env
"""A simple async web crawler"""

import asyncio
import argparse
import logging
import sys
import json
import re
from typing import Dict
from urllib import parse

import aiohttp
import async_timeout
import html5lib
from bs4 import BeautifulSoup
from pprint import pformat

from utils import get_parent_url

parser = argparse.ArgumentParser(description='Run web crawler asynchronously')
parser.add_argument('--url', type=str, required=True, help='Source URL to start the crawler from')
parser.add_argument('--workers', type=int, default=5, help='Number of workers to crawl')
parser.add_argument('--verbose', action='store_true', help='Detailed output to stdout')
parser.add_argument('--save', action='store_true', help='Save to crawler_recipes.json file')

logging.basicConfig(stream=sys.stdout, format="%(message)s")
log = logging.getLogger()
log.setLevel(logging.INFO)

crawled_urls = set()

def update_queue(queue: asyncio.Queue, soup: BeautifulSoup, regexp: str, parent_url: str) -> None:
  """Update queue with new links

  Args:
    queue (asyncio.Queue): The asyncio queue that the work has to be done on
    soup (BeautifulSoup): BeautifulSoup object for the page
    regex (str): Reg exp to validate recipe URL
    parent_url (str): Parent URL
  """

  for link in soup.find_all('a', href=True):
    new_link = parse.urljoin(parent_url, link['href'])
    if re.search(regexp, new_link) and get_parent_url(new_link) == parent_url and new_link not in crawled_urls:
      queue.put_nowait(new_link)

def process_recipe(recipe: Dict, verbose: bool = False, save: bool = False) -> None:
  """Process recipe dictionary based on verbose printing option or saving to JSON

  Args:
    recipe (Dict): Recipe information
    verbose (bool, optional): Defaults to False. Flag to log out to terminal
    save (bool, optional): Defaults to False. Flag to save to crawler_recipes.json file
  """

  if not recipe:
    pass
  if verbose:
    log.info(pformat(recipe))
  elif save:
    with open('crawler_recipes.json', 'a+') as f:
      f.write(json.dumps(recipe) + '\n')

def parse_page_soup(url: str, soup: bytes) -> Dict:
  """Parse the recipe page soup to get required recipe information

  Args:
    url (str): Recipe URL from which the page bytes are fetched
    soup (BeautifulSoup): BeautifulSoup object for the page

  Returns:
    Dict: Recipe information
  """
  try:
    recipe_name = soup.select('h1')[0].text
    recipe_photo = soup.select('div.recipe-details-image > img')[0]['src']
    ingredients = [' '.join([t.text for t in li.find_all('span', recursive=False)]).replace(
        u'\xa0', u' ').strip() for li in soup.select('div.recipe-ingredients li')]
    half_star_tags = soup.select('#reviews span.half-star')
    full_stars_tags = soup.select('a.recipe-details-rating span.full-star')
    half_star = float(half_star_tags[0]['data-star-number']) if half_star_tags else 0.0
    full_stars = float(full_stars_tags[-1]['data-star-number']) if full_stars_tags else 0.0
    ratings = full_stars + (0.5 if full_stars < 5 and half_star else 0.0)
    reviews = int(soup.select('#reviews span')[0].text.strip('()'))
    cook_time = ' '.join(t.text for t in soup.select(
        'div.recipe-summary-item.unit span')).replace(u'\xa0', u' ')
    serve_tags = soup.select('div.servings input')
    serve = int(serve_tags[0]['value']) if serve_tags else 1
    return dict(recipeName=recipe_name,
                recipePhoto=recipe_photo,
                ingredients=ingredients,
                ratings=ratings,
                reviews=reviews,
                cookTime=cook_time,
                serve=serve)
  except:
    return {}

async def get_page_bytes(url: str) -> bytes:
  """Get page in bytes from URL

  Args:
    url (str): URL to fetch the page from

  Returns:
    bytes: Page in bytes
  """

  async with aiohttp.ClientSession() as session:
    async with session.get(url) as response:
      if response.status == 200:
        page_bytes = await response.read()
        return page_bytes
      return None

async def crawl(queue: asyncio.Queue,
                parent_url: str,
                verbose: bool = False,
                save: bool = False,
                regex: str = r'/recipe/.+') -> None:
  """Crawls URLs in the queue while updating the queue itself with URLs

  Args:
    queue (asyncio.Queue): The asyncio queue that the work has to be done on
    parent_url (str): Parent URL
    verbose (bool, optional): Defaults to False. Flag to log out to terminal
    save (bool, optional): Defaults to False. Flag to save to crawler_recipes.json file
    regex (str, optional): Defaults to r'/recipe/.+'. Reg exp to validate recipe URL
  """

  global crawled_urls
  try:
    while True:
      url = await queue.get()
      if url not in crawled_urls:
        crawled_urls.add(url)
        page_bytes = await get_page_bytes(url)
        if page_bytes:
          soup = BeautifulSoup(page_bytes.decode('utf-8'), 'html5lib')
          recipe = parse_page_soup(url, soup)
          process_recipe(recipe, verbose, save)
          update_queue(queue, soup, regex, parent_url)
      queue.task_done()
  except asyncio.CancelledError:
    log.debug('Error: The worker has been cancelled; perhaps on purpose')
    pass

def main() -> None:
  """Parses commandline arguments, starts async IO events loop and creates workers to work on queue
  """

  args = parser.parse_args()
  parent_url = get_parent_url(args.url)
  if not parent_url:
    raise Exception('Unsupported URL provided')
  if not args.verbose and args.save:
    with open('crawler_recipes.json', 'w'):
      pass
  loop = asyncio.get_event_loop()
  queue = asyncio.Queue()
  queue.put_nowait(args.url)
  workers = [asyncio.ensure_future(crawl(queue, parent_url, verbose=args.verbose, save=args.save)) for _ in range(args.workers)]
  loop.run_until_complete(queue.join())
  for w in workers:
    w.cancel()
  loop.close()

if __name__ == '__main__':
  main()


# 3. Longest Substring Without Repeating Characters
# Medium
# 35.1K
# 1.6K
# Companies
# Given a string s, find the length of the longest
# substring
#  without repeating characters.

# Example 1:

# Input: s = "abcabcbb"
# Output: 3
# Explanation: The answer is "abc", with the length of 3.
# Example 2:

# Input: s = "bbbbb"
# Output: 1
# Explanation: The answer is "b", with the length of 1.
# Example 3:

# Input: s = "pwwkew"
# Output: 3
# Explanation: The answer is "wke", with the length of 3.
# Notice that the answer must be a substring, "pwke" is a subsequence and not a substring.

# Constraints:

# 0 <= s.length <= 5 * 104
# s consists of English letters, digits, symbols and spaces.

class Solution:
    def lengthOfLongestSubstring(self, s: str) -> int:
        ans = 0
        substring = []

        for char in s:
            while char in substring:
                substring.pop(0)
            substring.append(char)
            if ans < len(substring):
                ans = len(substring)
        return ans

sol = Solution()

print(sol.lengthOfLongestSubstring("abcabcbb"))

# beats 24.41%


"""
Modeling a Two Tank system with neural ODEs
"""

import torch
import numpy as np
import slim
import psl

from neuromancer import blocks, estimators, dynamics, arg, integrators, ode
from neuromancer.interpolation import LinInterp_Offline
from neuromancer.activations import activations
from neuromancer.visuals import VisualizerOpen
from neuromancer.trainer import Trainer
from neuromancer.problem import Problem
from neuromancer.simulators import OpenLoopSimulator, MultiSequenceOpenLoopSimulator
from neuromancer.callbacks import SysIDCallback
from neuromancer.loggers import BasicLogger, MLFlowLogger
from neuromancer.dataset import get_sequence_dataloaders
from neuromancer.constraint import variable
from neuromancer.loss import PenaltyLoss

torch.manual_seed(0)
device = "cpu"

system = psl.systems['TwoTank']

ts = 1.0
nsim = 2000
modelSystem = system(ts=ts, nsim=nsim)
raw = modelSystem.simulate()
psl.plot.pltOL(Y=raw['Y'])
psl.plot.pltPhase(X=raw['Y'])

t = (np.arange(nsim+1)*ts).reshape(-1, 1)
raw['Time'] = t[:-1]

#%% Interpolant for offline mode
t = torch.from_numpy(t)
u = raw['U'].astype(np.float32)
u = np.append(u,u[-1,:]).reshape(-1,2)
ut = torch.from_numpy(u)
interp_u = LinInterp_Offline(t, ut)

#%% Interpolation class for online mode
# Interp_Online = LinInterp_Online()

#%%
#  Train, Development, Test sets - nstep and loop format
nsteps = 30  # nsteps rollouts in training
nstep_data, loop_data, dims = get_sequence_dataloaders(raw, nsteps,
                            moving_horizon=True)

train_data, dev_data, test_data = nstep_data
train_loop, dev_loop, test_loop = loop_data

# %% Identity mapping
nx = dims['X'][1]

estim = estimators.FullyObservable(
    {**train_data.dataset.dims, "x0": (nx,)},
    linear_map=slim.maps['identity'],
    input_keys=["Yp"],
)

estim(train_data.dataset.get_full_batch())

# %% Instantiate the blocks, dynamics model:
nu = dims['U'][1]
black_box_ode = blocks.MLP(insize=nx+nu, outsize=nx, hsizes=[128, 128, 128],
                           linear_map=slim.maps['linear'],
                           nonlin=activations['gelu'])
fx_int = integrators.RK4(black_box_ode, interp_u=interp_u, h=modelSystem.ts)
fy = slim.maps['identity'](nx, nx)

dynamics_model = dynamics.ODENonAuto(fx_int, fy, extra_inputs=['Uf'],
                input_key_map={"x0": f"x0_{estim.name}", "Time": "Timef", 'Yf': 'Yf'},  #TBC2: sth wrong with input_key_map
                name='dynamics',  # must be named 'dynamics' due to some issue in visuals.py
                online_flag=False
                )

# %% Constraints + losses:
yhat = variable(f"Y_pred_{dynamics_model.name}")
y = variable("Yf")

yFD = (y[:, 1:, :] - y[:, :-1, :])
yhatFD = (yhat[:, 1:, :] - yhat[:, :-1, :])

fd_loss = 2.0*((yFD == yhatFD)^2)
fd_loss.name = 'FD_loss'

reference_loss = ((yhat == y)^2)
reference_loss.name = "ref_loss"

# %%
objectives = [reference_loss, fd_loss]
constraints = []
components = [estim, dynamics_model]
# create constrained optimization loss
loss = PenaltyLoss(objectives, constraints)
# construct constrained optimization problem
problem = Problem(components, loss)
# plot computational graph
problem.plot_graph()
problem = problem.to(device)

# %%
optimizer = torch.optim.Adam(problem.parameters(), lr=0.001)
logger = BasicLogger(args=None, savedir='test', verbosity=1,
                     stdout="nstep_dev_"+reference_loss.output_keys[0])

simulator = OpenLoopSimulator(
    problem, train_loop, dev_loop, test_loop, eval_sim=True, device=device,
) if isinstance(train_loop, dict) else MultiSequenceOpenLoopSimulator(
    problem, train_loop, dev_loop, test_loop, eval_sim=True, device=device,
)
visualizer = VisualizerOpen(
    dynamics_model,
    1,
    'test',
    training_visuals=False,
    trace_movie=False,
)
callback = SysIDCallback(simulator, visualizer)

trainer = Trainer(
    problem,
    train_data,
    dev_data,
    test_data,
    optimizer,
    callback=callback,
    patience=100,
    warmup=10,
    epochs=150,
    eval_metric="nstep_dev_"+reference_loss.output_keys[0],
    train_metric="nstep_train_loss",
    dev_metric="nstep_dev_loss",
    test_metric="nstep_test_loss",
    logger=logger,
    device=device,
)
# %%
best_model = trainer.train()
# %%
best_outputs = trainer.test(best_model)
# %%


import re
import os
from email.parser import Parser
import argparse
import shutil

class EmlDisassembly:
    def __init__(self):
        self.list_paths = []

    def message_count(self, path_file):
        for (paths,dirs,files) in os.walk(path_file):
            for file in files:
                path = paths + file
                self.list_paths.append(path)
        messages = '\n\033[47m\033[30m----Number of messages in a folder:\033[0m \033[32m{0}\033[0m\n'.format(str(len(self.list_paths)))
        print(messages)

    def movefile(self,path,login,file):
        shutil.copyfile(path, 'downloads\\' + login + '\\' + file)
        message = '----Message: {0}'.format(file)
        print(message)

    def makedir(self,login):
        os.mkdir('downloads\\' + login)
        print('\033[30m---Create folder with name---{0}\033[0m'.format(login))

    def check_downloads_folder(self):
        dirs = [dirs for (paths,dirs,files) in os.walk('.')]
        if dirs[0].count('downloads'):
            message = '\033[32mFolder <DOWNLOADS> exists\033[0m'
            print(message)
        else:
            os.mkdir('downloads')
            message = '\033[32mFolder <DOWNLOADS> make\033[0m'
            print(message)

    def folder_mails(self, path_file):
        for (paths,dirs,files) in os.walk(path_file):
            for file in files:
                path = paths + file
                with open(path) as fp:
                    emailcontent = Parser().parsestr(fp.read())
                    To = emailcontent['To']
                    with open('logins.txt') as logins_file:
                        for login_file in logins_file.readlines():
                            login_up = ''.join(login_file.split('\n'))
                            if re.findall(login_up, To):
                                try:
                                    self.makedir(login_up)
                                    self.movefile(path, login_up, file)
                                except OSError:
                                    self.movefile(path, login_up, file)
                            else:
                                pass

    def main(self):
        try:
            self.check_downloads_folder()
            self.message_count('files\\')
            self.folder_mails('files\\')
        except TypeError:
            pass

if __name__ == '__main__':
    obj = EmlDisassembly()
    obj.main()


import urllib.request
import csv
import json
import os.path
import string
import time
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn import model_selection
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC

# Runescape item category names
cat_names = ["Miscellaneous","Ammo", "Arrows", "Bolts", "Construction_Materials"
, "Construction_Projects", "Cooking_Ingredients","Costumes"
,"Crafting_Materials","Familiars","Farming_Produce","Fletching_Materials"
,"Food_and_Drink","Herblore_Materials","Hunting_Equipment","Hunting_Produce"
,"Jewellery","Mage_Armour","Mage_Weapons", "Melee_Armor_Low_Level","Melee_Armor_Mid_Level",
"Melee_Armor_High_Level","Melee_Weapons_Low_Level","Melee_Weapons_Mid_Level","Melee_Weapons_High_Level","Mining_and_Smithing","Potions","Prayer_Armour","Prayer_Materials",
"Range_Armour","Range_Weapons","Runecrafting","Runes_Spells_Teleports","Seeds","Summoning_Scrolls",
"Tools_and_Containers","Woodcutting_Products","Pocket_Items"]

def get_data_from_row(row, m, n):
    arr = list(row)
    # NORMALIZING DATA FROM ARR
    minn = min(arr)
    for x in range(len(arr)):
        arr[x] = arr[x] - minn
    maxx = max(arr)
    for x in range(len(arr)):
        if arr[x] != 0:
            arr[x] = maxx / arr[x]
    # NORMALIZING DATA FROM ARR
    row_X = []
    row_Y = []
    true_set = set()
    # Predict if price increases over the next n days based on the previous m
    for x in range(0, len(arr)-m-n, m+n+1):
        past_m = list(arr[x:x+m])
        row_X.append(past_m)
        next_n = True if arr[x+m] < arr[x+m+n] else False # True if price increased over next n days
        row_Y.append(next_n)
        true_set.add(next_n)
    # Making sure all data points aren't in the same class, not useful for ML
    if len(true_set) == 1:
        return [], []
    return row_X, row_Y
def test_models(X, Y):
    models = []
    models.append(('LogisticRegression', LogisticRegression(solver='liblinear', multi_class='ovr')))
    models.append(('LinearDiscriminantAnalysis', LinearDiscriminantAnalysis()))
    models.append(('KNeighborsClassifier', KNeighborsClassifier()))
    models.append(('DecisionTreeClassifier', DecisionTreeClassifier()))
    models.append(('GaussianNB', GaussianNB()))
    # SVM problems with more than 200,000 data points
    n_splits = 10 # Number of k-fold splits, decrease this number if it doesn't work due to large m + n size
    seed = 7
    scoring = "accuracy"
    results = []
    names = []
    for name, model in models:
        kfold = model_selection.KFold(n_splits=n_splits, random_state=seed)
        cv_results = model_selection.cross_val_score(model, X, Y, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
        print(msg)
def get_cat_data(headers, cat, m, n):
    df = pd.read_csv("item_graphs/"+cat+".csv", names=headers, index_col = 0)
    cat_X = []
    cat_Y = []
    for index, row in df.iterrows():
        row_X, row_Y = get_data_from_row(row, m, n)
        cat_X.extend(row_X)
        cat_Y.extend(row_Y)
    return cat_X, cat_Y
# machine_learning(m, n): Tests models for ML, price increase in next n days based on previous m days of data
def machine_learning(m, n):
    headers = ["Name"] + [x for x in range(-179, 1)]
    all_X = []
    all_Y = []
    for cat in cat_names:
        print("Data for "+cat+" Category: (" +str(m) + ","+str(n)+")")
        cat_X, cat_Y = get_cat_data(headers, cat, m, n)
        all_X.extend(cat_X)
        all_Y.extend(cat_Y)
    print("Testing Models:")
    test_models(all_X, all_Y)

machine_learning(4,2)
# Run with python3 -W ignore machine_learning.py  to suppress warnings about collinear variables


import pandas as pd
import csv
import argparse

def findPredictById(row,data_pred,task):
	val = data_pred[data_pred['idtwitter'] == row.idtwitter]
	print(val)
	if task == 'pos' :
		return val['pred_pos'].values[0]
	if task == 'neg' :
		return val['pred_neg'].values[0]
	return -1

if __name__ == "__main__":

	parser = argparse.ArgumentParser()
	parser.add_argument('--gold', type=str, help="gold file for evaluate sistem", default='./data/test_gold.csv')
	parser.add_argument('--predictPos', type=str, help="output dir", default='./res_bert_try_opos.csv')
	parser.add_argument('--predictNeg', type=str, help="Path of csv for train ", default='./res_bert_try_oneg.csv')
	parser.add_argument('--runName', type=str, help="name for run ", default='001')

	args = parser.parse_args()
	gold = args.gold
	predPos = args.predictPos
	predNeg = args.predictNeg
	data_data_gold = pd.read_csv(gold,sep='\t')
	data_pred = pd.read_csv(predPos,sep='|')
	data_pred.pred_pos.astype(str)

	print(data_pred.head())
	data_pred_neg = pd.read_csv(predNeg,sep='|')
	data_pred_neg.pred_neg.astype(str)

	data_data_gold['pred_pos'] = data_data_gold.apply(lambda row: findPredictById(row,data_pred,'pos'), axis=1)
	data_data_gold['pred_neg'] = data_data_gold.apply(lambda row: findPredictById(row,data_pred_neg,'neg'), axis=1)
	print(data_data_gold.head())

	data_to_save = data_data_gold[['idtwitter','subj','pred_pos','pred_neg','iro','lpos','lneg','top']]

	data_to_save.idtwitter = data_to_save.idtwitter.astype(str)
	data_to_save.subj = data_to_save.subj.astype(str)
	data_to_save.pred_pos = data_to_save.pred_pos.astype(str)
	data_to_save.pred_neg = data_to_save.pred_neg.astype(str)
	data_to_save.iro = data_to_save.iro.astype(str)
	data_to_save.lpos = data_to_save.lpos.astype(str)
	data_to_save.lneg = data_to_save.lneg.astype(str)
	data_to_save.top = data_to_save.top.astype(str)
	data_to_save.to_csv(r'./res_bert_to_evaluete_'+args.runName+'.csv',header=None,index=None, sep=',', mode='a', quotechar='"',quoting=csv.QUOTE_NONNUMERIC)


# myfuncs.py
"""
A python that contains a function that calculates the approximation of the cosine function using a taylor series
"""

from math import factorial

def approx_cos(x,n_terms):
    approx = 1
    for i in range(1,n_terms+1):
        term = ((-1)**i)  * ((x**(i*2)) / (factorial(i*2)))
        approx = approx + term

    return approx