#!/usr/bin/env python
# coding: utf-8

# ## App Creation
#
# First, import all necessary libraries:

# In[1]:

#App Libraries
import json
import dash
from dash import html, dcc, Input, Output, State, dash_table
import dash_bootstrap_components as dbc

#Distributions
from scipy.stats import gamma
from scipy.stats import lognorm
from scipy.stats import weibull_min

#Calculation libraries
import math
import pandas as pd
import numpy as np
import ast
import statsmodels.api as sm
import matplotlib.pyplot as plt
import scipy.stats as stats
from scipy.optimize import minimize
from scipy.integrate import odeint
from scipy.optimize import fsolve
#from sympy import symbols, Eq, solve

#Plot libraries
import plotly.express as px
import plotly.graph_objs as go
import plotly.figure_factory as ff
from plotly.subplots import make_subplots

# In[2]:

#==================================================================#
#                 CREATE GENERATION INTERVAL DATA                  #
#==================================================================#
def create_gi(pop_mean, sd, m):
    '''
    pop_mean: population mean of the standard deviation
    '''

    #Set seed for consistency:
    np.random.seed(1234)

    #=========GAMMA============
    gamma_shape = (pop_mean**2)/(sd**2)
    gamma_scale = (sd**2)/(pop_mean)
    gi_gamma_obs = np.random.gamma(gamma_shape, gamma_scale, m)

    #=========LOGNORMAL============
    log_mean = pop_mean
    log_sd = sd
    log_var = log_sd**2
    norm_mean = np.log(log_mean)-0.5*np.log((log_sd/log_mean)**2+1) #scale=e^norm_mean
    norm_var = np.log((log_sd/log_mean)**2+1)
    norm_sd = np.sqrt(norm_var) # equivalent to the shape
    gi_lognorm_obs = lognorm.rvs(s=norm_sd, scale=math.exp(norm_mean), size=m)

    #=========WEIBULL============
    weibull_mean = pop_mean
    weibull_std = sd

    def G(k):
        return math.gamma(1+2/k)/(math.gamma(1+1/k)**2)
    def f(k,b):
        return G(k)-b #function solves for k

    b = (weibull_std**2)/(weibull_mean**2)+1

    init = 1 # The starting estimate for the root of f(x) = 0.
    weibull_shape = fsolve(f,init,args=(b))[0]
    weibull_scale = weibull_mean/math.gamma(1+1/weibull_shape)
    gi_weibull_obs = weibull_min.rvs(weibull_shape,scale=weibull_scale, size=m)

    return gi_gamma_obs, gi_lognorm_obs, gi_weibull_obs

#==================================================================#
#               VISUALIZE GENERATION INTERVAL DATA                 #
#==================================================================#

def gi_visualize(gi_gamma, gi_lognorm, gi_weibull):

    color=["skyblue","darkorange","green"]

    fig = make_subplots(rows=2, cols=2,)

    fig.append_trace(go.Histogram(x=gi_gamma, histnorm='percent', name='Gamma',
                                marker_color=color[0], opacity=1,),row=1,col=1)

    fig.append_trace(go.Histogram(x=gi_lognorm, histnorm='percent', name='Lognorm',
                                marker_color=color[1], opacity=1), row=1,col=2)

    fig.append_trace(go.Histogram(x=gi_weibull, histnorm='percent', name='Weibull',
                                marker_color=color[2], opacity=1), row=2,col=1)

    group_labels = ['Gamma Curve', 'Lognormal Curve', 'Weibull Curve']
    hist_data = [gi_gamma, gi_lognorm, gi_weibull]
    distplfig = ff.create_distplot(hist_data, group_labels, colors=color,
                             bin_size=.2, show_hist=False, show_rug=False)

    for k in range(len(distplfig.data)):
        fig.append_trace(distplfig.data[k],
        row=2, col=2
    )
    fig.update_layout(barmode='overlay')

    return(fig)

#==================================================================#
#                          OBJECTIVE FUNCTION                      #
#==================================================================#

def objective(w_lamb,tau):
    '''
    Objective: To maximize the log likelihood of W(u) (ie min(-W(u)))

    Inputs:
        w_lamb= weights [w_1, w_2,...,w_n] and lambda in one list
        tau = set of times since infection [tau_1, tau_2,...,tau_m]
    Outputs:
        objective: value (-W(u))
    '''

    w=w_lamb[:-1]
    lamb=w_lamb[-1]

    n=len(w)

    objective = 0

    for tau_i in tau: #FOR EACH TIME SINCE EXPOSURE
        wlog_val = w[0]
        for j in range(1,n): #CALCULATE TERMS WITHIN LOG
            wlog_val = wlog_val + (w[j]*(((lamb*tau_i)**j)/math.factorial(j)))
        objective = objective + (math.log(wlog_val) - (lamb*tau_i) + math.log(lamb))

    return(-1 *objective)

#==================================================================#
#                         CONSTRAINT FUNCTION                      #
#==================================================================#
def constraint(w_lamb):
    '''
    Constraint 1:  Weights must sum to 1
    Inputs:
        w: list of weights

    Outputs:
        constraint1: value of 1 - sum of the weights
    '''
    w=w_lamb[:-1]
    n = len(w)

    constraint1 = 1

    for j in range(n):
        constraint1 = constraint1 - w[j]
    return constraint1

#==================================================================#
#              CALCULATE WEIGHTS, HOLDING PERIOD, RATES            #
#==================================================================#
def solver(tau, R_0, n, dist_type):
    '''
    The following function returns a list of weights given the 5 inputs.

    Inputs:
        tau: list of generation intervals times (in days)
        R_0: basic reproduction number
        n: number of infectious comparments

    Output:
        w_val: list of weights (based on minimization)
        lambda: lambda value   (based on minimization)
        b_val: list of betas   (based on minimization)
    '''

    wl_0 = np.zeros(n+1)

    if dist_type == "gamma" or dist_type == "lognorm":
        shape = (np.mean(tau)**2)/np.var(tau) #shape of the disribution
        w2 = shape - math.trunc(shape)        #expected weight of "second" compartment
        w1 = 1 - w2                           #expected weight of "first" compartment
        comps = [math.trunc(shape)-1, math.trunc(shape)] #location of the "first" and "second" compartments where weights should exceed one
        weights = [w1, w2]

        for c, w in zip(comps,weights):
            wl_0[c] = w

        wl_0[-1]= np.mean(tau)/np.var(tau)

    #elif dist_type == "lognorm":
    #    for i in range(n):
    #        wl_0[i] = 1/n

    #    log_mean = np.mean(tau)
    #    log_std = np.std(tau)
    #    norm_mean = np.log(log_mean)-0.5*np.log((log_std/log_mean)**2+1)
    #    wl_0[-1]=  norm_mean

    elif dist_type == "weibull":
        for i in range(n):
            wl_0[i] = 1/n

        wl_0[-1]= np.std(tau)

    b = (0, 1)
    bnds=()
    for i in range(n):
        bnds = bnds + (b,)

    b_lamb = (0.00000000001, None)
    bnds = bnds + (b_lamb,)

    #specify constraints
    con1 = {'type': 'eq', 'fun': constraint}
    cons = ([con1])

    #optimize
    solution = minimize(objective, wl_0, method='SLSQP',                        args=(tau), bounds=bnds,constraints=cons)

    #get weights
    w_val = solution.x[:-1]
    lamb = solution.x[-1]
    b_val = [weight*lamb*R_0 for weight in w_val]

    return(w_val, lamb, b_val)

#==================================================================#
#                          OBJECTIVE FUNCTION                      #
#==================================================================#
def solutions(gi_data,min_n, max_n, R_0, dist_type):

    weight = []
    lambda_n = []
    beta = []
    obj = []

    for n_val in list(range(min_n, max_n+1)):
        if n_val == min_n:
            str_p = "Solving: "+ str(dist_type)+ " with R_0="+str(R_0)+" for n in "+ str(min_n)+",...,"+str(max_n)
            print(str_p)

        w, l, b = solver(gi_data, R_0, n_val, dist_type)
        o = objective(list(w)+[l],gi_data)

        if n_val == int((max_n+1-min_n)/4+min_n):
            print("n=",str(n_val)," is done. 25% Done!")

        if n_val == int((max_n+1-min_n)/2+min_n):
            print("n=",str(n_val)," is done. Half way there!")

        if n_val == int(3*(max_n+1-min_n)/4 + min_n):
            print("n=",str(n_val)," is done. 75% Done!")

        if n_val == max_n:
            print("Done!")

        weight.append(w)
        lambda_n.append(l)
        beta.append(b)
        obj.append(o)

    return weight, lambda_n, beta, obj

#==================================================================#
#                       EXPECTED INFECTIOUS CURVE                  #
#==================================================================#
def beta_u(u_i,beta_vals, lambda_n):
    '''
    Beta(u): Find transmission rate for every time in u_i

    Inputs:
        u_i: list of generation intervals times (in days)
        beta_vals: list of betas (based on minimization)
        lambda_n: rate that infected move to the next compartment

    Outputs:
        y:
    '''
    n = len(beta_vals)
    y = []

    for u in u_i:
        transmission=0
        for j in range(n):
            transmission = transmission + beta_vals[j]*((np.exp(-lambda_n*u)*(lambda_n*u)**j)/math.factorial(j))

        y.append(transmission)

    return(y)

#==================================================================#
#                 VISUALIZE EXPECTED INFECTIOUS CURVE              #
#==================================================================#
def beta_u_plot(lambda_n, beta_vals):

    #create x axis
    x = np.linspace(0, 15, 100)

    #create df of x, y data
    beta_df = pd.DataFrame(x, columns=["x"])

    beta_df[str(len(beta_vals))] = [float(b) for b in beta_u(x,beta_vals,lambda_n)]

    fig = go.Figure()
    fig.add_trace(go.Scatter(x=beta_df.x, y=beta_df.iloc[:, 1]))

    #format graph
    fig.update_layout(legend_title_text='Compartment')
    fig.update_xaxes(title_text='Days', nticks=20)
    fig.update_yaxes(title_text='Transmission Rate')

    return(fig)

#==================================================================#
#               VISUALIZE EXPECTED INFECTIOUS CURVE                #
#==================================================================#
def plot_beta_dist(betas, lambdas):

    color=["skyblue","darkorange","green"]
    dist = ["Gamma", "Lognormal","Weibull"]
    count = 0

    fig = make_subplots(rows=1, cols=1)

    for beta,lamb in zip(betas,lambdas):
        data = beta_u_plot(lamb, beta)

        fig.add_trace(
            go.Scatter(x=data['data'][0]['x'], y=data['data'][0]['y'],
                      name=dist[count], line_color=color[count],
                      line=dict(width=3)),
            row=1, col=1
        )
        count+=1

    #-----STYLING--------
    fig.update_layout(
        title="Estimated Infectious Curve  of each Distribution", #β(𝜏)
        xaxis_title="Duration of Infection (days)",
        yaxis_title="Transmission Rate", legend_title="Legend", font=dict(size=14),
        legend=dict(yanchor="top", y=0.99, xanchor="right", x=0.99))

    return(fig)

#==================================================================#
#                          SInR(S) MODEL                           #
#==================================================================#
def SInR(y, t, N, c, h, beta, lambda_n):
    '''
    Inputs:
        y: initial condition vector (S, I, R)
        t: time points (in days)
        N: total population
        c: contact rate
        h: waning immunity rate
        beta: transmission rate
        lambda_n: optimized holding period

    Outputs:
        pop_status: returns a list of how many people are in each compartment
    '''
    S = y[0]
    I = y[1:-1]
    R = y[-1]

    npI = np.array(I)
    npBeta = np.array(beta)

    #Calculate dSdt
    dSdt = -S/N * np.sum(npI * npBeta)*c+ h*R

    #Calculate dI1dt
    dI1dt = S/N* np.sum(npI * npBeta)*c - lambda_n* I[0]

    #Calculate dI_dt values from n=2,..,n
    dIndt = []
    for index in range(len(I)-1):
        dIndt.append(lambda_n* I[index] - lambda_n* I[index+1])

    #Calculate dRdt
    dRdt = lambda_n * I[len(I)-1] - h*R

    #Create list of results for S through R
    pop_status = [dSdt, dI1dt]
    pop_status.extend(dIndt)
    pop_status.append(dRdt)

    return pop_status

#==================================================================#
#                      VISUALIZE SInR(S) MODEL                     #
#==================================================================#
def SInR_plot(y_t0, t, N, c, h, beta, lambda_n):
    # Integrate the SIR equations over the time grid, t.
    ret = odeint(SInR, y_t0, t, args=(N, c, h, beta, lambda_n))

    S = ret.T[0]
    I = sum(ret.T[1:-1])
    R = ret.T[-1]

    fig = go.Figure()
    fig.add_trace(go.Scatter(x=t, y=S/N,name="Susceptible"))
    fig.add_trace(go.Scatter(x=t, y=I/N,name="Sum of Infected Compartments"))
    fig.add_trace(go.Scatter(x=t, y=R/N,name="Recovered"))#

    fig.update_layout(legend_title_text='Compartment')
    fig.update_xaxes(title_text='Time (Days)')#,nticks=20)
    fig.update_yaxes(title_text='Percentage of the Population')

    return(fig)

# In[3]:

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])

##########################################################################
#                           HELPER FUNCTIONS                             #
##########################################################################
def create_dropdown_options(series):
    options = [{'label': i, 'value': i} for i in series.sort_values().unique()]
    return options
def create_dropdown_value(series):
    value = series.sort_values().unique().tolist()
    return value
def create_slider_marks(values): #NEED
    marks = {i: {'label': str(i)} for i in values}
    return marks

##########################################################################
#                    ADD ONS TO APP (images, files,etc)                  #
##########################################################################
#pull Gamma Data
gi_df = pd.read_excel ('GI_Values.xlsx')[["Source", "Mean","SD", "Dist_Type"]]
gi_df.sort_values(by=["Dist_Type",'Mean'], inplace=True)
gi_df.reset_index(drop=True, inplace=True)

colors = {'background': '#111111','text': 'black'}
subheader_size = 20

##########################################################################
#                             DASHBOARD APP                              #
##########################################################################
app.layout = html.Div(children=[
    dcc.Location(id="url",refresh=False),
    html.Div(id="output-div")
])

##########################################################################
#                               HOME PAGE                                #
##########################################################################
home_layout = html.Div(

    #==========Create "Header" Data=================
    children=[

    html.Div(
        [
            html.Img(src=app.get_asset_url("edinburgh.png"), height=50),
            html.Div([html.H4(children=('SI',html.Sub("n"),'R Covid-19 Modeling'), style={'textAlign': 'center', "font-weight": "bold"}),]),
        ], style={'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}
    ),

    #---Page Links---
    html.Div([dbc.Row(
    [
        dbc.Col(html.Div(dcc.Link('Home',href="/")), style={'textAlign': 'center'}),
        dbc.Col(html.Div(dcc.Link('Simulate Data',href="/simulate")), style={'textAlign': 'center'}),
        dbc.Col(html.A("Github Code", href='https://github.com/annette-bell/SInR-Covid-Dissertation',
                       target="_blank"), style={'textAlign': 'center'}),
    ],
    className="g-0",
    )]),

    #---Line Break---
    html.Div([html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700"}),]),

    #===============Home Page Information==========
    html.Div(
            [
                html.H6("About this app", style={"margin-top": "0","font-weight": "bold","text-align": "left"}),
                #html.Hr(),
                html.P("The Susceptible-Infected-Recovered (SIR) compartmental model is used in epidemiology to identify\
                        and categorize members of a population based on their status with regards to a disease. Less\
                        studied variations of this problem are the SInR and SInRS models. These models, which have applications\
                        in latent infection and varying transmission rates, will be used on three different generation\
                        interval—the time between primary exposure and secondary infection—distributions: gamma, lognormal,\
                        and Weibull. The distributions are ultimately tested against one another to see not only \
                        which provides most realistic data, but how these data-sets interact.\
                        This app is meant to help people understand dynamics of COVID-19 modeling through a simply dashboard application.\
                        To see a more in-depth explanation, please see the Github repository which includes my dissertation.",
                       className="control_label",style={"text-align": "justify"}),
            ],
            className="pretty_container almost columns",
        ),

    #============AUTHOR=============
    html.Div(
            [
                html.H6("Authors", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),

                html.P("Annette Bell (nettebell97@gmail.com)", style={"text-align": "center", "font-size":"10pt"}),

            ],
            className="pretty_container almost columns",
        ),

    #============ACKNOWLEDGEMENTS=============
    html.Div(
            [
                html.H6("Acknowledgements", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),

                html.P("John Dagpunar: Dr. Dagpunar was my thesis advisor and extremely helpful throughout the project.)", style={"text-align": "left", "font-size":"10pt"}),

            ],
            className="pretty_container almost columns",
        ),

    #============SOURCES=============
    html.Div(
            [
                html.H6("Sources", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                dcc.Markdown(
                    """\
                         - Code Layout was used from Plotly public dash application: https://dash.gallery/dash-food-consumption/
                         - I examined another dash application to better understand how to use it. In addition to dash application resources, I analyzed the source code to clarify how to implement dash: https://github.com/FranzMichaelFrank/health_eu
                         - Edinburgh PNG: https://uploads-ssl.webflow.com/5eb13d58c8c08b73773d6b1c/600ea3810bde89c60e317be7_uni-of-edinburgh.png
                        """
                ,style={"font-size":"10pt"}),

            ],
            className="pretty_container almost columns",
        ),

    ])

##########################################################################
#                           SIMULATION PAGE                             #
##########################################################################
sim_layout = html.Div(

    #==========Create "Header" Data=================
    children=[

    #---Title---
    html.Div(
        [
            html.Img(src=app.get_asset_url("edinburgh.png"), height=50),
            html.Div([html.H4(children=('SI',html.Sub("n"),'R Covid-19 Modeling'), style={'textAlign': 'center', "font-weight": "bold"}),]),
        ], style={'display': 'flex', 'align-items': 'center', 'justify-content': 'center'}
    ),

    #---Page Links---
    html.Div([dbc.Row(
    [
        dbc.Col(html.Div(dcc.Link('Home',href="/")), style={'textAlign': 'center'}),
        dbc.Col(html.Div(dcc.Link('Simulate Data',href="/simulate")), style={'textAlign': 'center'}),
        dbc.Col(html.A("Github Code", href='https://github.com/annette-bell/SInR-Covid-Dissertation',
                       target="_blank"), style={'textAlign': 'center'}),
    ], className="g-0",
    )]),

    #---Line Break---
    html.Div([html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700"}),]),

    #============OVERIEW OF THE SIMULATION DATA=================
        html.Div(
            [
                html.H6(["Overview of Distributions", html.Br()], style={"margin-top": "0","font-weight": "bold","text-align": "left"}),

                html.Div(
                [
                dbc.Row(
                    [#---Table of Previous Analysis---:
                        dbc.Col(dash_table.DataTable(gi_df.to_dict('records'), [{"name": i, "id": i} for i in gi_df.columns],
                                                     style_header={'text-align': 'center', 'fontWeight': 'bold',},
                                                     style_table={'height': '200px', 'overflowY': 'auto'},
                                                     style_cell={'textAlign': 'left'},), width=3),

                    #---Commentary on the table
                        dbc.Col(html.Div([html.P("There are three main commonly used distributions to model the generation interval-\
                        the time from primary exposure to secondary infectiousness. These distributions include gamma, weibull, and log-normal.\
                        To the left, you can see a table of means and standard deviations from others previous work.",className="control_label",style={"text-align": "justify"}),]))]),]),
            ],
            className="pretty_container",
        ),

    #================= GENERATION INTERVAL SIMULATION =====================
        html.Div(
            [
                html.Div(
                    #----------------INPUTS-----------
                    [
                    html.H6("Generation Interval Distribution:", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                    html.P("Please select the distribution you wish to base your simulated generation interval data off of. Note: Seed=1234.", className="control_label",style={"text-align": "justify"}),
                    html.Br(),

                    #Shows the inputs for the specified distribution
                    html.Div(
                        [

                            dbc.Row([
                                dbc.Col(html.Div([
                                    #---Mean---
                                    html.P("Input the population mean:", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='pop_mean',placeholder='', type='number', value= 4.9),

                                ])),
                                dbc.Col(html.Div([
                                    #---SD---
                                    html.P("Input the standard deviation:", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='stan_dev',placeholder='', type='number', value= 2),

                                ])),
                            ],),

                            #---Data Set Size---
                            html.P("Select size of data:", className="control_label", style={"font-weight": "bold", "text-align": "center"}),
                            dcc.Slider(id='gi_size', min=1000, max=10000, step=500, value=5000,
                                       marks=create_slider_marks(list(range(1000,10001))[::1000])),

                            html.Br(),

                            #---Update Button---
                            html.Button(id='update_button', children="Simulate", n_clicks=0,style=dict(width='220px')),
                        ],),

                    ],
                    className="pretty_container four columns",
                ),

                #----------------GI Plot-----------
                html.Div(
                    [
                        html.H6("Generation Interval Simulations", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),

                        #---Information Regarding Shape and Scale---
                        html.P(id='shape_scale', style={"text-align": "justify"}),

                        #---GI Histogram---
                        html.Div(id='gammaplot_container',children=[
                        dcc.Graph(id="gi_plots", style={'height': '80vh'}),
                        ]),

                    ],

                    className="pretty_container seven columns",
                ),

            ],
            className="row flex-display",
        ),

    #===============Transmission Rate==========
    html.Br(),
    html.Div([dbc.Row(
    [
        dbc.Col(html.Div(html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700", 'align':'right'},)), width=5),
        dbc.Col(html.Div(html.H6("Transmission Rate", style={"text-align": "center", "font-weight": "bold",
                                                            "margin-top": "14px", "color": "#384142"})), width=1.5),
        dbc.Col(html.Div(html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700", 'align':'left'},)), width=5),
    ],
    )]),

    html.Div(
            [
                html.Div(
                    [#----------------Parameters of Beta(u)-----------
                        html.H6("Parameters of \u03b2(u):", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                        html.P("As the transmission rate is not constant, we create a function that simulates transmission a non constant transmission rate.", className="control_label",style={"text-align": "justify"}),

                        #---R_0---
                        html.P("Input the basic reproduction number:", className="control_label", style={"font-weight": "bold", "text-align": "center"}),
                        dcc.Input(id='R0', placeholder='', type='number', value= 2.3),

                        html.Br(),
                        html.Br(),

                        #---Update Button---
                        html.Button(id='beta_update_button', children="Calculate B(u)", n_clicks=0, style=dict(width='220px')),

                        ], className="pretty_container four columns",
                ),

                html.Div(
                    [#----------------Beta(u) Plot-----------
                        html.H6("Expected Infectious Curve", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                        #html.P("Visualize the transmission rates. Note: the following results are based on the parameters and the GI Data simulated above.", className="control_label",style={"text-align": "justify"}),

                        html.Br(),

                        #---return weights and betas---
                        #html.P(id='weights_beta_info', style={"text-align": "justify"}),
                        #html.P(id='lambdas', style={"text-align": "justify"}),
                        #html.P(id='weights', style={"text-align": "justify"}),
                        #html.P(id='betas', style={"text-align": "justify"}),

                        dcc.Graph(id="beta_plot", style={'height': '60vh'}),
                    ],
                    className="pretty_container seven columns",
                ),
            ],
            className="row flex-display",
        ),

    #=====================SI(n)R Model=====================

    html.Br(),
    html.Div([dbc.Row(
    [
        dbc.Col(html.Div(html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700", 'align':'right'},)), width=5),
        dbc.Col(html.Div(html.H6("Modeling COVID-19", style={"text-align": "center", "font-weight": "bold",
                                                            "margin-top": "14px", "color": "#384142"})), width=1.5),
        dbc.Col(html.Div(html.Hr(style={'borderWidth': "0.3vh", "color": "#FEC700", 'align':'left'},)), width=5),
    ],
    )]),

    html.Div(
            [
                html.Div(
                    [#----------------Parameters of SInR Model-----------
                        html.H6("Parameters of the Model:", style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                        #html.P("Beta(u) was last calculated using the following:", className="control_label",style={"text-align": "justify"}),

                        html.Div([
                        dcc.RadioItems(
                            id='distribution-dropdown',
                            options=[
                                {'label': 'gamma', 'value': 'gamma'},
                                {'label': 'weibull', 'value': 'weibull'},
                                {'label': 'log-normal','value': 'lognorm'},
                                {'label': 'all','value': 'all'}],

                            value='all',

                            labelStyle={"display": "inline-block"},
                            style={"font-weight": "bold", "text-align": "center"},
                        ),],),

                        dbc.Row(
                            [
                                dbc.Col(html.Div([
                                     #---Total Population---
                                    html.P("Total Population (N):", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='N_size', placeholder='', type='number', value = 67886011),
                                ])),
                                dbc.Col(html.Div([
                                    #---simulated days---
                                    html.P("Total Days to Simulate Over:", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='t_days', placeholder='', type='number', value = 180),

                                ])),

                            ], className="g-0",
                            ),

                        dbc.Row(
                            [
                                dbc.Col(html.Div([
                                    #---Recovered---
                                    html.P("Initial Recovered (R):", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='R_size', placeholder='', type='number', value = 0),

                                ])),
                                dbc.Col(html.Div([
                                    #---simulated days---
                                    html.P("Contact Rate:", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='c', placeholder='', type='number', value = 1),

                                ])),
                            ], className="g-0",
                            ),

                        dbc.Row(
                            [
                                dbc.Col(html.Div([
                                    #---Infected---
                                    html.P("Initail Infected (I):", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='I_size', placeholder='', type='number', value = 1),

                                ])),
                                dbc.Col(html.Div([
                                    #---simulated days---
                                    html.P("Waning Immunity Rate:", className="control_label", style={"font-weight": "bold", "text-align": "left"}),
                                    dcc.Input(id='h', placeholder='', type='number', value = 0),

                                ])),
                            ], className="g-0",
                            ),

                        #---n_slider---
                        html.P("Select the compartment size: ", className="control_label", style={"font-weight": "bold", "text-align": "center"}),
                        dcc.Slider(1, 20, step=1, value=10, id='n_val'),

                        #---SInR Button---
                        html.Br(),
                        html.Br(),
                        html.Button(id='model_button', children="Model", n_clicks=0, style=dict(width='220px')),

                        ], className="pretty_container four columns",
                ),

                html.Div(
                    [#----------------SInR Plot-----------
                        html.H6(('SI',html.Sub("n"),'R Covid-19 Modeling'), style={"margin-top": "0","font-weight": "bold","text-align": "center"}),
                        html.P(id='model_parameters', style={"text-align": "justify"}),
                        #html.P("Visualize the how th population shifts.", className="control_label",style={"text-align": "justify"}),

                        dcc.Graph(id="SInR_plot", style={'height': '60vh'}),

                        html.Div([ ], id='plot1'),
                    ],
                    className="pretty_container seven columns",
                ),
            ],
            className="row flex-display",
        ),

    ],
    id="mainContainer",
    style={"display": "flex", "flex-direction": "column"},
)

##########################################################################
#                          LINK TO EACH PAGE                             #
##########################################################################
@app.callback(
    Output(component_id="output-div",component_property="children"),
    Input(component_id="url",component_property="pathname"))

def update_page_layout(pathname):
    if pathname == "/simulate":
        return  sim_layout
    else:
        return home_layout

##########################################################################
#             SIMULATE AND VISUALIZE DISTRIBUTION DATA SETS              #
##########################################################################
@app.callback(
    [Output('gi_plots', 'figure'),
     Output('shape_scale', 'children')],
    [Input(component_id='update_button', component_property='n_clicks')],
    [State('stan_dev', 'value'),
    State('pop_mean', 'value'),
    State('gi_size', 'value')]
)

def update_sim_gi(n_clicks, sd, mean, size):
    '''
    This callback and function combination simulates
    a desired distribution (either gamma, weibull, or log-normal)
    given the information.
    '''
    #----------CREATE DISTRIBUTIONS---------
    gamma_data, lognorm_data, weibull_data = create_gi(mean, sd, size)

    mean_vals = [np.mean(gamma_data), np.mean(lognorm_data), np.mean(weibull_data)]
    std_vals = [np.std(gamma_data), np.std(lognorm_data), np.std(weibull_data)]

    #--------------VISUALIZE----------------
    gi_fig = gi_visualize(gamma_data, lognorm_data, weibull_data)

    return gi_fig, f'Given the input mean and standard deviation of {mean} and {sd} respectively,    the distributions are as follows: Gamma (x\u0305={round(mean_vals[0],3)}, s={round(std_vals[0],3)}).    Lognormal(x\u0305 ={round(mean_vals[1],3)}, s={round(std_vals[1],3)}).    Weibull(x\u0305={round(mean_vals[2],3)}, s={round(std_vals[2],3)}).'

##########################################################################
#                         CREATE AND PLOT BETA(u)                        #
##########################################################################
@app.callback(
    [Output('beta_plot', 'figure')],  #Output('weights_beta_info', 'children'), Output('lambdas', 'children'), Output('weights', 'children'), Output('betas', 'children')],
    [Input(component_id='beta_update_button', component_property='n_clicks')],
    [State('R0', 'value'),  #DISTRIBUTION STATES
     State('stan_dev', 'value'),
     State('pop_mean', 'value'),
     State('gi_size', 'value'),
    ]
)

def update_beta_u_plot(n_click, R0, sd, mean, size):
    '''
    Function will run beta_u function once "Calculate Beta(u)" button is clicked
    '''

    #----------CREATE DISTRIBUTIONS---------
    gamma_data, lognorm_data, weibull_data = create_gi(mean, sd, size)

    #----determine mimnimal acceptable size -----
    g_min_comp = math.ceil((np.mean(gamma_data)**2)/(np.var(gamma_data)))
    l_min_comp = math.ceil((np.mean(lognorm_data)**2)/(np.var(lognorm_data)))
    w_min_comp = math.ceil((np.mean(weibull_data)**2)/(np.var(weibull_data)))

    min_acceptable = max(g_min_comp, l_min_comp, w_min_comp)

    #----------------CALC VALS------------------
    w_gamma, l_gamma, b_gamma = solver(gamma_data, R0, min_acceptable, "gamma")
    w_lognorm, l_lognorm, b_lognorm = solver(lognorm_data, R0, min_acceptable, "lognorm")
    w_weibull, l_weibull, b_weibull = solver(weibull_data, R0, min_acceptable, "weibull")

    #----------------PLOT Beta(u)------------------
    b_n = [b_gamma, b_lognorm, b_weibull]
    l_n = [l_gamma, l_lognorm, l_weibull]
    w_n = [w_gamma, w_lognorm, w_weibull]

    beta_plot = plot_beta_dist(b_n, l_n)

    return [go.Figure(data=beta_plot)]

##########################################################################
#                           UPDATE SInR MODEL                            #
##########################################################################
@app.callback(
    [Output('SInR_plot', 'figure'),],#Output(component_id="plot1", component_property="children"),],
    #Output(component_id='model_parameters', component_property='children')],
    [Input(component_id='model_button', component_property='n_clicks')],
    [State('stan_dev', 'value'),
    State('pop_mean', 'value'),
    State('gi_size', 'value'),
    State('distribution-dropdown', 'value'),
    State('N_size', 'value'),
    State('I_size', 'value'),
    State('R_size', 'value'),
    State('t_days', 'value'),
    State('n_val', 'value'),
    State('R0', 'value'),
    State('c', 'value'),
    State('h', 'value')]
)

def update_SInR_plot(n_click, sd, mean, size, show, N, I1_t0, R_t0, days, n, R0, c_val, h_val):
    '''
    Visualize the SInR(S) plot
    '''

    gamma_data, lognorm_data, weibull_data = create_gi(mean, sd, size)

    #----determine mimnimal acceptable size -----
    g_min_comp = math.ceil((np.mean(gamma_data)**2)/(np.var(gamma_data)))
    l_min_comp = math.ceil((np.mean(lognorm_data)**2)/(np.var(lognorm_data)))
    w_min_comp = math.ceil((np.mean(weibull_data)**2)/(np.var(weibull_data)))

    #----------------CALC VALS------------------
    w_gamma, l_gamma, b_gamma = solver(gamma_data, R0, n, "gamma")
    w_lognorm, l_lognorm, b_lognorm = solver(lognorm_data, R0, n, "lognorm")
    w_weibull, l_weibull, b_weibull = solver(weibull_data, R0, n, "weibull")

    #-------Create lists of data-----
    b_n = [b_gamma, b_lognorm, b_weibull]
    l_n = [l_gamma, l_lognorm, l_weibull]
    w_n = [w_gamma, w_lognorm, w_weibull]

    print(b_n)

    color=["skyblue","darkorange","green"]
    dist = ["Gamma", "Lognormal","Weibull"]
    count=0

    #----specify compartments-------
    I_t0 = [I1_t0]+(n-1)*[0]
    S_t0 = N - sum(I_t0) - R_t0
    y_t0 = [S_t0]+ I_t0 +[R_t0]
    t = np.array(list(range(0,days+1)))

    #----specify model type-----
    if h_val== 0:
        model_type = "SI\u2099R Model of "
    else:
        model_type = "SI\u2099RS Model of "

    if show == "all":
        SInR_compare = go.Figure()
        dash = ["dot","dash"]

        for b,l in zip(b_n, l_n):
            #SIR MODEL
            if count == 0:
                fig = SInR_plot(y_t0, t, N, c_val, h_val, b, l)

                s_data = list(fig['data'][0]['y'])
                i_data = list(fig['data'][1]['y'])
                r_data = list(fig['data'][2]['y'])

                SInR_compare.update_layout(title=model_type+" for all Generation Intervals",
                                 legend=dict(yanchor="top", y=-0.2, xanchor="left",x=0.02, orientation="h"),
                                           font_size=14)

                SInR_compare.add_trace(go.Scatter(x=t, y= s_data, name= "Susceptible: " +dist[count]+" GI",
                                         line_color="blue",))
                SInR_compare.add_trace(go.Scatter(x=t, y= i_data, name= "Infected:     " +dist[count]+" GI",
                                         line_color="red",))
                SInR_compare.add_trace(go.Scatter(x=t, y= r_data, name= "Recovered:  " +dist[count]+" GI",
                                         line_color="green"))

                count+=1

            else:
                fig2 = SInR_plot(y_t0, t, N, c_val, h_val, b, l)
                s_data = list(fig2['data'][0]['y'])
                i_data = list(fig2['data'][1]['y'])
                r_data = list(fig2['data'][2]['y'])

                SInR_compare.add_trace(go.Scatter(x=t, y= s_data, name= "Susceptible: " +dist[count]+" GI",
                                         line_color="blue", line_dash=dash[count-1]))
                SInR_compare.add_trace(go.Scatter(x=t, y= i_data, name= "Infected:     " +dist[count]+" GI",
                                         line_color="red", line_dash=dash[count-1]))
                SInR_compare.add_trace(go.Scatter(x=t, y= r_data, name= "Recovered:  " +dist[count]+" GI",
                                         line_color="green", line_dash=dash[count-1]))

                count+=1
        SInR_compare.update_xaxes(title_text='Time (Days)')#,nticks=20)
        SInR_compare.update_yaxes(title_text='Percentage of the Population')

        return [go.Figure(data=SInR_compare)]

    else:
        if show == "gamma":
            index = 0
        if show == "lognorm":
            index = 1
        else:
            index=2

        I_t0 = [I1_t0]+(n-1)*[0]
        S_t0 = N - sum(I_t0) - R_t0
        y_t0 = [S_t0]+ I_t0 +[R_t0]
        t = np.array(list(range(0,days+1)))

        #SIR MODEL
        fig = SInR_plot(y_t0, t, N, c_val, h_val, b_n[index], l_n[index])
        fig.update_layout(title=model_type+" of "+dist[index]+" Generation Interval",
                         legend=dict(yanchor="top", y=0.35, xanchor="left", x=0.01),font_size=14)
        return [go.Figure(data=fig)]

##########################################################################
#                             RUN THE APP                                #
##########################################################################
if __name__ == "__main__":
    app.run_server(debug=False)

#TABLE STYLING:
#https://dash.plotly.com/datatable/style

# <br><br><br><br>


from django.urls import path, include
from django.conf.urls import url

from mysite import views

urlpatterns = [

    path('search/', views.search_view, name='search'),
    url(r'^request/(?P<record_id>[-\w]+)/$', views.send_req_view, name='request'),
    path('requests/', views.req_view, name='requests'),
    path('profile/', views.profile_view, name='profile'),
    url(r'^profile/edit/$', views.edit_profile, name='edit_profile'),
    path('', views.home_view, name='home'),
    path('about/', views.about_view, name='about'),
    url(r'^requests/delete/(?P<req_id>[-\w]+)/$', views.delete_req_view, name='d_request'),
    url(r'^requests/accept/(?P<req_id>[-\w]+)/$', views.accept_req_view, name='accept_request'),
    url(r'^requests/reject/(?P<req_id>[-\w]+)/$', views.reject_req_view, name='reject_request'),
    path('notifications/', views.notification_view, name='notifications'),
    path('profile/become_donor', views.become_donor_view, name='become_donor'),
    path('add_record', views.add_record_view, name='add_record'),

]


import bluetooth

class Alpha1S:

    """
    Class to control the Ubtech Alpha 1S robot
    """

    def __init__(self, name="ALPHA 1S"):
        self.__bt = self.Alpha1S_bluetooth(name)

    def battery(self):
        """
        Get battery information.

        Returns:
        dict: Dictionary with fields:
            percent: Remaining battery capacity
            state:
                0: Battery not charging
                1: Battery charging
                2: Battery not present
            mV: Battery voltage in mV
        """
        msg = b'\x18\x00'
        parameter_len = 4
        ans = self.__bt.read(msg, parameter_len)
        if ans is not None:
            battery = {
                "percent": int.from_bytes(ans[3:], "big"),
                "state": int.from_bytes(ans[2:3], "big"),
                "mV": int.from_bytes(ans[:2], "big")
            }
            return battery
        return None

    def leds(self, state):
        """
        Turn LEDs on or off.

        Parameters:
        state: Set True to turn on, False to turn off.
        """
        if state:
            state = b'\x01'
        else:
            state = b'\x00'
        msg = b'\x0D' + state
        self.__bt.write(msg)

    def servo_read(self, servo_id):
        """
        Read the position of a single servo.
        Note: Reading a servo will automatically power it off.

        Parameters:
        servo_id: Servo id between 0-15, integer

        Returns:
        int: Position of the servo between 0-180
        """
        # Adding 1 to the servo_id because the robot starts counting at 1
        servo_id = bytes([servo_id+1])
        msg = b'\x24' + servo_id
        parameter_len = 2
        ans = self.__bt.read(msg, parameter_len)
        if ans is not None:
            # Check that the received value corresponds to the specified servo
            if ans[:1] == servo_id:
                return int.from_bytes(ans[1:], "big")
        return None

    def servo_read_all(self):
        """
        Read the positions for all the servos simultaneously.
        Note: Reading a servo will automatically power it off.

        Returns:
        [int]: List of 16 integer positions between 0-180
        """
        msg = b'\x25\x00'
        parameter_len = 16
        ans = self.__bt.read(msg, parameter_len)
        if ans is not None:
            return [x for x in ans]
        return None

    def servo_write(self, servo_id, angle, travelling=20):
        """
        Set a specific servo to an angle.

        Parameters:
        servo_id: Servo id between 0-15, integer
        angle: Angle between 0-180, integer
        travelling: Time the servo takes to move to the position

        Returns:
        int: Error code:
            0: Success
            1: Wrong servo servo_id
            2: Allow servo angle excess
            3: No reply from servo
        """
        # Adding 1 to the servo_id because the robot starts counting at 1
        servo_id = bytes([servo_id+1])
        angle = bytes([angle])
        run_time = bytes([travelling])
        time_frames = b'\x00\x10'
        msg = b'\x22' + servo_id + angle + run_time + time_frames
        parameter_len = 2
        ans = self.__bt.read(msg, parameter_len)
        if ans is not None:
            # Check that the received value corresponds to the specified servo
            if ans[:1] == servo_id:
                return int.from_bytes(ans[1:], "big")
        return None

    def servo_write_all(self, angles, travelling=20):
        """
        Set all servos to the specified positions simultaneously.

        Parameters:
        angles: List of integer angles between 0-180
        travelling: Time the servo takes to move to the position, integer

        Returns:
        [int]: List of error codes for each servo:
            0: Success
            1: Wrong servo servo_id
            2: Allow servo angle excess
            3: No reply from servo
        """
        if len(angles) != 16:
            return None
        angles = bytearray(angles)
        run_time = bytes([travelling])
        time_frames = b'\x00\x10'
        msg = b'\x23' + angles + run_time + time_frames
        parameter_len = 16
        ans = self.__bt.read(msg, parameter_len)
        if ans is not None:
            return [x for x in ans]
        return None

    def servo_off(self):
        """
        Send command to power off all the servos in the robot.
        """
        msg = b'\x0C\x00'
        self.__bt.write(msg)

    class Alpha1S_bluetooth:

        """
        Class to handle the Alpha1S' bluetooth protocol
        Download Bluetooth protocol datasheet from
        https://assets-new.ubtrobot.com/downloads/Alpha%201%20Series%20Bluetooth%20communication%20protocol?download
        """  # noqa

        def __init__(self, name):
            address = self.__discover(name)
            assert(address is not None), f"Error: {name} not found"
            self.__connect(address)

        def __del__(self):
            self.sock.close()

        def __discover(self, name):
            address = None
            devices = bluetooth.discover_devices(lookup_names=True)
            for add, text in devices:
                if text == name:
                    address = add
                    break
            return address

        def __connect(self, addr):
            self.sock = bluetooth.BluetoothSocket(bluetooth.RFCOMM)
            self.sock.connect((addr, 6))
            self.sock.settimeout(10.0)

        def write(self, msg):
            """
            Compose an outgoing message following the required format and send
            over the bluetooth socket. Takes bytes as input.
            """
            cmd = self.__compose(msg)
            self.sock.send(cmd)

        def read(self, msg, ans_len):
            """
            Use the write() function to send a command and receive its answer.

            Returns the 'Parameter' field in bytes if the message was received
            correctly, None otherwise.
            """
            self.write(msg)
            # Length is sum of header(2), length, check, cmd, ans_len and end
            length = 6 + ans_len
            ans = self.sock.recv(length)
            if self.__check(ans):
                return ans[4:-2]
            return None

        def __compose(self, msg):
            """
            Compose a byte message with the header, length, check and end
            bytes in the required format.
            """
            header = b'\xFB\xBF'
            end = b'\xED'
            # Length is sum of header(2), length, check + msg bytes
            length = bytes([4 + len(msg)])
            # Check is sum of length + msg (length+(cmd+params)), with modulus
            # to fit into a single byte
            check_list = bytearray(length)
            check_list.extend(msg)
            check = bytes([sum(check_list) % 256])
            return header + length + msg + check + end

        def __check(self, msg):
            """
            Check that the received message follows the correct format and that
            the check byte is correct.

            Returns True if message is correct, False otherwise
            """
            msg = bytearray(msg)
            # Check that header is correct
            if msg[:2] != b'\xFB\xBF':
                return False
            # Check that ending is correct
            elif msg[-1:] != b'\xED':
                return False
            # Check that check byte is correct
            elif msg[-2:-1] != bytes([sum(msg[2:-2]) % 256]):
                return False
            else:
                return True


# pylint: disable=protected-access
# pylint: disable=redefined-outer-name
# pylint: disable=unused-argument

import urllib.parse
from pathlib import Path
from random import choice
from typing import Awaitable, Callable

import pytest
from aiohttp import web
from aiohttp.test_utils import TestClient
from faker import Faker
from models_library.api_schemas_storage import FileMetaDataGet, SimcoreS3FileID
from models_library.projects import ProjectID
from models_library.users import UserID
from pydantic import ByteSize, parse_obj_as
from pytest_simcore.helpers.utils_assert import assert_status

pytest_simcore_core_services_selection = ["postgres"]
pytest_simcore_ops_services_selection = ["adminer"]

async def test_get_files_metadata(
    upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]],
    client: TestClient,
    user_id: UserID,
    location_id: int,
    project_id: ProjectID,
    faker: Faker,
):
    assert client.app

    url = (
        client.app.router["get_files_metadata"]
        .url_for(location_id=f"{location_id}")
        .with_query(user_id=f"{user_id}")
    )

    # this should return an empty list
    response = await client.get(f"{url}")
    data, error = await assert_status(response, web.HTTPOk)
    assert not error
    list_fmds = parse_obj_as(list[FileMetaDataGet], data)
    assert not list_fmds

    # now add some stuff there
    NUM_FILES = 10
    file_size = parse_obj_as(ByteSize, "15Mib")
    files_owned_by_us = []
    for _ in range(NUM_FILES):
        files_owned_by_us.append(await upload_file(file_size, faker.file_name()))
    # we should find these files now
    response = await client.get(f"{url}")
    data, error = await assert_status(response, web.HTTPOk)
    assert not error
    list_fmds = parse_obj_as(list[FileMetaDataGet], data)
    assert len(list_fmds) == NUM_FILES
    # create some more files but with a base common name
    NUM_FILES = 10
    file_size = parse_obj_as(ByteSize, "15Mib")
    files_with_common_name = []
    for _ in range(NUM_FILES):
        files_with_common_name.append(
            await upload_file(file_size, f"common_name-{faker.file_name()}")
        )
    # we should find these files now
    response = await client.get(f"{url}")
    data, error = await assert_status(response, web.HTTPOk)
    assert not error
    list_fmds = parse_obj_as(list[FileMetaDataGet], data)
    assert len(list_fmds) == (2 * NUM_FILES)
    # we can filter them now
    response = await client.get(f"{url.update_query(uuid_filter='common_name')}")
    data, error = await assert_status(response, web.HTTPOk)
    assert not error
    list_fmds = parse_obj_as(list[FileMetaDataGet], data)
    assert len(list_fmds) == (NUM_FILES)

@pytest.mark.xfail(
    reason="storage get_file_metadata must return a 200 with no payload as long as legacy services are around!!"
)
async def test_get_file_metadata_is_legacy_services_compatible(
    client: TestClient,
    user_id: UserID,
    location_id: int,
    simcore_file_id: SimcoreS3FileID,
):
    assert client.app

    url = (
        client.app.router["get_file_metadata"]
        .url_for(
            location_id=f"{location_id}",
            file_id=f"{urllib.parse.quote(simcore_file_id, safe='')}",
        )
        .with_query(user_id=f"{user_id}")
    )
    # this should return an empty list
    response = await client.get(f"{url}")
    await assert_status(response, web.HTTPNotFound)

async def test_get_file_metadata(
    upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]],
    client: TestClient,
    user_id: UserID,
    location_id: int,
    project_id: ProjectID,
    simcore_file_id: SimcoreS3FileID,
    faker: Faker,
):
    assert client.app

    url = (
        client.app.router["get_file_metadata"]
        .url_for(
            location_id=f"{location_id}",
            file_id=f"{urllib.parse.quote(simcore_file_id, safe='')}",
        )
        .with_query(user_id=f"{user_id}")
    )
    # this should return an empty list
    response = await client.get(f"{url}")
    # await assert_status(response, web.HTTPNotFound)

    # NOTE: This needs to be a Ok response with empty data until ALL legacy services are gone, then it should be changed to 404! see test above
    assert response.status == web.HTTPOk.status_code
    assert await response.json() == {"data": {}, "error": "No result found"}

    # now add some stuff there
    NUM_FILES = 10
    file_size = parse_obj_as(ByteSize, "15Mib")
    files_owned_by_us = []
    for _ in range(NUM_FILES):
        files_owned_by_us.append(await upload_file(file_size, faker.file_name()))
    selected_file, selected_file_uuid = choice(files_owned_by_us)
    url = (
        client.app.router["get_file_metadata"]
        .url_for(
            location_id=f"{location_id}",
            file_id=f"{urllib.parse.quote(selected_file_uuid, safe='')}",
        )
        .with_query(user_id=f"{user_id}")
    )
    response = await client.get(f"{url}")
    data, error = await assert_status(response, web.HTTPOk)
    assert not error
    assert data
    fmd = parse_obj_as(FileMetaDataGet, data)
    assert fmd.file_id == selected_file_uuid
    assert fmd.file_size == selected_file.stat().st_size


import cv2 as cv

# Cargamos la imagen y transformamos en blanco y negro
img_original = cv.imread("imgs/Cuadrados.jpg")
img_bnw = cv.cvtColor(img_original, cv.COLOR_BGR2GRAY)

# Aplicamos la funcion de deteccion de esquinas
maxCorners = 20
esquinas = cv.goodFeaturesToTrack(img_bnw, maxCorners, 0.01, 10)

# Definimos el criterio de stop para la precision subpixel y lo aplicamos
criterio_stop = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, maxCorners, 0.0001)
esquinas_final = cv.cornerSubPix(img_bnw, esquinas, (5,5), (2,2), criterio_stop)

# Dibujamos sobre la imagen
for esquina in esquinas_final:
    x, y = esquina.ravel()
    cv.circle(img_original, (int(x), int(y)), 2, (0,0,0), -1)

cv.namedWindow("original", cv.WINDOW_NORMAL)
cv.imshow("original", img_original)
cv.waitKey()
cv.destroyAllWindows()


#coding: utf-8
import urllib
import BaseHTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import webbrowser
import os
from threading import Timer
import base64
from ui import Image
import console
import sys
import io

sys.stderr = io.StringIO()

console.show_activity('Creating images…')

imagefilenames = [os.path.splitext(i)[0] for i in open(os.path.expanduser('~/Pythonista.app/Typicons-M.txt')).readlines()]

imagenames = [i.replace('Typicons96_', '') for i in imagefilenames]

images = {n: Image.named(imagefilenames[i]) for (i, n) in enumerate(imagenames)}

imageurls = {k:'data:image/png;base64,'+base64.b64encode(images[k].to_png()) for k in images}

choosecolorpath = os.path.dirname(sys.argv[0].split('/Documents/',1)[1]) + '/choosecolor.py'

tagtemplate = ''

imagetags = [tagtemplate%(k,imageurls[k]) for k in imagenames]

imagesstring = ''.join(imagetags)

html = '''
<!DOCTYPE html>

<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">

Choose an Icon

%s

''' % imagesstring

class RequestHandler (SimpleHTTPRequestHandler):
	def do_GET(self):
		self.send_response(200)
		self.send_header("Content-type", "text/html")
		self.end_headers()
		self.wfile.write(html)
	def log_message(self, format, *args):
		pass

serv = BaseHTTPServer.HTTPServer(('', 0), RequestHandler)
port = serv.server_port

Timer(1, webbrowser.open, ('http://localhost:%d'%port,)).start()

console.show_activity('Starting server…')
serv.handle_request()
console.hide_activity()


import numpy as np
import logging
import sys
import pkg_resources
import pytz
import datetime
import os
import re
from numpy import cos,sin

# Get the version
version_file = pkg_resources.resource_filename('pynortek','VERSION')

# Setup logging module
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
logger = logging.getLogger('pynortek')

def xyz2enu(u,v,w,head,pitch,roll,inverse=False):
    """
    Transforms velocities in XYZ coordinates to ENU, or vice versa if
    inverse=True. Transformation is done according to the Nortek
    convention

    """
    # convert to radians
    hh = np.pi*(head-90)/180
    pp = np.pi*pitch/180
    rr = np.pi*roll/180

    ut = np.zeros(np.shape(u))
    vt = np.zeros(np.shape(u))
    wt = np.zeros(np.shape(u))

    for i in range(len(head)):
        # generate heading matrix
        H = np.matrix([[cos(hh[i]), sin(hh[i]), 0],[-sin(hh[i]), cos(hh[i]), 0],[0, 0, 1]])
        # generate combined pitch and roll matrix
        P = [[cos(pp[i]), -sin(pp[i])*sin(rr[i]), -cos(rr[i])*sin(pp[i])],
             [0,           cos(rr[i]),                       -sin(rr[i])],
             [sin(pp[i]),  sin(rr[i])*cos(pp[i]),  cos(pp[i])*cos(rr[i])]]

        R = H*P
        #print(R)
        if(inverse):
            R = np.inv(R)

        # do transformation
        ut[i]  = R[0,0]*u[i] + R[0,1]*v[i] + R[0,2]*w[i];
        vt[i]  = R[1,0]*u[i] + R[1,1]*v[i] + R[1,2]*w[i];
        wt[i]  = R[2,0]*u[i] + R[2,1]*v[i] + R[2,2]*w[i];

    return [ut,vt,wt]

raw_data_files = ['.prf','.vec'] # Names of raw binary data files
class pynortek():
    """A Nortek parsing object

    Author: Peter Holtermann (peter.holtermann@io-warnemuende.de)

    Usage:
       >>>filename='test'
       >>>aquadopp = pynortek(filename)

    """
    def __init__(self,filename, verbosity=logging.DEBUG, timezone=pytz.UTC):
        """
        """
        logger.setLevel(verbosity)
        self.timezone = timezone
        self.deployment = os.path.split(filename)[-1]
        self.fpath = os.path.split(filename)[0]
        self.rawdata = {}
        print(self.deployment)
        print(self.fpath)
        filename_hdr = filename + '.hdr'
        logger.debug('Trying to open header file: ' + filename_hdr)
        try:
            fhdr = open(filename_hdr)
        except Exception as e:
            logger.warning('Could not open header file, exiting')
            return

        header = self.parse_header(fhdr)
        self.header = header
        print(header)
        print('Loading files')
        for fread in header['files']:
            print(fread)
            IS_RAW = False
            for rawname in raw_data_files:
                if(rawname in fread.lower()):
                    IS_RAW=True

            if(IS_RAW == False):
                print('Loading ' + fread)
                suffix = fread.split('.')[-1]
                fname_tmp = os.path.join(self.fpath,fread)
                print(fname_tmp)
                data_tmp = np.loadtxt(fname_tmp)
                self.rawdata[suffix] = data_tmp

        # Process the raw data just loaded
        self.process_rawdata()

    def parse_header(self,fhdr):
        """ Parses a nortek header file
        """
        header = {}
        datefmt = '%d.%m.%Y %H:%M:%S'
        header_field = None
        header['files'] = []
        while True:
            l = fhdr.readline()
            if(len(l) == 0):
                break

            # Find all files to be read
            if((l[0] == '[')):
                ftmp = l.split("\\")[-1].replace(']','').replace('\n','')
                header['files'].append(ftmp)
                # If we have a sensor file, check position of fields
                if('.sen' in l[-7:]):
                    print('Sensor file')
                    header_field = 'sensors'
                    header[header_field] = {}

            # Transducer distance
            if(('Beam' in l) and ('Vertical' in l)):
                print('Transducer distance')
                header_field = 'distance'
                header[header_field] = {'cell':[],'beam':[],'vertical':[]}
                continue

            # Check for the header field
            if('User setup' in l):
                print('User setup')
                header_field = 'User setup'
                header[header_field] = {}
            elif('Hardware configuration' in l):
                print('Hardware configuration')
                header_field = 'Hardware configuration'
                header[header_field] = {}
            elif('Head configuration' in l):
                header_field = 'Head configuration'
                header[header_field] = {}

            #print(l)
            if(header_field is not None): # Check if field is over (one empty line)
                if(len(l) <= 2):
                    print('Header ' + header_field + ' over')
                    header_field = None

            # Check for a one line list
            ind = l.find('  ')
            if(ind >= 0):
                if('Number of measurements' in l):
                    header['Number of measurements'] = int(l.split()[-1])
                elif('Coordinate system' in l):
                    header['Coordinate system'] = l.split()[-1]
                    logger.debug('Coordinate system found: ' + header['Coordinate system'])
                elif('Horizontal velocity range' in l):
                    header['Horizontal velocity range'] = float(l.split()[-2])
                    logger.debug('Horizontal velocity range: ' + str(header['Horizontal velocity range']))
                elif('Vertical velocity range' in l):
                    header['Vertical velocity range'] = float(l.split()[-2])
                    logger.debug('Vertical velocity range: ' + str(header['Vertical velocity range']))
                elif('Orientation' in l):
                    header['Orientation'] = l.split()[-1]
                    if('DOWN' in header['Orientation']):
                        header['updown'] = True
                    else:
                        header['updown'] = False

                    logger.debug('Orientation ' + header['Orientation'] + ' updown:' + str(header['updown']))

                elif('Number of checksum errors' in l):
                    header['Number of checksum errors'] = int(l.split()[-1])
                elif('Time of first measurement' in l):
                    ind2 = l.rfind('  ')
                    tstr = l[ind2+2:].replace('\n','')
                    ttmp = datetime.datetime.strptime(tstr,datefmt)
                    ttmp = ttmp.replace(tzinfo=self.timezone)
                    header['Time of first measurement'] = ttmp
                elif('Time of last measurement' in l):
                    ind2 = l.rfind('  ')
                    tstr = l[ind2+2:].replace('\n','')
                    ttmp = datetime.datetime.strptime(tstr,datefmt)
                    ttmp = ttmp.replace(tzinfo=self.timezone)
                    header['Time of last measurement'] = ttmp
                elif('Transformation matrix' in l):
                    logger.debug('Transformation matrix found')
                    header['Transformation matrix'] = np.zeros((3,3))
                    # Get all three lines
                    tmp = []
                    tmp.append(l)
                    tmp.append(fhdr.readline())
                    tmp.append(fhdr.readline())
                    for i in range(3):
                        T_tmp = np.asarray(tmp[i].split()[-3:]).astype(np.float)
                        header['Transformation matrix'][i,:] = T_tmp

                    logger.debug(str(header['Transformation matrix']))

                elif('Magnetometer calibration matrix' in l):
                    logger.debug('Magnetometer calibration matrix found')
                    header['Magnetometer calibration matrix'] = np.zeros((3,3))
                    # Get all three lines
                    tmp = []
                    tmp.append(l)
                    tmp.append(fhdr.readline())
                    tmp.append(fhdr.readline())
                    for i in range(3):
                        T_tmp = np.asarray(tmp[i].split()[-3:]).astype(np.float)
                        header['Magnetometer calibration matrix'][i,:] = T_tmp

                    logger.debug(str(header['Magnetometer calibration matrix']))

                else:
                    pass

                if(header_field is not None):
                    if(header_field == 'sensors'):
                        l = l.replace('\n','').replace('\r','').strip() # remove return and trailing/leading blanks
                        lsp = re.sub("  +" , "\t", l).split('\t')
                        print('sensors',lsp)
                        field = lsp[1]
                        value = lsp[0]
                        header[header_field][field] = int(value)
                    elif(header_field == 'distance'):
                        l = l.replace('\n','').replace('\r','').strip() # remove return and trailing/leading blanks
                        lsp = re.sub("  +" , "\t", l).split('\t')
                        cell = lsp[0]
                        beam = lsp[1]
                        vertical = lsp[2]
                        print(cell,beam,vertical)
                        header[header_field]['cell'].append(int(cell))
                        header[header_field]['beam'].append(float(beam))
                        header[header_field]['vertical'].append(float(vertical))
                    else:
                        ind2 = l.rfind('  ')
                        data = l[ind2+2:].replace('\n','').replace('\r','')
                        field = l[:ind].replace('\n','').replace('\r','')
                        header[header_field][field] = data

            #print(l.split())

        return header

    def process_rawdata(self):
        """ Processes .sen data stored in data['sen'] and the remaining rawdata
        """
        print('Creating time axis')
        t  = []
        tu = []
        for i in range(np.shape(self.rawdata['sen'][:,0])[0]):
            month  = int(self.rawdata['sen'][i,0])
            day    = int(self.rawdata['sen'][i,1])
            year   = int(self.rawdata['sen'][i,2])
            hour   = int(self.rawdata['sen'][i,3])
            minute = int(self.rawdata['sen'][i,4])
            millis = self.rawdata['sen'][i,5]%1
            second = int(self.rawdata['sen'][i,5] - millis)
            micro  = int(millis*1000*1000)
            ttmp = datetime.datetime(year,month,day,hour,minute,second,micro,tzinfo=self.timezone)
            t.append(ttmp)
            tu.append(ttmp.timestamp())

        self.t = t # datetime time
        self.tu = tu # unix time
        self.data = {}
        for k in self.header['sensors'].keys():
            ind_key = self.header['sensors'][k] - 1
            self.data[k] = self.rawdata['sen'][:,ind_key]

        # Processing the remaining data
        # For a profiler (Aquadopp)
        aquadopp_keys = ['v1','v2','v3','a1','a2','a3','c1','c2','c3']
        for key in aquadopp_keys:
            if(key in self.rawdata.keys()):
               print('Getting data from: ' + key + ' (profiler)')
               self.data[key] = self.rawdata[key][:,2:]

        if('distance' in self.header.keys()):
            self.data['dis_beam'] = np.asarray(self.header['distance']['beam'])
            self.data['dis_vertical'] = np.asarray(self.header['distance']['vertical'])

        vector_keys = ['dat']
        for key in vector_keys:
            if(key in self.rawdata.keys()):
               print('Getting data from: ' + key + ' (Vector)')
               self.data[key] = self.rawdata[key][:,2:]

    def rot_vel(self,coord,updown=None,save=False):
        """ Rotates the velocities to different coordinate system
        Args:
            coord:
            updown:
            save:
        """
        logger.debug('trans_coord():')
        T = self.header['Transformation matrix'][:]
        if(updown == None):
            updown = self.header['updown']

        # flip axes if instrument is pointing downward
        # (so from here on, XYZ refers to a right-handed coordinate system
        # with z pointing upward)

        if updown:
            logger.debug('Downlooking, changing matrix')
            T[1,:] = -T[1,:];
            T[2,:] = -T[2,:];

        v1_rot = np.zeros(np.shape(self.data['v1']))
        v2_rot = np.zeros(np.shape(self.data['v2']))
        v3_rot = np.zeros(np.shape(self.data['v3']))
        try:
            v1_rep_rot = np.zeros(np.shape(self.data['v1_rep']))
            v2_rep_rot = np.zeros(np.shape(self.data['v2_rep']))
            v3_rep_rot = np.zeros(np.shape(self.data['v3_rep']))
            repaired = True
        except:
            repaired_false = True
            pass

        print(np.shape(self.data['v1']))
        if(coord == 'XYZ'):
            if(self.header['Coordinate system'] == 'BEAM'):
                logger.debug('BEAM to XYZ')
                for i in range(np.shape(v1_rot)[0]):
                    for j in range(np.shape(v1_rot)[1]):
                        v1_rot[i,j] = T[0,0] * self.data['v1'][i,j] + T[0,1] * self.data['v2'][i,j] + T[0,2] * self.data['v3'][i,j]
                        v2_rot[i,j] = T[1,0] * self.data['v1'][i,j] + T[1,1] * self.data['v2'][i,j] + T[1,2] * self.data['v3'][i,j]
                        v3_rot[i,j] = T[2,0] * self.data['v1'][i,j] + T[2,1] * self.data['v2'][i,j] + T[2,2] * self.data['v3'][i,j]
                        if repaired:
                            v1_rep_rot[i,j] = T[0,0] * self.data['v1_rep'][i,j] + T[0,1] * self.data['v2_rep'][i,j] + T[0,2] * self.data['v3_rep'][i,j]
                            v2_rep_rot[i,j] = T[1,0] * self.data['v1_rep'][i,j] + T[1,1] * self.data['v2_rep'][i,j] + T[1,2] * self.data['v3_rep'][i,j]
                            v3_rep_rot[i,j] = T[2,0] * self.data['v1_rep'][i,j] + T[2,1] * self.data['v2_rep'][i,j] + T[2,2] * self.data['v3_rep'][i,j]

        if save:
            logger.debug('saving data in trans')
            try: # Check if self.trans is existing
                self.trans
            except:
                self.rotvel = {}
            if(coord == 'XYZ'):
                self.rotvel['u'] = v1_rot[:]
                self.rotvel['v'] = v2_rot[:]
                self.rotvel['w'] = v3_rot[:]
                if repaired:
                    # Save the repaired data as well
                    self.rotvel['u_rep'] = v1_rep_rot[:]
                    self.rotvel['v_rep'] = v2_rep_rot[:]
                    self.rotvel['w_rep'] = v3_rep_rot[:]

        return [v1_rot,v2_rot,v3_rot]

    def repair_phase_shift(self,vel=None,threshold=None, save = False):
        """Tries to repair a phase shift in pulse coherent measurements. It
        assumes that the first measured value is correct.

        """

        if(vel == None):
            vel = self.data
            logger.debug('repairing native velocity')
            coordinate_system = self.header['Coordinate system']
            vel_all = [self.data['v1'],self.data['v2'],self.data['v3']]
        else:
            vel_all = [vel]

        vel_rep_all = []
        for vel_tmp in vel_all:
        # Compute threshold from header data
            if( coordinate_system == 'BEAM'):
                logger.debug('Using thresholds for beam coordinates')
                # Get the factor for the beam from the vertical velocity
                fac = np.linalg.inv(self.header['Transformation matrix'])[0,2]
                threshold_tmp = self.header['Vertical velocity range']# * fac
            else:
                logger.debug('Unknown threshold, returning')
                return

            vel_rep = np.zeros(np.shape(vel_tmp))
            for i in range(np.shape(vel_rep)[1]):
                vel_rep[:,i] = self.repair_phase_shift_vector(vel_tmp[:,i],threshold_tmp)

            vel_rep_all.append(vel_rep)

        print('hallo',vel is self.data)
        if((vel is self.data) and save):
            logger.debug("Saving data as data['v1_rep'] etc")
            self.data['v1_rep'] = vel_rep_all[0]
            self.data['v2_rep'] = vel_rep_all[1]
            self.data['v3_rep'] = vel_rep_all[2]

    def repair_phase_shift_vector(self,vel,threshold):
        """Tries to repair a phase shift in pulse coherent measurements. It
        assumes that the first measured value is correct.

        """

        vel_rep = vel.copy()

        vthresh = threshold - 0.3 * threshold
        for i in range(1,len(vel)):
            if((np.sign(vel_rep[i-1]) != np.sign(vel_rep[i])) and (abs(vel_rep[i-1]) > vthresh) and (abs(vel_rep[i]) > vthresh)):
               #print('Phase shift!')
               dv = threshold - abs(vel_rep[i])
               vel_rep[i] =  np.sign(vel_rep[i-1]) * (threshold + dv)

        return vel_rep


# ------------------------------------------------------------------#
# AMPBA - 2021 Winter :: Data Collection assignment - PART2         #
# Group Id : 3                                                      #
# Authors:                                                          #
#       Nishant Jalasutram      -   PG ID: 012020051                #
#       Ila Barshilia           -   PG ID: 012020022                #
#       Deep Kamal Singh        -   PG ID: 012020053                #
# ------------------------------------------------------------------#

'''
Part 2:
For each match, go to the scorecard link like
https://www.espncricinfo.com/series/icc-cricket-world-cup-2019-1144415/india-vs-new-zealand-1st-semi-final-1144528/full-scorecard and extract the following:
1. Player of the match with the picture. Save the url to the picture in the csv.
2. Country that the player of the match belongs to.
3. Runs scored by every batsman.
4. Balls played by every batsman.
5. Strike rate for every batsman.
6. Wickets taken by every bowler.
7. Economy rate for every bowler.
8. which country won the toss.
9. who were the umpires?
10. who was the match referee

Save results in a file [your group no]_matchDetails.csv.
Name the .py file as [your group no]_matchDetails.py.

'''

import scrapy

# As per Assignments requirement we need to start our file name with GROUP number, thus we have to use import this
# way, instead of simply writing from <File_name> import <ClassName>
CricinfoPlayerProfileSpider = __import__(
    '3_playerDetails').CricinfoPlayerProfileSpider  # ,fromlist=['3_playerDetails'])

class CWC2019MatchStatsSpider(scrapy.Spider):
    name = "cwc_2019_Scorecard_spider"
    cricinfo_host = 'https://www.espncricinfo.com'
    playerProfileParser = CricinfoPlayerProfileSpider(scrapy.Spider)

    # This method will be called for every inning of the match , and it will extract Batsmen and Bowlers
    def parseInning(self, anInning):
        return {
            "batsmen": self.parseBatsmen(anInning.css("table[class='table batsman']")),
            "bowlers": self.parseBowler(anInning.css("table[class='table bowler']")),
        }

    # This method extracts match details - iterates over every TR and creates a KV Pair dictionary
    #    thus making it independent of sequence of occurrence
    def parseMatchDetails(self, matchDetailSection):
        trs = matchDetailSection.css("tbody").css("tr")
        returnDict = {}
        for aRow in trs:
            tds = aRow.css("td")

            if tds is not None and len(tds) > 1:
                returnDict[tds[0].css('::text').extract()[0]] = tds[1].css('::text').extract()[0] if len(
                    tds[1].css('::text').extract()) == 1 else ",".join(tds[1].css('::text').extract())

        return returnDict

    # This method extract Player of the match information
    # however, we found the player of the match image is lazy loaded and Scrapy is not good enough for this
    # we tried with Selenium as well as with Beautifulsoup which made the code messy and inconsistent,
    # so we are now getting player pic in PART3 execution,and setting it back inside PART2 data set
    def parsePlayerOfMatchSection(self, playerOfMatchSection):

        return {
            "player_of_the_match": playerOfMatchSection.css('div[class="best-player-name"]').css(
                'a::text').extract_first(),
            "player_of_the_match_profile": playerOfMatchSection.css('div[class="best-player-name"]').css(
                'a::attr(href)').extract_first(),
            "player_of_the_match_image_url": "",  # We are not loading image now, as it will be lazyimg.png anyway -
            # we will get the player image from PART3 output dictionary later
            "player_of_the_match_country": playerOfMatchSection.css(
                'span[class="best-player-team-name"]::text').extract_first()
        }

    # Extracts batsmen details, also takes care of batsmen who did not bat
    def parseBatsmen(self, battingTable):
        # batting table parsing
        batsmenList = []
        for aBattingRow in battingTable.css("tbody").css("tr"):
            tds = aBattingRow.css("td::text").extract()

            if aBattingRow.css('.batsman-cell').css("a::text").extract_first() is not None:
                # Found that when batsman is NOT out we get back "not out" in first element instead of RUNS,
                # handling this
                if tds[0].isnumeric():
                    batsmenList.append({
                        "name": aBattingRow.css('.batsman-cell').css("a::text").extract_first().strip(),
                        "profile_url": aBattingRow.css('.batsman-cell').css("a::attr('href')").extract_first(),
                        "runs": tds[0],
                        "balls_played": tds[1],
                        "strike_rate": tds[5]
                    })
                else:
                    batsmenList.append({
                        "name": aBattingRow.css('.batsman-cell').css("a::text").extract_first().strip(),
                        "profile_url": aBattingRow.css('.batsman-cell').css("a::attr('href')").extract_first(),
                        "runs": tds[1],
                        "balls_played": tds[2],
                        "strike_rate": tds[6]
                    })

        # Are there any "Yet to bat" players - lets add them too
        if len(batsmenList) < 11 and len(battingTable.css("tfoot").css("tr")) > 1:
            didNotBatLinks = battingTable.css("tfoot").css("tr")[1].css("td")[0].css("div")[0].css("a")
            for aPlayer in didNotBatLinks:
                batsmenList.append({
                    "name": aPlayer.css('span::text').extract_first().strip().replace(" ", ""),
                    "profile_url": aPlayer.css("::attr(href)").extract_first(),
                    "runs": "",
                    "balls_played": "",
                    "strike_rate": ""
                })
        return batsmenList

    # Extracts Bowler details
    def parseBowler(self, bowlingScores):
        # parsing bowling scores to extract each bowler
        bowlerList = []
        for aBowlingStatRow in bowlingScores.css("tbody").css("tr"):
            tds = aBowlingStatRow.css("td::text").extract()
            if aBowlingStatRow.css('.text-nowrap').css("a::text").extract_first() is not None:
                bowlerList.append({
                    "name": aBowlingStatRow.css('.text-nowrap').css("a::text").extract_first().strip(),
                    "profile_url": aBowlingStatRow.css('.text-nowrap').css("a::attr('href')").extract_first(),
                    "wickets": tds[3],
                    "econ": tds[4]
                })
        return bowlerList

    # This function is called when Part 1 yields a scrapy request
    # which is processed by Scrapy and response is handed over to this function
    # in addition it is passed with match_details_dict which is dictionary to store the match details
    # after crawling ends, the dictionary is used to output CSV file
    # This method checks whether Match is having an outcome or is it abandoned or draw
    def parse(self, resp, match_number, match_details_dict, match_players_dict):
        inning = {}
        batsmen = []
        bowlers = []
        # tsvDict = {}
        # checking if match is  abandoned
        if len(resp.css(".Collapsible").css(".Collapsible__contentInner").css(
                "table[class='w-100 table batsman']")) > 0:

            # Match seems abandoned, iterate over .batsman .small , to get player list
            for aBatsman in resp.css(".Collapsible").css(".Collapsible__contentInner"). \
                    css("table[class='w-100 table batsman']"). \
                    css("tbody").css("tr").css("td").css("a"):
                batsmen.append({
                    "name": aBatsman.css('::text').extract_first().strip().replace(" ", ""),
                    "profile_url": aBatsman.css('::attr(href)').extract_first(),
                    "runs": "",
                    "balls_played": "",
                    "strike_rate": ""
                })

            match_detail = self.parseMatchDetails(resp.css("table[class='w-100 table match-details-table']"))

            tsvDict = {
                "player_of_the_match": "",
                "player_of_the_match_image_url": "",
                "player_of_the_match_country": "",
                "batsmen_runs": "",
                "batsmen_ball_played": "",
                "batsmen_strike_rate": "",
                "bowlers_wickets": "",
                "bowlers_econ_rate": "",
                "toss_won_by": match_detail["Toss"].split(",")[0].strip(),
                "umpires": match_detail["Umpires"] + ",TV:" + match_detail["TV Umpire"] + ",Reserve:" + match_detail[
                    "Reserve Umpire"],
                "match_referee": match_detail["Match Referee"]
            }

        else:
            # valid non-abandoned match, follow normal processing

            best_player_details = self.parsePlayerOfMatchSection(resp.css("div[class='best-player']"))

            for anInning in resp.css(".Collapsible"):
                inningCountry = anInning.css("h5").css(".header-title.label::text").extract_first().split("INNINGS")[
                    0].strip()
                inning[inningCountry] = self.parseInning(anInning.css(".Collapsible__contentInner"))

                batsmen.extend(inning[inningCountry]["batsmen"])
                bowlers.extend(inning[inningCountry]["bowlers"])

            batsmen_run_csv = ",".join([batter["runs"] for batter in batsmen])
            batsmen_balls_csv = ",".join([batter["balls_played"] for batter in batsmen])
            batsmen_strike_rate_csv = ",".join([batter["strike_rate"] for batter in batsmen])

            bowlers_wickets_csv = ",".join([bowler["wickets"] for bowler in bowlers])
            bowlers_econ_rate_csv = ",".join([bowler["econ"] for bowler in bowlers])

            scrapedScorecard = {"innings": inning,
                                "match_detail": self.parseMatchDetails(
                                    resp.css("table[class='w-100 table match-details-table']"))}

            tsvDict = {
                "player_of_the_match": best_player_details["player_of_the_match"],
                "player_of_the_match_image_url": best_player_details["player_of_the_match_image_url"],
                "player_of_the_match_country": best_player_details["player_of_the_match_country"],
                "batsmen_runs": batsmen_run_csv,
                "batsmen_ball_played": batsmen_balls_csv,
                "batsmen_strike_rate": batsmen_strike_rate_csv,
                "bowlers_wickets": bowlers_wickets_csv,
                "bowlers_econ_rate": bowlers_econ_rate_csv,
                "toss_won_by": scrapedScorecard["match_detail"]["Toss"].split(",")[0].strip(),
                "umpires": scrapedScorecard["match_detail"]["Umpires"] + ",TV:" + scrapedScorecard["match_detail"][
                    "TV Umpire"] + ",Reserve:" + scrapedScorecard["match_detail"]["Reserve Umpire"],
                "match_referee": scrapedScorecard["match_detail"]["Match Referee"]
            }

        match_details_dict[match_number] = tsvDict

        players = batsmen
        players.extend(bowlers)
        # Invoke processing for part 3 for every player
        for aPlayer in players:
            # Duplication check :: SCRAPY checks that automatically,
            #           will fetch only if the new request is not already fetched
            yield scrapy.Request(resp.urljoin(aPlayer["profile_url"]), callback=self.playerProfileParser.parse,
                                 cb_kwargs={'match_players_dict': match_players_dict})


import os
import util
import config

scanned_files = 0

for directory, _, files_list in os.walk(config.td):
    #directory looks like "c:\user\box\...\phonexx\images"

    for ea_filename in files_list:
        #ea_filename looks like "ADX-....jpg"

        file_path = (directory+"\\"+ea_filename)
        #file_path looks like "c:\td\...\[filename].jpg"

        relative_dir = directory.replace(config.td, "")
        # looks like "\phonexx\images"

        file_attributes = ea_filename.split("_")

        scanned_files += 1
        if ( scanned_files % 1000 ) == 0 : print("Scanned ",scanned_files," files.")

        if file_attributes[0] == 'PXL':
            dest_dir = config.dd+"\\"+relative_dir
            util.move_this_file(file_path,dest_dir,ea_filename)
            config.moved_files += 1
            if (config.moved_files % 1000) == 0 : print("Moved",config.moved_files,"files.")
        else: continue

print("Moved",config.moved_files,"Files.")


import pandas as pd
import sklearn
import matplotlib.pyplot as plt
import seaborn
import load_data

train = load_data.get_data()[0]
testX = load_data.get_data()[1]
inv_labels = load_data.get_inverted_labels()

trainX = train.drop(['SalePrice'], axis = 1)
trainY = train['SalePrice']

def info_discrete(column):
    scatter = seaborn.stripplot(x = column, y = train['SalePrice'], data = train)
    if column in inv_labels.keys():
        scatter.set_xticklabels(inv_labels[column].values())
    else:
        pass
    plt.show()

def unsignificant_deletion():
    correlation = train.corrwith(train.SalePrice)
    to_delete = list()
    for col_name, corr_value in correlation.items():
        if corr_value < 0.35 and corr_value > -0.35:
            to_delete.append(col_name)

    return to_delete

trainX.drop(unsignificant_deletion(), axis = 1, inplace = True)
testX.drop(unsignificant_deletion(), axis = 1, inplace = True)

trainX['Alley'].fillna(2, inplace = True)
trainX['GarageType'].fillna(6, inplace = True)
trainX['GarageYrBlt'].fillna(0, inplace = True)
trainX['GarageFinish'].fillna(3, inplace = True)
trainX['BsmtQual'].fillna(4, inplace = True)
trainX['MasVnrArea'].fillna(0, inplace = True)
trainX['LotFrontage'].fillna(0, inplace = True)
trainX['PoolQC'].fillna(3, inplace = True)

# testX['Alley'].fillna(2, inplace = True)
# testX['GarageType'].fillna(6, inplace = True)
# testX['GarageYrBlt'].fillna(0, inplace = True)
# testX['GarageFinish'].fillna(3, inplace = True)
# testX['BsmtQual'].fillna(4, inplace = True)
# testX['MasVnrArea'].fillna(0, inplace = True)
# testX['LotFrontage'].fillna(0, inplace = True)
# testX['PoolQC'].fillna(3, inplace = True)

# print(trainX['Alley'].unique())
print(testX.info())
print(testX['KitchenQual'].unique())


#Imprting libraries
from dash import Dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
from random import randint
import plotly.express as px

# Creating dash environment
app = Dash(__name__)

# Constructing the layout
app.layout = html.Div([
    # Title
    html.Div(dcc.Markdown("Random number plotter"), style={"textAlign":"center", 'font-size':'300%'}),
    # Num. of point selection area, our N
    html.Div(["Number of points: ", dcc.Input(id='number_of_pts', value=100, type='number', min=1, style={'height': '25px', 'width':'100px'})], style={'padding-left':'80px', 'padding-top':'30px', 'display': 'inline-block', 'font-size':'150%'}),
    # Max range selection area, our K
    html.Div(["Max range: ", dcc.Input(id='upper_bound', value=100, type='number', min=1, style={'height': '25px', 'width': '100px'})], style={'padding-left': '50px', 'display': 'inline-block', 'font-size':'150%'}),
    # Our scatter plot
    dcc.Graph(id='random_graph', style={'height':'800px'}),
    # Title for the selected data area
    html.Div([dcc.Markdown("Selected points: ", style={'padding-left':'80px', 'font-size':'200%'})]),
    # Selected data area
    html.Div(html.Pre(id='selected_data', style={'border': 'thin lightgrey solid', 'overflowY': 'scroll', 'font-size':'200%'}), style={'width':'90%', 'padding-left':'80px'})])

# Callback function for number of points and range selection
@app.callback(
    Output("random_graph", "figure"),
    [Input("number_of_pts", "value"),
     Input("upper_bound", "value")])
def update_graph(number_of_pts, max_range):
    if(number_of_pts and max_range != None):  # Check whether arguments are null
        A_array = []
        for i in range(number_of_pts):
            A_array.append(randint(1, max_range))
        fig = px.scatter(y=A_array, labels={"x":"index", "y":"value"})
        fig.update_layout(showlegend=False)
        return fig       # return updated scatter plot
    return px.scatter()  # Return empty scatter plot

# Callback function for graph selection
@app.callback(
    Output("selected_data", "children"),
    [Input("random_graph", "selectedData")])
def update_selecteData(data):
    try:
        data = data["points"]
        print(data)
        points = []
        for point in data:
            points.append("Index: {}, Value: {} \n".format(point["x"], point["y"]))  # Make a string list of selected data, from Indexes and Values
        return points
    except:
        return ""

# Run Dash app
app.run_server()