from bs4 import BeautifulSoup
import urllib.request
import requests

class BRScraper:

    def __init__(self, server_url="http://www.baseball-reference.com/"):
        self.server_url = server_url

    def parse_tables(self, resource, table_ids=None, verbose=False, get_pos=False,std_only=False):
        """
        Given a resource on the baseball-reference server (should consist of
        the url after the hostname and slash), returns a dictionary keyed on
        table id containing arrays of data dictionaries keyed on the header
        columns. table_ids is a string or array of strings that can optionally
        be used to filter out which stats tables to return.
        """

        def is_parseable_table(tag):
            if not tag.has_attr("class"): return False
            return tag.name == "table" and "stats_table" in tag["class"] and "sortable" in tag["class"]

        def is_parseable_row(tag):
            if not tag.name == "tr": return False
            if not tag.has_attr("class"): return True  # permissive
            return "league_average_table" not in tag["class"] and "stat_total" not in tag["class"] and "minors_table" not in tag["class"]

        if isinstance(table_ids, str): table_ids = [table_ids]

        page = requests.get(self.server_url + resource)
#        print(page.headers)
#        print(page.encoding)
#        print(page.text)
        soup = BeautifulSoup(page.text,"html.parser")

        # Filter out batting tables for pitchers and pitching tables for hitters - AE

        def get_role(soup):
            if soup.find(itemprop="role"):
                role = soup.find(itemprop="role").get_text()
#            print("Role found: ",role)
            return role
        #       print(table_ids)

        if get_pos:
            role = ''
            if "players" in resource:
                role = get_role(soup)
            if "Pitcher" in role:
                if std_only: table_ids = ["pitching_standard"]
            elif role:
                if std_only: table_ids = ["batting_standard"]

        tables = soup.find_all(is_parseable_table)
        data = {}

#        print("Parsing with table_ids set to ",table_ids)

        # Read through each table, read headers as dictionary keys
        for table in tables:
#            print("Table found with id ",table["id"])

            if table_ids != None and table["id"] not in table_ids: continue
            if verbose: print("Processing table " + table["id"])
            data[table["id"]] = []

            headers = table.find("thead").find_all("th")
            header_names = []
            for header in headers:
                if header.string == None:
                    base_header_name = u""
                else: base_header_name = header.string.strip()
                if base_header_name in header_names:
                    i = 1
                    header_name = base_header_name + "_" + str(i)
                    while header_name in header_names:
                        i += 1
                        header_name = base_header_name + "_" + str(i)
                    if verbose:
                        if base_header_name == "":
                            print("Empty header relabeled as %s" % header_name)
                        else:
                            print("Header %s relabeled as %s" % (base_header_name, header_name))
                else:
                    header_name = base_header_name
                header_names.append(header_name)

            rows = table.find("tbody").find_all(is_parseable_row)
            for row in rows:
                entries = row.find_all("td")
                entry_data = []
                for entry in entries:
#                   if entry == entries[0]:
#                        print(type(entry))
#                        print(entry)
                    if entry.string == None:
                        if len(entry.contents) > 1:
                            entry_data.append(entry.contents[0])
                            continue
                        entry_data.append(u"")
                    else:
                        entry_data.append(entry.string.strip())
                if len(entry_data) > 0:
                    data[table["id"]].append(dict(zip(header_names, entry_data)))

        if get_pos: return data,role
        else: return data

scraper = BRScraper()
#data = scraper.parse_tables("players/r/rodrial01.shtml",['batting_standard'],True)
#print(data)


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
#import plotly.express as px
#import plotly.graph_objects as go
#from plotly.subplots import make_subplots
import seaborn as sns

computer = input("personal or work computer - ")

Household_removal = [1045]
Household_Removal_PUMP_1N = [1045]
Household_Removal_PUMP_1H = [2016]
Household_Removal_PUMP_2N = [1045]
Household_Removal_PUMP_2H = [1045]
Household_Removal_PUMP_3N = [1045]
Household_Removal_PUMP_3H = [2016]
Household_Removal_PUMP_4N = [1045]

if computer == "work":
    USB = "D"
else:
    USB = "E"

path_1N = USB+":/PUMP FILES/Pump_and_HAP_1N.csv"
path_1H = USB+":/PUMP FILES/Pump_and_HAP_1H.csv"
path_2N = USB+":/PUMP FILES/Pump_and_HAP_2N.csv"
path_2H = USB+":/PUMP FILES/Pump_and_HAP_2H.csv"
path_3N = USB+":/PUMP FILES/Pump_and_HAP_3N.csv"
path_3H = USB+":/PUMP FILES/Pump_and_HAP_3H.csv"
path_4N = USB+":/PUMP FILES/Pump_and_HAP_4N.csv"

Frame_1N = pd.read_csv(path_1N, delimiter=',')
Frame_1H = pd.read_csv(path_1H, delimiter=',')
Frame_2N = pd.read_csv(path_2N, delimiter=',')
Frame_2H = pd.read_csv(path_2H, delimiter=',')
Frame_3N = pd.read_csv(path_3N, delimiter=',')
Frame_3H = pd.read_csv(path_3H, delimiter=',')
Frame_4N = pd.read_csv(path_4N, delimiter=',')

HH_1N = []
Pump_1N = []
Hapex_1N = []
Cook_1N = []
for val, hh in enumerate(Frame_1N.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_1N_remove in Household_Removal_PUMP_1N:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_1N_remove):
                print('----- Removed from 1N--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_1N_remove) :
                    HH_1N.append((Frame_1N.iloc[val,0]))
                    Pump_1N.append((Frame_1N.iloc[val,1]))
                    Hapex_1N.append((Frame_1N.iloc[val,2]))
                    Cook_1N.append((Frame_1N.iloc[val,3]))
                    count = 1
print(HH_1N)
HH_1H = []
Pump_1H = []
Hapex_1H = []
Cook_1H = []
for val, hh in enumerate(Frame_1H.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_1H_remove in Household_Removal_PUMP_1H:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_1H_remove):
                print('----- Removed from 1H--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_1H_remove) :
                    HH_1H.append((Frame_1H.iloc[val,0]))
                    Pump_1H.append((Frame_1H.iloc[val,1]))
                    Hapex_1H.append((Frame_1H.iloc[val,2]))
                    Cook_1H.append((Frame_1H.iloc[val,3]))
                    count = 1
HH_2N = []
Pump_2N = []
Hapex_2N = []
Cook_2N = []
for val, hh in enumerate(Frame_2N.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_2N_remove in Household_Removal_PUMP_2N:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_2N_remove):
                print('----- Removed from 2N--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_2N_remove) :
                    HH_2N.append((Frame_2N.iloc[val,0]))
                    Pump_2N.append((Frame_2N.iloc[val,1]))
                    Hapex_2N.append((Frame_2N.iloc[val,2]))
                    Cook_2N.append((Frame_2N.iloc[val,3]))
                    count = 1
print('2n array', HH_2N)
HH_2H = []
Pump_2H = []
Hapex_2H = []
Cook_2H = []
for val, hh in enumerate(Frame_2H.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_2H_remove in Household_Removal_PUMP_2H:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_2H_remove):
                print('----- Removed from 2H--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_2H_remove) :
                    HH_2H.append((Frame_2H.iloc[val,0]))
                    Pump_2H.append((Frame_2H.iloc[val,1]))
                    Hapex_2H.append((Frame_2H.iloc[val,2]))
                    Cook_2H.append((Frame_2H.iloc[val,3]))
                    count = 1
HH_3N = []
Pump_3N = []
Hapex_3N = []
Cook_3N = []
for val, hh in enumerate(Frame_3N.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_3N_remove in Household_Removal_PUMP_3N:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_3N_remove):
                print('----- Removed from 3N--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_3N_remove) :
                    HH_3N.append((Frame_3N.iloc[val,0]))
                    Pump_3N.append((Frame_3N.iloc[val,1]))
                    Hapex_3N.append((Frame_3N.iloc[val,2]))
                    Cook_3N.append((Frame_3N.iloc[val,3]))
                    count = 1
HH_3H = []
Pump_3H = []
Hapex_3H = []
Cook_3H = []
for val, hh in enumerate(Frame_3H.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_3H_remove in Household_Removal_PUMP_3H:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_3H_remove):
                print('----- Removed from 3H--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_3H_remove) :
                    HH_3H.append((Frame_3H.iloc[val,0]))
                    Pump_3H.append((Frame_3H.iloc[val,1]))
                    Hapex_3H.append((Frame_3H.iloc[val,2]))
                    Cook_3H.append((Frame_3H.iloc[val,3]))
                    count = 1
HH_4N = []
Pump_4N = []
Hapex_4N = []
Cook_4N = []
for val, hh in enumerate(Frame_4N.iloc[:,0]):
    count = 0
    for HH_total_remove in Household_removal:
        if (int(hh) == HH_total_remove):
            break
        for HH_4N_remove in Household_Removal_PUMP_4N:
            if (int(hh) == HH_total_remove) or (int(hh) == HH_4N_remove):
                print('----- Removed from 4N--------------------', hh)
                break
            if count != 1:
                if (int(hh) != HH_total_remove) or (int(hh) != HH_4N_remove) :
                    HH_4N.append((Frame_4N.iloc[val,0]))
                    Pump_4N.append((Frame_4N.iloc[val,1]))
                    Hapex_4N.append((Frame_4N.iloc[val,2]))
                    Cook_4N.append((Frame_4N.iloc[val,3]))
                    count = 1
#df = px.data.tips()

#fig = px.box(df, x="day", y="total_bill", color="smoker")
#fig.update_traces(quartilemethod="exclusive") # or "inclusive", or "linear" by default
#fig.show()

fig, ax = plt.subplots()
plt.title('No-Hood Pump Box Plots')

#1N
quant_1_1N = np.percentile(Pump_1N, [25,50,75])
Top_lim_1_1N = quant_1_1N[2] + 1.5*(quant_1_1N[2] - quant_1_1N[0])
Low_lim_1_1N = quant_1_1N[0] - 1.5*(quant_1_1N[2] - quant_1_1N[0])

bp_1 = plt.boxplot(Pump_1N, positions = [1], widths = 0.6)
pump_1N_outlier = []
for v,a in enumerate(Pump_1N):
    if a > Top_lim_1_1N or a < Low_lim_1_1N:
        pump_1N_outlier.append(HH_1N[v])
        plt.text(1,a,HH_1N[v])
plt.text(1,0.1,'1N',color='b')

#2N
quant_1_2N = np.percentile(Pump_2N, [25,50,75])
Top_lim_1_2N = quant_1_2N[2] + 1.5*(quant_1_2N[2] - quant_1_2N[0])
Low_lim_1_2N = quant_1_2N[0] - 1.5*(quant_1_2N[2] - quant_1_2N[0])

bp_1 = plt.boxplot(Pump_2N,positions = [2], widths = 0.6)
pump_2N_outlier = []
for v,a in enumerate(Pump_1N):
    if a > Top_lim_1_2N or a < Low_lim_1_2N:
        pump_2N_outlier.append(HH_2N[v])
        plt.text(2,a,HH_2N[v])
plt.text(2,0.1,'2N', color= 'g')
#3N
quant_1_3N = np.percentile(Pump_3N, [25,50,75])
Top_lim_1_3N = quant_1_3N[2] + 1.5*(quant_1_3N[2] - quant_1_3N[0])
Low_lim_1_3N = quant_1_3N[0] - 1.5*(quant_1_3N[2] - quant_1_3N[0])

bp_1 = plt.boxplot(Pump_2N,positions = [3], widths = 0.6)
count = 0
pump_3N_outlier = []
for v,a in enumerate(Pump_2N):
    if a > Top_lim_1_3N or a < Low_lim_1_3N:
        pump_3N_outlier.append(HH_3N[v])
        plt.text(3,a,HH_3N[v])

plt.text(3,0.1,'3N', color='r')

#4N
quant_1_4N = np.percentile(Pump_4N, [25,50,75])
Top_lim_1_4N = quant_1_4N[2] + 1.5*(quant_1_4N[2] - quant_1_4N[0])
Low_lim_1_4N = quant_1_4N[0] - 1.5*(quant_1_4N[2] - quant_1_4N[0])

bp_1 = plt.boxplot(Pump_3N,positions = [4], widths = 0.6)
pump_4N_outlier = []
for v,a in enumerate(Pump_3N):
    if a > Top_lim_1_4N or a < Low_lim_1_4N:
        pump_4N_outlier.append(HH_4N[v])
        plt.text(4,a,HH_4N[v])
plt.text(4,0.1,'4N', color='y')

plt.xlim(0,5)
plt.ylim(0,1000)
print('1N had these values as outliers   ', pump_1N_outlier)
print('2N had these values as outliers   ', pump_2N_outlier)
print('3N had these values as outliers   ', pump_3N_outlier)
print('4N had these values as outliers   ', pump_4N_outlier)
plt.show()

#no hood compare
Pump_2N_1N = []
hh_2N_1N = []
Pump_3N_1N = []
hh_3N_1N = []
Pump_4N_1N = []
hh_4N_1N = []
Pump_3N_2N = []
hh_3N_2N = []
Pump_4N_3N = []
hh_4N_3N = []
Pump_4N_2N = []
hh_4N_2N = []
#2N to 1N
for v, h in enumerate(HH_1N):
    for vv, hh in enumerate(HH_2N):
        if hh == h:
            hh_2N_1N.append(hh)
            divisi = Pump_2N[vv]/ Pump_1N[v]
            Pump_2N_1N.append(divisi)

fig_2, ax2 = plt.subplots()
plt.title('% No_hood Change from Pump PM 2.5' )

quant_1_2N_1N = np.percentile(Pump_2N_1N, [25,50,75])
Top_lim_1_2N_1N = quant_1_2N_1N[2] + 1.5*(quant_1_2N_1N[2]-quant_1_2N_1N[0])
Low_lim_1_2N_1N = quant_1_2N_1N[0] - 1.5*(quant_1_2N_1N[2]-quant_1_2N_1N[0])
bp_1_1 = plt.boxplot(Pump_2N_1N, positions=[1], widths= 0.6)
pump_2N_1N_outlier = []
for v,a in enumerate(Pump_2N_1N):
    if a > Top_lim_1_2N_1N or a < Low_lim_1_2N_1N:
        pump_2N_1N_outlier.append(hh_2N_1N[v])
        plt.text(1, a, hh_2N_1N[v])
plt.text(0.5, -0.5, '2N / 1N', color= 'g')

#3N to 1N
for v, h in enumerate(HH_1N):
    for vv, hh in enumerate(HH_3N):
        if hh == h:
            hh_3N_1N.append(hh)
            divisi = Pump_3N[vv]/ Pump_1N[v]
            Pump_3N_1N.append(divisi)

quant_1_3N_1N = np.percentile(Pump_3N_1N, [25,50,75])
Top_lim_1_3N_1N = quant_1_3N_1N[2] + 1.5*(quant_1_3N_1N[2]-quant_1_3N_1N[0])
Low_lim_1_3N_1N = quant_1_3N_1N[0] - 1.5*(quant_1_3N_1N[2]-quant_1_3N_1N[0])

bp_1_1 = plt.boxplot(Pump_3N_1N, positions=[2], widths= 0.6)
pump_3N_1N_outlier = []
for v,a in enumerate(Pump_3N_1N):
    if a > Top_lim_1_3N_1N or a < Low_lim_1_3N_1N:
        pump_3N_1N_outlier.append(hh_3N_1N[v])
        plt.text(2, a, hh_3N_1N[v])
plt.text(1.5, -0.5, '3N / 1N', color= 'r')

#4N to 1N
for v, h in enumerate(HH_1N):
    for vv, hh in enumerate(HH_4N):
        if hh == h:
            hh_4N_1N.append(hh)
            divisi = Pump_4N[vv]/ Pump_1N[v]
            Pump_4N_1N.append(divisi)

quant_1_4N_1N = np.percentile(Pump_4N_1N, [25,50,75])
Top_lim_1_4N_1N = quant_1_4N_1N[2] + 1.5*(quant_1_4N_1N[2]-quant_1_4N_1N[0])
Low_lim_1_4N_1N = quant_1_4N_1N[0] - 1.5*(quant_1_4N_1N[2]-quant_1_4N_1N[0])
bp_1_1 = plt.boxplot(Pump_4N_1N, positions=[3], widths= 0.6)
pump_4N_1N_outlier = []
for v,a in enumerate(Pump_4N_1N):
    if a > Top_lim_1_4N_1N or a < Low_lim_1_4N_1N:
        pump_4N_1N_outlier.append(hh_4N_1N[v])
        plt.text(3, a, hh_4N_1N[v])
plt.text(2.5, -0.5, '4N / 1N', color= 'y')

#3N to 2N
for v, h in enumerate(HH_2N):
    for vv, hh in enumerate(HH_3N):
        if hh == h:
            hh_3N_2N.append(hh)
            divisi = Pump_3N[vv]/ Pump_2N[v]
            Pump_3N_2N.append(divisi)

quant_1_3N_2N = np.percentile(Pump_3N_2N, [25,50,75])
Top_lim_1_3N_2N = quant_1_3N_2N[2] + 1.5*(quant_1_3N_2N[2]-quant_1_3N_2N[0])
Low_lim_1_3N_2N = quant_1_3N_2N[0] - 1.5*(quant_1_3N_2N[2]-quant_1_3N_2N[0])
bp_1_1 = plt.boxplot(Pump_3N_2N, positions=[4], widths= 0.6)
pump_3N_2N_outlier = []
for v,a in enumerate(Pump_3N_2N):
    if a > Top_lim_1_3N_2N or a < Low_lim_1_3N_2N:
        pump_3N_2N_outlier.append(hh_3N_2N[v])
        plt.text(4, a, hh_3N_2N[v])
plt.text(3.5, -0.5, '3N / 2N', color= 'm')

#4N to 3N
for v, h in enumerate(HH_4N):
    for vv, hh in enumerate(HH_3N):
        if hh == h:
            hh_4N_3N.append(hh)
            divisi = Pump_4N[v]/ Pump_3N[vv]
            Pump_4N_3N.append(divisi)

quant_1_4N_3N = np.percentile(Pump_4N_3N, [25,50,75])
Top_lim_1_4N_3N = quant_1_4N_3N[2] + 1.5*(quant_1_4N_3N[2]-quant_1_4N_3N[0])
Low_lim_1_4N_3N = quant_1_4N_3N[0] - 1.5*(quant_1_4N_3N[2]-quant_1_4N_3N[0])
bp_1_1 = plt.boxplot(Pump_4N_3N, positions=[5], widths= 0.6)
pump_4N_3N_outlier = []
for v,a in enumerate(Pump_4N_3N):
    if a > Top_lim_1_4N_3N or a < Low_lim_1_4N_3N:
        pump_4N_3N_outlier.append(hh_4N_3N[v])
        plt.text(5, a, hh_4N_3N[v])
plt.text(4.5, -0.5, '4N / 3N', color= 'k')

#4N to 2N
for v, h in enumerate(HH_4N):
    for vv, hh in enumerate(HH_2N):
        if hh == h:
            hh_4N_2N.append(hh)
            divisi = Pump_4N[v]/ Pump_2N[vv]
            Pump_4N_2N.append(divisi)

quant_1_4N_2N = np.percentile(Pump_4N_2N, [25,50,75])
Top_lim_1_4N_2N = quant_1_4N_2N[2] + 1.5*(quant_1_4N_2N[2]-quant_1_4N_2N[0])
Low_lim_1_4N_2N = quant_1_4N_2N[0] - 1.5*(quant_1_4N_2N[2]-quant_1_4N_2N[0])
bp_1_1 = plt.boxplot(Pump_4N_2N, positions=[6], widths= 0.6)
pump_4N_2N_outlier = []
for v,a in enumerate(Pump_4N_2N):
    if a > Top_lim_1_4N_2N or a < Low_lim_1_4N_2N:
       pump_4N_2N_outlier.append(hh_4N_2N[v])
       plt.text(6, a, hh_4N_2N[v])
plt.text(5.5, -0.5, '4N / 2N', color= 'tab:orange')

plt.xlim(0,7)
plt.ylim(-0.5,8)
print('2N/1N had these values as outliers   ', pump_2N_1N_outlier,)
print('3N/1N had these values as outliers   ', pump_3N_1N_outlier)
print('4N/1N had these values as outliers   ', pump_4N_1N_outlier)
print('3N/2N had these values as outliers   ', pump_3N_2N_outlier)
print('4N/3N had these values as outliers   ', pump_4N_3N_outlier)
print('4N/2N had these values as outliers   ', pump_4N_2N_outlier)
plt.show()

#no hood tables
quant_1_1N = np.append(quant_1_1N, np.average(Pump_1N))
quant_1_2N = np.append(quant_1_2N, np.average(Pump_2N))
quant_1_3N = np.append(quant_1_3N, np.average(Pump_3N))
quant_1_4N = np.append(quant_1_4N, np.average(Pump_4N))

D_50_quant_phase_phase = {'Percentile %': ['25','50','75', 'Avg'], '1N': quant_1_1N, '2N': quant_1_2N,'3N' : quant_1_3N,'4N': quant_1_4N}
Pump_50_phase_no_hood = pd.DataFrame(data=D_50_quant_phase_phase, columns=['Percentile %','1N', '2N', '3N','4N'])

quant_1_2N_1N = np.append(quant_1_2N_1N , np.average(Pump_2N_1N))
quant_1_3N_1N = np.append(quant_1_3N_1N , np.average(Pump_3N_1N))
quant_1_4N_1N = np.append(quant_1_4N_1N , np.average(Pump_4N_1N))
quant_1_3N_2N = np.append(quant_1_3N_2N , np.average(Pump_3N_2N))
quant_1_4N_3N = np.append(quant_1_4N_3N , np.average(Pump_4N_3N))
quant_1_4N_2N = np.append(quant_1_4N_2N , np.average(Pump_4N_2N))

D_50_quant_percent ={'Percentile %': ['25','50','75', 'Avg'],'2N / 1N': quant_1_2N_1N,'3N / 1N': quant_1_3N_1N,'4N / 1N': quant_1_4N_1N,
                               '3N / 2N': quant_1_3N_2N,'4N / 3N': quant_1_4N_3N,'4N / 2N': quant_1_4N_2N}
pump_50_percent_change_no_hood = pd.DataFrame(data=D_50_quant_percent, columns=['Percentile %','2N / 1N','3N / 1N', '4N / 1N'
                                                                                ,'3N / 2N','4N / 3N','4N / 2N'])
print('Numbers of Values (1N)', len(Pump_1N))
print('Numbers of Values (2N)', len(Pump_2N))
print('Numbers of Values (3N)', len(Pump_3N))
print('Numbers of Values (4N)', len(Pump_4N))
print(Pump_50_phase_no_hood)
print('Numbers of Values (2N/1N)', len(Pump_2N_1N))
print('Numbers of Values (3N/1N)', len(Pump_3N_1N))
print('Numbers of Values (4N/1N)', len(Pump_4N_1N))
print('Numbers of Values (3N/2N)', len(Pump_3N_2N))
print('Numbers of Values (4N/3N)', len(Pump_4N_3N))
print('Numbers of Values (4N/2N)', len(Pump_4N_2N))
print(pump_50_percent_change_no_hood)

# hood portion

fig, ax = plt.subplots()
plt.title('Hood Pump Box Plots')

quant_1_1H = np.percentile(Pump_1H, [25,50,75])
Top_lim_1_1H = quant_1_1H[2] + 1.5*(quant_1_1H[2] - quant_1_1H[0])
Low_lim_1_1H = quant_1_1H[0] - 1.5*(quant_1_1H[2] - quant_1_1H[0])

bp_1 = plt.boxplot(Pump_1H, positions = [1], widths = 0.6)
pump_1H_outlier = []
for v,a in enumerate(Pump_1H):
    if a > Top_lim_1_1H or a < Low_lim_1_1H:
        pump_1H_outlier.append(HH_1H[v])
        plt.text(1,HH_1H[v])
plt.text(1,0,'1H',color='b')

quant_1_2H = np.percentile(Pump_2H, [25,50,75])
Top_lim_1_2H = quant_1_2H[2] + 1.5*(quant_1_2H[2] - quant_1_2H[0])
Low_lim_1_2H = quant_1_2H[0] - 1.5*(quant_1_2H[2] - quant_1_2H[0])

bp_1 = plt.boxplot(Pump_2H,positions = [2], widths = 0.6)
count = 0
pump_2H_outlier = []
for v,a in enumerate(Pump_2H):
    if a > Top_lim_1_2H or a < Low_lim_1_2H:
        pump_2H_outlier.append(HH_2H[v])
        plt.text(2,a,HH_2H[v])
plt.text(2,0,'2H', color= 'g')

quant_1_3H = np.percentile(Pump_3H, [25,50,75])
Top_lim_1_3H = quant_1_3H[2] + 1.5*(quant_1_3H[2] - quant_1_3H[0])
Low_lim_1_3H = quant_1_3H[0] - 1.5*(quant_1_3H[2] - quant_1_3H[0])

bp_1 = plt.boxplot(Pump_3H,positions = [3], widths = 0.6)
pump_3H_outlier = []
for v,a in enumerate(Pump_3H):
    if a > Top_lim_1_3H or a < Low_lim_1_3H:
        pump_3H_outlier.append(HH_3H[v])
        count = count + 1
        plt.text(3,a,HH_a_3H[v])
    plt.text(3,0,'3H', color='r')

plt.xlim(-0,4)
plt.ylim(0,1000)
print('1H had these values as outliers   ', pump_1H_outlier)
print('2H had these values as outliers   ', pump_2H_outlier)
print('3H had these values as outliers   ', pump_3H_outlier)
plt.show()

Pump_2H_1H= []
hh_2H_1H = []
Pump_3H_1H = []
hh_3H_1H = []
Pump_3H_2H = []
hh_3H_2H = []

    #% change of fuel perday per adult between each phase
fig_2, ax2 = plt.subplots()
plt.title('% hood Change from Pump pm 2.5' )
#2H to 1H
for v, h in enumerate(HH_2H):
    for vv, hh in enumerate(HH_1H):
        if hh == h:
            hh_2H_1H.append(hh)
            divisi = Pump_2H[v]/ Pump_1H[vv]
            Pump_2H_1H.append(divisi)

quant_1_2H_1H = np.percentile(Pump_2H_1H, [25,50,75])
Top_lim_1_2H_1H = quant_1_2H_1H[2] + 1.5*(quant_1_2H_1H[2]-quant_1_2H_1H[0])
Low_lim_1_2H_1H = quant_1_2H_1H[0] - 1.5*(quant_1_2H_1H[2]-quant_1_2H_1H[0])
bp_1_1 = plt.boxplot(Pump_2H_1H, positions=[1], widths= 0.6)
pump_2H_1H_outlier = []
for v,a in enumerate(Pump_2H_1H):
    if a > Top_lim_1_2H_1H or a < Low_lim_1_2H_1H:
        pump_2H_1H_outlier.append(hh_2H_1H[v])
        plt.text(1, a, hh_2H_1H[v])
plt.text(0.75, -0.25, '2H / 1H', color= 'g')

#3H to 1H
for v, h in enumerate(HH_3H):
    for vv, hh in enumerate(HH_1H):
        if hh == h:
            hh_3H_1H.append(hh)
            divisi = Pump_3H[v]/ Pump_1H[vv]
            Pump_3H_1H.append(divisi)

quant_1_3H_1H = np.percentile(Pump_3H_1H, [25,50,75])
Top_lim_1_3H_1H = quant_1_3H_1H[2] + 1.5*(quant_1_3H_1H[2]-quant_1_3H_1H[0])
Low_lim_1_3H_1H = quant_1_3H_1H[0] - 1.5*(quant_1_3H_1H[2]-quant_1_3H_1H[0])
bp_1_1 = plt.boxplot(Pump_3H_1H, positions=[2], widths= 0.6)
pump_3H_1H_outlier = []
count = 0
for v,a in enumerate(Pump_3H_1H):
    if a > Top_lim_1_3H_1H or a < Low_lim_1_3H_1H:
        pump_3H_1H_outlier.append(hh_3H_1H[v])
        count = count + 1
        if count == 1:
            plt.text(2, a, hh_3H_1H[v], ha='left',va='bottom')
        elif count != 1:
            plt.text(2, a, hh_3H_1H[v], ha='right',va='bottom')
plt.text(1.75, -0.25, '3H / 1H', color= 'r')

#3H to 2H
for v, h in enumerate(HH_3H):
    for vv, hh in enumerate(HH_2H):
        if hh == h:
            hh_3H_2H.append(hh)
            divisi = Pump_3H[v]/ Pump_2H[vv]
            Pump_3H_2H.append(divisi)

quant_1_3H_2H = np.percentile(Pump_3H_2H, [25,50,75])
Top_lim_1_3H_2H = quant_1_3H_2H[2] + 1.5*(quant_1_3H_2H[2]-quant_1_3H_2H[0])
Low_lim_1_3H_2H = quant_1_3H_2H[0] - 1.5*(quant_1_3H_2H[2]-quant_1_3H_2H[0])
bp_1_1 = plt.boxplot(Pump_3H_2H, positions=[3], widths= 0.6)
pump_3H_2H_outlier = []
for v,a in enumerate(Pump_3H_2H):
    if a > Top_lim_1_3H_2H or a < Low_lim_1_3H_2H:
        pump_3H_2H_outlier.append(hh_3H_2H[v])
        plt.text(3, a, hh_3H_2H[v])
plt.text(2.75, -0.25, '2H / 1H', color= 'm')

plt.xlim(-0,4)
plt.ylim(-0.25,2)
print('2H/1H had these values as outliers   ', pump_2H_1H_outlier)
print('3H/1H had these values as outliers   ', pump_3H_1H_outlier)
print('3H/2H had these values as outliers   ', pump_3H_2H_outlier)
plt.show()

quant_1_1H = np.append(quant_1_1H, np.average(Pump_1H))
quant_1_2H = np.append(quant_1_2H, np.average(Pump_2H))
quant_1_3H = np.append(quant_1_3H, np.average(Pump_3H))

D_50_quant_phase_hood = {'Percentile %': ['25','50','75', 'Avg'], '1H': quant_1_1H, '2H': quant_1_2H,'3H' : quant_1_3H}
Pump_50_phase_hood = pd.DataFrame(data=D_50_quant_phase_hood, columns=['Percentile %','1H', '2H','3H'] )

quant_1_2H_1H = np.append(quant_1_2H_1H , np.average(Pump_2H_1H))
quant_1_3H_1H = np.append(quant_1_3H_1H , np.average(Pump_3H_1H))
quant_1_3H_2H = np.append(quant_1_3H_2H , np.average(Pump_3H_2H))

D_50_quant_percent_pump_hood ={'Percentile %': ['25','50','75', 'Avg'],'2H / 1H': quant_1_2H_1H,'3H / 1H': quant_1_3H_1H,'3H / 2H': quant_1_3H_2H}
Pump_50_percent_change_hood = pd.DataFrame(data=D_50_quant_percent_pump_hood, columns=['Percentile %','2H / 1H','3H / 1H','3H / 2H'])
print('Numbers of Values (1H)', len(Pump_1H))
print('Numbers of Values (2H)', len(Pump_2H))
print('Numbers of Values (3H)', len(Pump_3H))
print(Pump_50_phase_hood)
print('Numbers of Values (2H/1H)', len(Pump_2H_1H))
print('Numbers of Values (3H/1H)', len(Pump_3H_1H))
print('Numbers of Values (3H/2H)', len(Pump_3H_2H))
print(Pump_50_percent_change_hood)

# plotly graph of hapex and pump

Hapex_1H_filter_1 = []
Names_1H_filter_1 = []
Pump_1H_filter_1 = []
for c, a in enumerate(Hapex_1H):
    if (a <= 0 ):
        Hapex_1H_filter_1.append(Hapex_1H[c])
        Names_1H_filter_1.append(HH_1H[c])
        Pump_1H_filter_1.append(Pump_1H[c])

Names_1H = ['2007', '2010', '2011', '2013', '2015']
fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_1H, y=Pump_1H_filter_1),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_1H, y=Hapex_1H_filter_1)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='1H Pump and Hapex')
#fig.show()

Names_2H = ['2001', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011', '2013', '2015']
fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_2H, y=Pump_2H),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_2H, y=Hapex_2H)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='2H Pump and Hapex')
#fig.show()

Names_3H = ['2001', '2003','2006', '2007', '2009', '2010', '2011', '2013', '2014', '2015']
fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_3H, y=Pump_3H),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_3H, y=Hapex_3H)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='3H Pump and Hapex')
#fig.show()

# bar graph plotting no hood

Names_1N = ['1001', '1005', '1007', '1008', '1009', '1010', '1011', '1012', '1013', '1014', '1015', '1017', '1018', '1022', '1023', '1027', '1028', '1029', '1030', '1033', '1035', '1036', '1037', '1039', '1040']
fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_1N, y=Pump_1N),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_1N, y=Hapex_1N)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='1N Pump and Hapex')
#fig.show()

Names_2N = ['1001', '1002', '1003', '1005', '1008', '1009', '1011', '1012', '1013', '1014', '1015', '1017', '1019', '1020', '1021', '1023', '1024', '1026', '1027', '1028', '1029', '1030', '1031', '1032', '1033', '1035', '1036', '1037', '1038', '1039', '1040']
fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_2N, y=Pump_2N),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_2N, y=Hapex_2N)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='2N Pump and Hapex')
#fig.show()

Names_3N = ['1001', '1002', '1003', '1004', '1005', '1006', '1007', '1008', '1009', '1010', '1011', '1012','1013', '1014', '1015', '1017', '1018', '1019', '1020', '1021', '1022', '1023', '1024', '1025', '1026', '1027', '1028', '1030', '1031', '1032', '1035', '1036', '1037', '1038', '1039', '1040']

fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_3N, y=Pump_3N),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_3N, y=Hapex_3N)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='3N Pump and Hapex')
#fig.show()

Names_4N = ['1001', '1002', '1004', '1005', '1006', '1007', '1008', '1009', '1011', '1012', '1013', '1014', '1015', '1017', '1018', '1019', '1020', '1021', '1022', '1023', '1024', '1025', '1026', '1027', '1028', '1030', '1031', '1032', '1033', '1035', '1036', '1037', '1038', '1039', '1040']

fig = go.Figure(data=[
                go.Bar(name='Pump (ug/m^3)', x=Names_4N, y=Pump_4N),
                go.Bar(name='Hapex (24 hour PM Average/ min)', x=Names_4N, y=Hapex_4N)])

fig.update_layout(barmode='group')
fig.update_layout(title_text='4N Pump and Hapex')
#fig.show()


from ReportGenerator.Geom import Geom
from ReportGenerator.Top_ReportGenerator import Top_ReportGenerator
from ReportComponents.WebFile import WebFile
from ReportGenerator.Charges import Charges
from ReportComponents.Plot import Plot

import logging
log = logging.getLogger(__name__)

class DRC(Top_ReportGenerator):

    def __init__(self,we,parsed):
        super().__init__(we,parsed)

    def __init__(self,we,parsed):
        super().__init__(we,parsed)

    def prepare_for_report(self):
        self.n_steps = len(self.parsed.data) - 1
        self.prepare_drc_plot()
        self.charges = Charges(self.we, self.parsed)
        self.plot_drc = True

    def prepare_drc_plot(self):

        self.drc_t = [o.last_value('P_drc_time') for o in self.parsed.data[1:]]
        self.drc_e = [o.last_value('P_drc_poten') for o in self.parsed.data[1:]]

        if len(self.drc_t)!=len(self.drc_e):
            self.plot_drc = False
            log.error('Different lengths of drc_x and drc_e!')

    def opt_conv_plot_html(self):
        plt = Plot(fname='-drc.png', xlab='time', ylab='Energy', legend='DRC Plot', x=self.drc_t, y=self.drc_e)
        if plt.nonempty:
            plt.save_plot()
            return self.img_tag(plt.web_path)
        else:
            return 'Not enough data to produce convergence plot'

    def save_geom(self):
        v = [str(Geom(self.we,g)) for g in self.parsed.data]
        self.multiple_geoms = (len(v) > 1)
        s_geoms = "\n".join(v)
        if not s_geoms:
            return ''
        return WebFile(fname='-drc.xyz',content=s_geoms).write()

    def load_in_jmol(self, webpath):
        label = 'DRC, step @{_modelNumber}'
        cmd = [
            self.we.jmol_load_file(webpath),
            'frame last',
            self.we.jmol_text(label)
            ]
        return "; ".join(cmd)

    def report_text(self):
        pass

    def report_html(self):
        webpath = self.save_geom()
        if webpath:
            load_command = self.load_in_jmol(webpath)
            out_html = [
                self.we.jmol_command_to_html(load_command), # load immediately
                'DRC: ',
                self.we.html_button(load_command, 'DRC'), # and create button to load (useful for several step-jobs)
                self.br_tag
            ]
            [self.add_left(s) for s in out_html]

            if self.multiple_geoms:
                self.add_left(self.we.html_geom_play_controls())
                self.add_left(self.br_tag)
        else:
            self.add_left(self.color_tag('No coordinates found!','err'))

        self.add_right(self.strong_tag("DRC"))
        self.add_right(self.br_tag)

        if self.plot_drc:
            self.add_right(self.opt_conv_plot_html())
            self.add_right(self.br_tag)

        if webpath and self.charges is not None:
            self.add_both(self.charges.button_bar(load_command))

        return self.get_cells()


#!/usr/bin/env python

"""Creates JADE configuration for stage 2 of pydss_simulation pipeline."""

import os
import sys

from jade.utils.subprocess_manager import run_command
from jade.utils.utils import load_data

def main():
    pydss_folder = os.path.dirname(os.path.realpath(__file__))
    status = load_data(os.environ["JADE_PIPELINE_STATUS_FILE"])
    cur_stage = status["stages"][-1]
    cur_stage_output = cur_stage["output_directory"]
    previous_stage = status["stages"][-2]
    previous_stage_output = previous_stage["output_directory"]
    script = os.path.join(pydss_folder, "merge_feeders_results.py")

    commands_file = "merge_feeders_commands.txt"
    with open(commands_file, "w") as f_out:
        cmd = f"python {script} run {previous_stage_output} {cur_stage_output}"
        f_out.write(cmd + "\n")

    cmd = f"jade config create {commands_file} -c config-stage2.json"
    try:
        ret = run_command(cmd)
    finally:
        os.remove(commands_file)
    sys.exit(ret)

if __name__ == "__main__":
    main()


# Функция для изменения названия чата
from datetime import datetime, timedelta

from telebot.util import user_link

from Week_oddity import week_oddity
from tgbot import chat_id, bot

def change_chat_title():
    chat_title = bot.get_chat(chat_id).title
    if chat_title.startswith("(НН)") or chat_title.startswith("(ЧН)"):
        new_title = chat_title[5:]
        title = f"(НН) {new_title}" if week_oddity() else f"(ЧН) {new_title}"
    else:
        title = f"(НН) {chat_title}" if week_oddity() else f"(ЧН) {chat_title}"
    bot.set_chat_title(chat_id, title)

def update_command_counter(user, command, instance):
    if user.id not in instance.command_counter:
        instance.command_counter[user.id] = {'Username:': user_link(user), 'Commands:': {}}
    if command not in instance.command_counter[user.id]['Commands:']:
        instance.command_counter[user.id]['Commands:'][command] = 0
    instance.command_counter[user.id]['Commands:'][command] += 1

def can_user_use_command(user_id, instance):
    now = datetime.now()
    if user_id in instance.last_command_time:
        time_since_last_command = now - instance.last_command_time[user_id]
        if time_since_last_command < timedelta(minutes=5):
            return False
    instance.last_command_time[user_id] = now
    return True


import pytest
from unittest import mock
from etl.etl_logs import etl_server_logs
from pyspark.sql.readwriter import DataFrameWriter
from pyspark import SparkContext

class test_etl_logs():
    def test_commons():
        assert callable(getattr(etl_server_logs,'read_data'))
        assert callable(getattr(etl_server_logs,'transform'))
        assert callable(getattr(etl_server_logs,'export_data'))

    @mock.path.object(SparkContext, 'textFile')
    def test_read_data(text_file_patch):
        test_input_path = 'abc'

        etl_server_logs.read_data(test_input_path)

        assert text_file_patch.called

    def test_row_can_be_structured():
        test_row = 'in24.inetnebr.com - - [01/Aug/1995:00:00:01 -0400] "GET /shuttle/missions/sts-68/news/sts-68-mcc-05.txt HTTP/1.0" 200 1839'
        expected_value = True

        assert etl_server_logs.row_can_be_structured(test_row) == expected_value

        test_row = 'abc dfnf 33232'
        expected_value = False

        assert etl_server_logs.row_can_be_structured(test_row) == expected_value

    @mock.patch.object(etl_server_logs, 'row_can_be_structured')
    def test_structure_log(row_can_be_structured_patch):
        test_row = 'in24.inetnebr.com - - [01/Aug/1995:00:00:01 -0400] "GET /shuttle/missions/sts-68/news/sts-68-mcc-05.txt HTTP/1.0" 200 1839'

        expected_value = Row(
            host='in24.inetnebr.com',
            timestamp='01/Aug/1995:00:00:01 -0400',
            request='GET /shuttle/missions/sts-68/news/sts-68-mcc-05.txt HTTP/1.0',
            status_code='200',
            total_bytes='1839'
        )

        assert etl_server_logs.structure_log(test_row) == expected_value

    def test_convert_dataframe():
        test_data = Row(
            host='abc',
            request='abc',
            status_code='200',
            timestamp='01/Aug/1995:00:00:01 -0400',
            total_bytes='1000'
        )

        test_df = spark.createDataFrame(test_data)

        expected_data = Row(
            host='abc',
            request='abc',
            status_code=200,
            timestamp='01/Aug/1995:00:00:01 -0400',
            total_bytes=1000
        )

        expected_df = spark.createDataFrame(expected_data)
        expected_df = expected_df.withColumn('timestamp', F.to_timestamp(df.timestamp, 'dd/MMM/yyyy:HH:mm:ss Z'))

        test_df = etl_server_logs.convert_dataframe(test_df)

        assert expected_df.collect() == test_df.collect()

    @mock.pathc.object(DataFrameWriter, 'parquet')
    def test_export_data(parquet_patch):
        test_data = Row(col1='abc')
        test_df = spark.createDataFrame(test_data)

        test_output_path = 'abc'

        etl_server_logs.export_data(test_df, test_output_path)

        assert parquet_patch.called

    def test_transform():
        test_data = ['in24.inetnebr.com - - [01/Aug/1995:00:00:01 -0400] "GET /shuttle/missions/sts-68/news/sts-68-mcc-05.txt HTTP/1.0" 200 1839']
        test_rdd = sc.parallelize(test_data)

        expected_data = Row(
            host='abc',
            request='abc',
            status_code=200,
            timestamp='01/Aug/1995:00:00:01 -0400',
            total_bytes=1000
        )

        expected_df = spark.createDataFrame(expected_data)
        expected_df = expected_df.withColumn('timestamp', F.to_timestamp(df.timestamp, 'dd/MMM/yyyy:HH:mm:ss Z'))

        assert transform(test_rdd).collect() == expected_df.collect()

    @mock.patch.object(etl_server_logs, 'read_data')
    @mock.patch.object(etl_server_logs, 'export_data')
    def test_run(read_data_patch, export_data_patch):
        mock_data = ['in24.inetnebr.com - - [01/Aug/1995:00:00:01 -0400] "GET /shuttle/missions/sts-68/news/sts-68-mcc-05.txt HTTP/1.0" 200 1839']
        mock_rdd = sc.parallelize(mock_data)

        read_data_patch.return_value = mock_rdd

        etl_server_logs.run()

        assert read_data_patch.called
        assert export_data_patch.called


import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np

class agent():
	def __init__(self, lr, s_size,a_size,h_size):
		#These lines established the feed-forward part of the network. The agent takes a state and produces an action.
		self.state_in= tf.placeholder(shape=[None,s_size],dtype=tf.float32)
		hidden = slim.fully_connected(self.state_in,h_size,biases_initializer=None,activation_fn=tf.nn.relu)
		self.output = slim.fully_connected(hidden,a_size,activation_fn=tf.nn.softmax,biases_initializer=None)
		self.chosen_action = tf.argmax(self.output,1)

		#The next six lines establish the training proceedure. We feed the reward and chosen action into the network
		#to compute the loss, and use it to update the network.
		self.reward_holder = tf.placeholder(shape=[None],dtype=tf.float32)
		self.action_holder = tf.placeholder(shape=[None],dtype=tf.int32)

		self.indexes = tf.range(0, tf.shape(self.output)[0]) * tf.shape(self.output)[1] + self.action_holder
		self.responsible_outputs = tf.gather(tf.reshape(self.output, [-1]), self.indexes)
		#self.responsible_outputs_array = tf.slice(self.output, [0, self.action_holder], [tf.shape(self.output)[0], 1])
		#self.responsible_outputs = tf.reshape(self.responsible_outputs_array, [-1])

		self.loss = -tf.reduce_mean(tf.log(self.responsible_outputs)*self.reward_holder)

		tvars = tf.trainable_variables()
		self.gradient_holders = []
		for idx,var in enumerate(tvars):
			placeholder = tf.placeholder(tf.float32,name=str(idx)+'_holder')
			self.gradient_holders.append(placeholder)

		self.gradients = tf.gradients(self.loss,tvars)

		optimizer = tf.train.AdamOptimizer(learning_rate=lr)
		self.update_batch = optimizer.apply_gradients(zip(self.gradient_holders,tvars))


from tkinter import *
from child_window import ChildWindow

class Window:
    def __init__(self, width, height, title="MyWindow", resizable=(False, False), icon=r"resources/feather.ico"):
        self.root = Tk()
        self.root.title(title)
        self.root.geometry(f"{width}x{height}+200+200")
        # self.root.resizable(resizable[0], resizable[1])
        if icon:
            self.root.iconbitmap(icon)

        # self.label = Label(self.root, text="I'm a label", bg="#e84ae5", relief=RIDGE, wraplength=40, font="Consolas 15")

    def run(self):
        self.draw_widgets()
        self.root.mainloop()

    def draw_widgets(self):
        # top_frame = Frame(self.root, borderwidth=30, relief=SUNKEN, width=300, height=200)
        # bottom_frame = Frame(self.root, borderwidth=30, relief=SUNKEN, width=300, height=200)
        # top_frame.pack()
        # bottom_frame.pack()

        l1 = Label(self.root, width=300, height=20, bg='red', text="First")
        l2 = Label(self.root, width=30, height=2, bg='orange', text="Second")

        l1.place(x=10, y=10)
        l2.place(in_=l1, x=10, y=10)

        # Label(self.root, width=30, height=2, bg='red', text="First").place(relx=0.1, rely=0.5, relwidth=0.1, relheight=0.2)
        # Label(self.root, width=30, height=2, bg='orange', text="Second").place(x=100, y=100, width=100, height=50)
        # Label(top_frame, width=30, height=2, bg='yellow', text="Third").place(x=10, y=10, bordermode=OUTSIDE)
        # Label(bottom_frame, width=30, height=2, bg='green', text="Fourth").place(x=10, y=10, bordermode=INSIDE)
        # Label(self.root, width=30, height=2, bg='cyan', text="Fifth")

    def create_child(self, width, height, title="Child", resizable=(False, False), icon=None):
        ChildWindow(self.root, width, height, title, resizable, icon)

if __name__ == "__main__":
    window = Window(500, 500, "TKINTER")
    # window.create_child(200, 100)
    window.run()


import time

import pytest
from pytest_bdd import scenarios, given, when, then, parsers
from selenium.webdriver.support import expected_conditions as ec

from Pages import Main, Search

scenarios('../features/searching_products.feature')

@pytest.fixture(scope='function')
def context():
    return {}

@given('the user go to main site')
def the_user_go_to_main_site(selenium_driver):
    site = Main.MainSite(selenium_driver)
    site.go_to(site.BASE_URL)

@when(parsers.parse('the user enter to search bar words "{product}" and see hint'))
def the_user_find_product(selenium_driver, context, product):
    driver = Main.MainSite(selenium_driver)
    driver.search_product_by_name(product)
    driver.hint_is_displayed()
    amount = driver.amount_of_find_products_is_displayed()
    print(amount)
    context['amount_from_hint'] = amount

@when('the user click on search button')
def click_on_button(selenium_driver):
    driver = Main.MainSite(selenium_driver)
    driver.click_on_search_button()

@then(parsers.parse('the user is on search site with url "{url}"'))
def check_confirm_url(url):
    assert ec.url_contains(url)

@then(parsers.parse('the text in header of search is "{word}"'))
def check_word(selenium_driver, word):
    driver = Search.SearchPage(selenium_driver)
    assert word in driver.search_word_is_displayed_in_search_header()

@then('the user see list of products after click on search button')
def check_display_list_of_products(selenium_driver, context):
    driver = Search.SearchPage(selenium_driver)
    amount2 = driver.list_of_products()
    context['amount_from_list'] = str(amount2)
    assert amount2 > 0

@then('amount displayed product is equal with amount in hint')
def compare_amount_of_products(context):
    assert context['amount_from_hint'] == context['amount_from_list']


from __future__ import print_function

import sys
sys.path.append('utils')
import argparse
import gzip
import os
import pickle
import urllib
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data
from torch.optim import lr_scheduler
from torchvision import datasets, transforms

from whitening import WTransform2d

usps_dataset_multiplier = 6

class USPS(data.Dataset):
    url = "https://raw.githubusercontent.com/mingyuliutw/CoGAN/master/cogan_pytorch/data/uspssample/usps_28x28.pkl"

    def __init__(self, root, train=True, transform=None, transform_aug=None, download=False):
        """Init the USPS data set"""
        self.root = os.path.expanduser(root)
        self.filename = "usps_28x28.pkl"
        self.train = train
        # Num of train = 7438, Num of test = 1860
        self.transform = transform
        self.transform_aug = transform_aug
        self.dataset_size = None

        if download:
            self.download()

        if not self._check_exists():
            raise RuntimeError("Dataset no found." +
                               " You can use download=True to download it.")
        self.train_data, self.train_labels = self.load_samples()

        if self.train:
            self.train_data = np.repeat(self.train_data, usps_dataset_multiplier, axis=0)
            self.train_labels = np.repeat(self.train_labels, usps_dataset_multiplier, axis=0)

            total_num_samples = self.train_labels.shape[0]
            indices = np.arange(total_num_samples)
            np.random.shuffle(indices)
            self.train_data = self.train_data[indices[0: usps_dataset_multiplier * self.dataset_size], ::]
            self.train_labels = self.train_labels[indices[0: usps_dataset_multiplier * self.dataset_size]]

        # self.train_data *= 255.0
        self.train_data = self.train_data.transpose((0, 2, 3, 1))  # NCHW

    def __getitem__(self, index):
        """ Get images and target labels for data loader

        Args:
            index (int): Index
        Returns:
            tuple (image, target): where target is the index of the class
        """

        img, label = self.train_data[index], self.train_labels[index]

        if self.transform_aug is not None:
            img_aug = self.transform_aug(img)

        if self.transform is not None:
            img = self.transform(img)

        label = torch.squeeze(torch.LongTensor([np.int64(label).item()]))

        if self.transform_aug is not None:
            return img, img_aug, label
        else:
            return img, label

    def __len__(self):
        """ Return the size of the dataset """
        if self.train:
            return usps_dataset_multiplier * self.dataset_size
        else:
            return self.dataset_size

    def _check_exists(self):
        return os.path.exists(os.path.join(self.root, self.filename))

    def download(self):
        filename = os.path.join(self.root, self.filename)
        dirname = os.path.dirname(filename)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if os.path.isfile(filename):
            return
        print("Download %s to %s" % (self.url, os.path.abspath(filename)))
        urllib.request.urlretrieve(self.url, filename)
        print("Done")
        return

    def load_samples(self):
        filename = os.path.join(self.root, self.filename)
        f = gzip.open(filename, "rb")
        data_set = pickle.load(f, encoding="bytes")
        f.close()

        if self.train:
            images = data_set[0][0]
            labels = data_set[0][1]
            self.dataset_size = labels.shape[0]
        else:
            images = data_set[1][0]
            labels = data_set[1][1]
            self.dataset_size = labels.shape[0]
        return images, labels

class MNIST(data.Dataset):
    """`MNIST <http://yann.lecun.com/exdb/mnist/>`_ Dataset.
    Args:
        root (string): Root directory of dataset where ``processed/training.pt``
            and  ``processed/test.pt`` exist.
        train (bool, optional): If True, creates dataset from ``training.pt``,
            otherwise from ``test.pt``.
        download (bool, optional): If true, downloads the dataset from the internet and
            puts it in root directory. If dataset is already downloaded, it is not
            downloaded again.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
    """

    processed_folder = 'processed'
    training_file = 'training.pt'
    test_file = 'test.pt'

    def __init__(self, root, train=True, transform=None, transform_aug=None):
        self.root = os.path.expanduser(root)
        self.transform = transform
        self.transform_aug = transform_aug
        self.train = train  # training set or test set

        if self.train:
            data_file = self.training_file
        else:
            data_file = self.test_file
        self.data, self.targets = torch.load(os.path.join(self.root, self.processed_folder, data_file))

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target_label, rot_label) where target_label is index of the target class
            and rot_label is the rotation index
        """
        img, target_label = self.data[index], self.targets[index]

        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img.numpy(), mode='L')

        if self.transform_aug is not None:
            img_aug = self.transform_aug(img)

        if self.transform is not None:
            img = self.transform(img)

        if self.transform_aug is not None:
            return img, img_aug, target_label
        else:
            return img, target_label

    def __len__(self):
        return len(self.data)

class EntropyLoss(nn.Module):
    ''' Module to compute entropy loss '''
    def __init__(self):
        super(EntropyLoss, self).__init__()

    def forward(self, x):
        p = F.softmax(x, dim=1)
        q = F.log_softmax(x, dim=1)
        b = p * q
        b = -1.0 * b.sum(-1).mean()
        #b = -1.0 * b.sum()
        return b

class LeNet(nn.Module):
    def __init__(self, group_size):
        super(LeNet, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, kernel_size=5, padding=2)
        self.ws1 = WTransform2d(num_features=32, group_size=group_size)
        self.wt1 = WTransform2d(num_features=32, group_size=group_size)
        self.gamma1 = nn.Parameter(torch.ones(32, 1, 1))
        self.beta1 = nn.Parameter(torch.zeros(32, 1, 1))
        #self.conv1_drop = nn.Dropout2d()

        self.conv2 = nn.Conv2d(32, 48, kernel_size=5, padding=2)
        self.ws2 = WTransform2d(num_features=48, group_size=group_size)
        self.wt2 = WTransform2d(num_features=48, group_size=group_size)
        self.gamma2 = nn.Parameter(torch.ones(48, 1, 1))
        self.beta2 = nn.Parameter(torch.zeros(48, 1, 1))
        #self.conv2_drop = nn.Dropout2d()

        self.fc3 = nn.Linear(2352, 100)
        self.bns3 = nn.BatchNorm1d(100, affine=False)
        self.bnt3 = nn.BatchNorm1d(100, affine=False)
        self.gamma3 = nn.Parameter(torch.ones(1, 100))
        self.beta3 = nn.Parameter(torch.zeros(1, 100))

        self.fc4 = nn.Linear(100, 100)
        self.bns4 = nn.BatchNorm1d(100, affine=False)
        self.bnt4 = nn.BatchNorm1d(100, affine=False)
        self.gamma4 = nn.Parameter(torch.ones(1, 100))
        self.beta4 = nn.Parameter(torch.zeros(1, 100))

        self.fc5 = nn.Linear(100, 10)
        self.bns5 = nn.BatchNorm1d(10, affine=False)
        self.bnt5 = nn.BatchNorm1d(10, affine=False)
        self.gamma5 = nn.Parameter(torch.ones(1, 10))
        self.beta5 = nn.Parameter(torch.zeros(1, 10))

    def forward(self, x):

        if self.training:
            x = self.conv1(x)
            x_source, x_target = torch.split(x, split_size_or_sections=x.shape[0] // 2, dim=0)
            #x = self.conv1_drop(F.max_pool2d(F.relu(torch.cat((self.ws1(x_source), self.wt1(x_target)), dim=0)*self.gamma1 + self.beta1), kernel_size=2, stride=2))
            x = F.max_pool2d(F.relu(torch.cat((self.ws1(x_source), self.wt1(x_target)), dim=0)*self.gamma1 + self.beta1), kernel_size=2, stride=2)

            x = self.conv2(x)
            x_source, x_target = torch.split(x, split_size_or_sections=x.shape[0] // 2, dim=0)
            #x = self.conv2_drop(F.max_pool2d(F.relu(torch.cat((self.ws2(x_source), self.wt2(x_target)), dim=0)*self.gamma2 + self.beta2), kernel_size=2, stride=2))
            x = F.max_pool2d(F.relu(torch.cat((self.ws2(x_source), self.wt2(x_target)), dim=0)*self.gamma2 + self.beta2), kernel_size=2, stride=2)

            x = x.view(x.shape[0], -1)
            x = self.fc3(x)
            x_source, x_target = torch.split(x, split_size_or_sections=x.shape[0] // 2, dim=0)
            #x = F.dropout(F.relu(torch.cat((self.bns3(x_source), self.bnt3(x_target)), dim=0)*self.gamma3 + self.beta3), training=self.training)
            x = F.relu(torch.cat((self.bns3(x_source), self.bnt3(x_target)), dim=0)*self.gamma3 + self.beta3)

            x = self.fc4(x)
            x_source, x_target = torch.split(x, split_size_or_sections=x.shape[0] // 2, dim=0)
            #x = F.dropout(F.relu(torch.cat((self.bns4(x_source), self.bnt4(x_target)), dim=0)*self.gamma4 + self.beta4), training=self.training)
            x = F.relu(torch.cat((self.bns4(x_source), self.bnt4(x_target)), dim=0)*self.gamma4 + self.beta4)

            x = self.fc5(x)
            x_source, x_target = torch.split(x, split_size_or_sections=x.shape[0] // 2, dim=0)
            x = torch.cat((self.bns5(x_source), self.bnt5(x_target)), dim=0)*self.gamma5 + self.beta5
        else:
            x = self.conv1(x)
            #x = self.conv1_drop(F.max_pool2d(F.relu(self.wt1(x)*self.gamma1 + self.beta1), kernel_size=2, stride=2))
            x = F.max_pool2d(F.relu(self.wt1(x)*self.gamma1 + self.beta1), kernel_size=2, stride=2)

            x = self.conv2(x)
            #x = self.conv2_drop(F.max_pool2d(F.relu(self.wt2(x)*self.gamma2 + self.beta2), kernel_size=2, stride=2))
            x = F.max_pool2d(F.relu(self.wt2(x)*self.gamma2 + self.beta2), kernel_size=2, stride=2)

            x = x.view(x.shape[0], -1)
            x = self.fc3(x)
            #x = F.dropout(F.relu(self.bnt3(x)*self.gamma3 + self.beta3), training=self.training)
            x = F.relu(self.bnt3(x)*self.gamma3 + self.beta3)

            x = self.fc4(x)
            #x = F.dropout(F.relu(self.bnt4(x)*self.gamma4 + self.beta4), training=self.training)
            x = F.relu(self.bnt4(x)*self.gamma4 + self.beta4)

            x = self.fc5(x)
            x = self.bnt5(x)*self.gamma5 + self.beta5
        return x

def train(args, model, device, source_train_loader, target_train_loader, optimizer, epoch, lambda_entropy_loss):
    model.train()
    for batch_idx, (source, target) in enumerate(zip(source_train_loader, target_train_loader)):
        source_data = source[0]
        source_y = source[1]
        target_data = target[0]

        data = torch.cat((source_data, target_data), dim=0)  # concat the source and target mini-batches
        data, source_y = data.to(device), source_y.to(device)

        optimizer.zero_grad()
        output = model(data)

        source_output, target_output = torch.split(output, split_size_or_sections=output.shape[0] // 2, dim=0)

        entropy_criterion = EntropyLoss()

        cls_loss = F.nll_loss(F.log_softmax(source_output), source_y)
        entropy_l = lambda_entropy_loss * entropy_criterion(target_output)

        loss = cls_loss + entropy_l
        loss.backward()
        optimizer.step()

        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tClassification Loss: {:.6f} \tEntropy Loss: {:.6f}'.format(
                epoch, batch_idx * len(target_data), len(source_train_loader.dataset),
                       100. * batch_idx / len(source_train_loader), cls_loss.item(), entropy_l.item()))

def test(args, model, device, target_test_loader):
    model.eval()
    test_cls_loss = 0.
    correct = 0
    with torch.no_grad():
        for data, target in target_test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_cls_loss += F.nll_loss(F.log_softmax(output, dim=1), target, size_average=False).item()
            pred = F.softmax(output, dim=1).max(1, keepdim=True)[1] # get the index of max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

        test_cls_loss /= len(target_test_loader.dataset)
        print('\nTest set: Classification loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
            test_cls_loss, correct, len(target_test_loader.dataset),
            100. * correct / len(target_test_loader.dataset)))

    return 100. * correct / len(target_test_loader.dataset)

def main():
    # Training settings
    parser = argparse.ArgumentParser(description='PyTorch DIAL example')
    parser.add_argument('--num_workers', default=2, type=int)
    parser.add_argument('--source_batch_size', type=int, default=32, help='input source batch size for training (default: 32)')
    parser.add_argument('--target_batch_size', type=int, default=32, help='input target batch size for training (default: 32)')
    parser.add_argument('--test_batch_size', type=int, default=100, help='input batch size for testing (default: 100)')
    parser.add_argument('--source', type=str, default='usps', help='source dataset name')
    parser.add_argument('--target', type=str, default='mnist', help='target dataset name')
    parser.add_argument('--epochs', type=int, default=120, help='number of epochs to train (default: 10)')
    parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.01)')
    parser.add_argument('--sgd_momentum', type=float, default=0.5, help='SGD momentum (default: 0.5)')
    parser.add_argument('--running_momentum', type=float, default=0.1, help='Running momentum for statistics(default: 0.1)')
    parser.add_argument('--lambda_entropy_loss', type=float, default=0.1, help='Value of lambda for the entropy loss (default: 0.1)')
    parser.add_argument('--log_interval', type=int, default=100, help='how many batches to wait before logging training status')
    parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)')
    parser.add_argument('--from_script', action='store_true', help="use this flag for bulk running from script")
    parser.add_argument('--run', default=0, type=int, help="use this flag for bulk running from script")
    parser.add_argument('--method', default='bn', help="use this flag for bulk running from script")
    parser.add_argument('--group_size', type=int, default=32, help='group size for the whitening matrix (default: 32)')
    args = parser.parse_args()
    assert args.source != args.target, "source and target datasets can not be the same"
    torch.manual_seed(args.seed)
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    """ MNIST train and test data loaders """
    train_loader_mnist = torch.utils.data.DataLoader(
        MNIST('../data/mnist', train=True,
              transform=transforms.Compose([
                  transforms.ToTensor(),
                  transforms.Normalize(mean=[0.1307], std=[0.3081])
              ])),
        batch_size=args.source_batch_size, shuffle=True, num_workers=args.num_workers, drop_last=True)

    test_loader_mnist = torch.utils.data.DataLoader(
        MNIST('../data/mnist', train=False,
              transform=transforms.Compose([
                  transforms.ToTensor(),
                  transforms.Normalize(mean=[0.1307], std=[0.3081])
              ])),
        batch_size=args.test_batch_size, shuffle=True, num_workers=args.num_workers)

    """ USPS train and test data loaders """
    train_loader_usps = torch.utils.data.DataLoader(
        USPS(root='../data/usps', train=True,
             transform=transforms.Compose([
                 transforms.ToTensor(),
                 transforms.Normalize(mean=[0.5], std=[0.5])
             ])
             ,download=True), batch_size=args.target_batch_size, shuffle=True, num_workers=args.num_workers, drop_last=True)

    test_loader_usps = torch.utils.data.DataLoader(
        USPS(root='../data/usps', train=False,
             transform=transforms.Compose([
                 transforms.ToTensor(),
                 transforms.Normalize(mean=[0.5], std=[0.5])
             ])
             ,download=False), batch_size=args.test_batch_size, shuffle=True, num_workers=args.num_workers)

    model = LeNet(group_size=args.group_size).to(device)
    optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=5e-4)
    exp_lr_scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[50, 80], gamma=0.1)

    if args.source == 'mnist' and args.target == 'usps':
        source_train_loader = train_loader_mnist
        target_train_loader = train_loader_usps
        test_loader = test_loader_usps
    elif args.source == 'usps' and args.target == 'mnist':
        source_train_loader = train_loader_usps
        target_train_loader = train_loader_mnist
        test_loader = test_loader_mnist

    for epoch in range(args.epochs):
        exp_lr_scheduler.step()
        train(args, model, device, source_train_loader, target_train_loader, optimizer, epoch, args.lambda_entropy_loss)
        test(args, model, device, test_loader)

if __name__ == '__main__':
    main()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.inputs, name='inputs'),
    path('disclaimer/', views.disclaimer, name='disclaimer'),
    path('platform/', views.platform, name='platform'),
    path('circuit_data/', views.index, name='index'),
    path('transformer_data/', views.transformer_data, name='transformer_data'),
    path('load_data/', views.load_data, name='load_data'),
    path('ph_data/', views.ph_fault_data, name='ph_fault_data'),
    path('earth_data/', views.earth_fault_data, name='earth_fault_data'),
    path('inputs/', views.inputs, name='inputs'),
]


from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.shortcuts import redirect, render
from django.urls import reverse_lazy
from django.views.generic import ListView, CreateView, DetailView, TemplateView, View
from django.core.paginator import Paginator
import folium

from trip.filters import EventFilter, RestaurantFilter, ThingToDoFilter
from trip.forms import EventForm, ReviewForm, ThingToDoForm
from trip.models import Event, Restaurant, Review, ThingToDo

class ReviewCreateView(LoginRequiredMixin, CreateView):
    template_name = 'reviews/create_review.html'
    model = Review
    form_class = ReviewForm

    def get_success_url(self):
        return self.request.META['HTTP_REFERER']

    def form_valid(self, form):
        if form.is_valid():
            review = form.save(commit=False)
            review.user = self.request.user
            review.save()
            review_model = self.request.POST.get('review_model')
            id_ = self.request.POST.get('id')
            if review_model == 'event':
                event = Event.objects.get(pk=id_)
                event.reviews.add(review)
                event.full_clean()
                event.save()
            elif review_model == 'restaurant':
                restaurant = Restaurant.objects.get(pk=id_)
                restaurant.reviews.add(review)
                restaurant.full_clean()
                restaurant.save()
            elif review_model == 'thing_to_do':
                thing_to_do = ThingToDo.objects.get(pk=id_)
                thing_to_do.reviews.add(review)
                thing_to_do.full_clean()
                thing_to_do.save()
            return redirect(self.get_success_url())

class EventCreateView(LoginRequiredMixin, CreateView):
    template_name = 'events/create_event.html'
    model = Event
    form_class = EventForm
    success_url = 'home'

    def form_valid(self, form):
        if form.is_valid():
            event = form.save(commit=False)
            event.creator = self.request.user
            event.save()

            return redirect(self.success_url)

class EventListView(ListView):
    template_name = 'events/list_of_events.html'
    model = Event
    context_object_name = 'all_events'
    paginate_by = 5

    def get_context_data(self, **kwargs):
        data = super(EventListView, self).get_context_data()

        filtered_events = EventFilter(self.request.GET, queryset=Event.objects.all())
        event_page_obj = get_page_obj(self, filtered_events.qs)

        data['filtered_events'] = filtered_events
        data['event_page_obj'] = event_page_obj

        return data

class EventDetailView(DetailView):
    template_name = 'events/event_details.html'
    model = Event

    def get_context_data(self, **kwargs):
        context = super().get_context_data()

        if self.object.lat and self.object.long:
            my_map = create_map(self)
            context['map'] = my_map

        context['form'] = ReviewForm()

        return context

class RestaurantCreateView(PermissionRequiredMixin, CreateView):
    template_name = 'restaurants/create_restaurant.html'
    model = Restaurant
    form_class = EventForm
    success_url = reverse_lazy('create_new_restaurant')

class RestaurantListView(ListView):
    template_name = 'restaurants/list_of_restaurants.html'
    paginate_by = 5
    model = Restaurant
    context_object_name = 'all_restaurants'

    def get_context_data(self, **kwargs):
        data = super(RestaurantListView, self).get_context_data()

        filtered_restaurants = RestaurantFilter(self.request.GET, queryset=Restaurant.objects.all())
        restaurant_page_obj = get_page_obj(self, filtered_restaurants.qs)

        data['filtered_restaurants'] = filtered_restaurants
        data['restaurant_page_obj'] = restaurant_page_obj

        return data

class RestaurantDetailView(DetailView):
    template_name = 'restaurants/restaurant_details.html'
    model = Restaurant

    def get_context_data(self, **kwargs):
        context = super().get_context_data()

        other_restaurants = self.get_other_restaurants()

        if self.object.lat and self.object.long:
            my_map = create_map(self)
            context['map'] = my_map

        context['type_restaurants'] = other_restaurants
        context['form'] = ReviewForm()

        return context

    def get_other_restaurants(self):
        all_restaurants = Restaurant.objects.all().order_by('-avg_rating')
        type_restaurants = filter(lambda r: r.type == self.object.type and r != self.object, all_restaurants)
        restaurants = list(type_restaurants)[:4]

        return restaurants

class ThingToDoCreateView(PermissionRequiredMixin, CreateView):
    template_name = 'things_to_do/create_ttd.html'
    model = ThingToDo
    form_class = ThingToDoForm
    success_url = reverse_lazy('create_new_ttd')

class ThingToDoListView(ListView):
    template_name = 'things_to_do/all_ttd.html'
    model = ThingToDo
    paginate_by = 5
    context_object_name = 'all_ttd'

    def get_context_data(self, **kwargs):
        data = super(ThingToDoListView, self).get_context_data()

        filtered_ttds = ThingToDoFilter(self.request.GET, queryset=ThingToDo.objects.all())
        ttd_page_obj = get_page_obj(self, filtered_ttds.qs)

        data['filtered_ttds'] = filtered_ttds
        data['ttd_page_obj'] = ttd_page_obj

        return data

class ThingToDoDetailView(DetailView):
    template_name = 'things_to_do/ttd_details.html'
    model = ThingToDo

    def get_context_data(self, **kwargs):
        context = super().get_context_data()

        other_ttds = self.get_other_ttds()

        if self.object.lat and self.object.long:
            my_map = create_map(self)
            context['map'] = my_map

        context['type_ttds'] = other_ttds
        context['form'] = ReviewForm()

        return context

    def get_other_ttds(self):
        all_ttds = ThingToDo.objects.all().order_by('-avg_rating')
        type_ttds = filter(lambda t: t.type == self.object.type and t != self.object, all_ttds)
        things_to_do = list(type_ttds)[:4]

        return things_to_do

class SearchListView(View):

    def post(self, request):
        searched = request.POST['searched']

        restaurants = Restaurant.objects.filter(name__contains=searched).order_by('-avg_rating')
        events = Event.objects.filter(name__contains=searched).order_by('-avg_rating')
        things_to_do = ThingToDo.objects.filter(name__contains=searched).order_by('-avg_rating')

        context = {
            'searched': searched,
            'restaurants': restaurants,
            'events': events,
            'things_to_do': things_to_do
        }

        return render(request, 'search.html', context)

class HomeTemplateView(TemplateView):
    template_name = 'home/home.html'

def get_page_obj(obj, filtered_qs):
    paginated_filtered_obj = Paginator(filtered_qs, obj.paginate_by)
    page_number = obj.request.GET.get('page')
    page_obj = paginated_filtered_obj.get_page(page_number)

    return page_obj

def create_map(obj):
    coordinates = [obj.object.lat, obj.object.long]
    my_map = folium.Map(coordinates, zoom_start=17)

    icon, color = '', ''
    if obj.model == Restaurant:
        icon, color = 'cutlery', 'red'
    elif obj.model == ThingToDo:
        icon, color = 'binoculars', 'green'
    elif obj.model == Event:
        icon, color = 'calendar', 'purple'

    folium.Marker(
        coordinates,
        tooltip=f'{obj.object.name}',
        icon=folium.Icon(color=color, prefix='fa', icon=icon)
    ).add_to(my_map)

    my_map = my_map._repr_html_()

    return my_map


from random import randint
import math

print(f'Jogo do Prédio')

distanciaPredio = randint(5, 100)
print(f'Você está a {distanciaPredio} metros do prédio')

andar = randint(1, 15)
print(f'Acerte a janela do andar {andar}º!')

angulo = float(input(f'Em qual ângulo que você vai atirar? '))
d = math.sqrt((andar * 3) ** 2 + distanciaPredio ** 2)

hMax = andar * 3
hMin = (andar - 1) * 3

x = d*math.cos(angulo)
y = d * math.cos(angulo)


def divide():
    try:
        num1 = int(input("Ingrese el primer número: "))
        num2 = int(input("Ingrese el segundo número: "))
        print("El cociente de los números es: " + str(num1 / num2))
    except ValueError:
        print("El valor ingresado es incorrecto...!!!")
    except ZeroDivisionError:
        print("No se puede dividir entre 0...!!!")
    finally:
        print("Calculo finalizado...!!!")

divide()


import numpy as np
from sklearn.cluster import KMeans
import xarray as xr
from scipy.spatial import distance_matrix
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import matplotlib.colors as colors
from matplotlib import cm

def KMA_simple(xds_PCA, num_clusters, repres=0.95):
    '''
    KMeans Classification for PCA data

    xds_PCA:
        (n_components, n_components) PCs
        (n_components, n_features) EOFs
        (n_components, ) variance
    num_clusters
    repres

    returns a xarray.Dataset containing KMA data
    '''

    # PCA data
    variance = xds_PCA.variance.values[:]
    EOFs = xds_PCA.EOFs.values[:]
    PCs = xds_PCA.PCs.values[:]

    var_anom_std = xds_PCA.pred_std.values[:]
    var_anom_mean = xds_PCA.pred_mean.values[:]
    time = xds_PCA.pred_time.values[:]

    # APEV: the cummulative proportion of explained variance by ith PC
    APEV = np.cumsum(variance) / np.sum(variance)*100.0
    nterm = np.where(APEV <= repres*100)[0][-1]

    PCsub = PCs[:, :nterm+1]
    data = PCsub
    data_std = np.std(data, axis=0)
    data_mean = np.mean(data, axis=0)

    # normalize but keep PCs weigth
    data_norm = np.ones(data.shape)*np.nan
    for i in range(PCsub.shape[1]):
        data_norm[:,i] = np.divide(data[:,i]-data_mean[i], data_std[0])

    EOFsub = EOFs[:nterm+1, :]

    # KMEANS
    kma = KMeans(n_clusters=num_clusters, n_init=2000).fit(PCsub)

    # groupsize
    _, group_size = np.unique(kma.labels_, return_counts=True)

    # groups
    d_groups = {}
    for k in range(num_clusters):
        d_groups['{0}'.format(k)] = np.where(kma.labels_==k)
    # TODO: STORE GROUPS WITHIN OUTPUT DATASET

    # centroids
    centroids = np.dot(kma.cluster_centers_, EOFsub)
    # centroids
    # centroids = np.zeros((num_clusters, data.shape[1]))
    # for k in range(num_clusters):
    #     centroids[k,:] = np.mean(data[d_groups['{0}'.format(k)],:], axis=1)
    # km, x and var_centers
    km = np.multiply(
        centroids,
        np.tile(var_anom_std[0], (num_clusters, 1))
    ) + np.tile(var_anom_mean, (num_clusters, 1))

    # sort kmeans
    kma_order = np.argsort(np.mean(-km, axis=1))

    # reorder clusters: bmus, km, cenEOFs, centroids, group_size
    sorted_bmus = np.zeros((len(kma.labels_),),)*np.nan
    for i in range(num_clusters):
        posc = np.where(kma.labels_ == kma_order[i])
        sorted_bmus[posc] = i
    sorted_km = km[kma_order]
    sorted_cenEOFs = kma.cluster_centers_[kma_order]
    sorted_centroids = centroids[kma_order]
    sorted_group_size = group_size[kma_order]

    return xr.Dataset(
        {
            'bmus': (('n_pcacomp'), sorted_bmus.astype(int)),
            'cenEOFs': (('n_clusters', 'n_features'), sorted_cenEOFs),
            'centroids': (('n_clusters','n_pcafeat'), sorted_centroids),
            'Km': (('n_clusters','n_pcafeat'), sorted_km),
            'group_size': (('n_clusters'), sorted_group_size),

            # PCA data
            'PCs': (('n_pcacomp','n_features'), PCsub),
            'variance': (('n_pcafeat',), variance),
            'time': (('n_pcacomp',), time),
        }
    )

def sort_cluster_gen_corr_end(centers, dimdim):
    '''
    SOMs alternative
    '''
    # TODO: DOCUMENTAR.

    # get dimx, dimy
    dimy = np.floor(np.sqrt(dimdim)).astype(int)
    dimx = np.ceil(np.sqrt(dimdim)).astype(int)

    if not np.equal(dimx*dimy, dimdim):
        # TODO: RAISE ERROR
        pass

    dd = distance_matrix(centers, centers)
    qx = 0
    sc = np.random.permutation(dimdim).reshape(dimy, dimx)

    # get qx
    for i in range(dimy):
        for j in range(dimx):

            # row F-1
            if not i==0:
                qx += dd[sc[i-1,j], sc[i,j]]

                if not j==0:
                    qx += dd[sc[i-1,j-1], sc[i,j]]

                if not j+1==dimx:
                    qx += dd[sc[i-1,j+1], sc[i,j]]

            # row F
            if not j==0:
                qx += dd[sc[i,j-1], sc[i,j]]

            if not j+1==dimx:
                qx += dd[sc[i,j+1], sc[i,j]]

            # row F+1
            if not i+1==dimy:
                qx += dd[sc[i+1,j], sc[i,j]]

                if not j==0:
                    qx += dd[sc[i+1,j-1], sc[i,j]]

                if not j+1==dimx:
                    qx += dd[sc[i+1,j+1], sc[i,j]]

    # test permutations
    q=np.inf
    go_out = False
    for i in range(dimdim):
        if go_out:
            break

        go_out = True

        for j in range(dimdim):
            for k in range(dimdim):
                if len(np.unique([i,j,k]))==3:

                    u = sc.flatten('F')
                    u[i] = sc.flatten('F')[j]
                    u[j] = sc.flatten('F')[k]
                    u[k] = sc.flatten('F')[i]
                    u = u.reshape(dimy, dimx, order='F')

                    f=0
                    for ix in range(dimy):
                        for jx in range(dimx):

                            # row F-1
                            if not ix==0:
                                f += dd[u[ix-1,jx], u[ix,jx]]

                                if not jx==0:
                                    f += dd[u[ix-1,jx-1], u[ix,jx]]

                                if not jx+1==dimx:
                                    f += dd[u[ix-1,jx+1], u[ix,jx]]

                            # row F
                            if not jx==0:
                                f += dd[u[ix,jx-1], u[ix,jx]]

                            if not jx+1==dimx:
                                f += dd[u[ix,jx+1], u[ix,jx]]

                            # row F+1
                            if not ix+1==dimy:
                                f += dd[u[ix+1,jx], u[ix,jx]]

                                if not jx==0:
                                    f += dd[u[ix+1,jx-1], u[ix,jx]]

                                if not jx+1==dimx:
                                    f += dd[u[ix+1,jx+1], u[ix,jx]]

                    if f<=q:
                        q = f
                        sc = u

                        if q<=qx:
                            qx=q
                            go_out=False

    return sc.flatten('F')

def Plot_DWTs_Mean_Anom(xds_KMA, xds_var, kind='mean', mask_land=None, p_export=None):
    '''
    Plot Daily Weather Types (bmus mean)
    kind - mean/anom
    '''

    bmus = xds_KMA['sorted_bmus'].values[:]
    n_clusters = len(xds_KMA.n_clusters.values[:])

    var_max = np.max(xds_var.values)
    var_min = np.min(xds_var.values)
    scale = 1/100.0  # scale from Pa to mbar

    # get number of rows and cols for gridplot
    n_rows, n_cols = GetBestRowsCols(n_clusters)

    # get cluster colors
    cs_dwt = colors_dwt(n_clusters)

    # plot figure
    #fig = plt.figure(figsize=(_faspect*_fsize, _fsize))
    fig = plt.figure(figsize=(10, 10))

    gs = gridspec.GridSpec(n_rows, n_cols, wspace=0.0, hspace=0.0)
    gr, gc = 0, 0

    for ic in range(n_clusters):

        if kind=='mean':
            # data mean
            it = np.where(bmus==ic)[0][:]
            c_mean = xds_var.isel(time=it).mean(dim='time')
            c_plot = np.multiply(c_mean, scale)  # apply scale

        elif kind=='anom':
            # data anomally
            it = np.where(bmus==ic)[0][:]
            t_mean = xds_var.mean(dim='time')
            c_mean = xds_var.isel(time=it).mean(dim='time')
            c_anom = c_mean - t_mean
            c_plot = np.multiply(c_anom, scale)  # apply scale

        # dwt color
        clr = cs_dwt[ic]

        # axes plot
        ax = plt.subplot(gs[gr, gc])
        pc = axplot_DWT(
            ax, np.flipud(c_plot),
            vmin = var_min, vmax = var_max,
            wt_num = ic+1,
            land = mask_land, wt_color = clr,
        )

        # anomalies colorbar center at 0
        if kind == 'anom':
            pc.set_clim(-6,6)

        # get lower positions
        if gr==n_rows-1 and gc==0:
            pax_l = ax.get_position()
        elif gr==n_rows-1 and gc==n_cols-1:
            pax_r = ax.get_position()

        # counter
        gc += 1
        if gc >= n_cols:
            gc = 0
            gr += 1

    # add a colorbar
    cbar_ax = fig.add_axes([pax_l.x0, pax_l.y0-0.05, pax_r.x1 - pax_l.x0, 0.02])
    cb = fig.colorbar(pc, cax=cbar_ax, orientation='horizontal')
    if kind=='mean':
        cb.set_label('Pressure (mbar)')
    elif kind=='anom':
        cb.set_label('Pressure anomalies (mbar)')

    # show / export
#    if not p_export
#     plt.show()
#    else:
#        fig.savefig(p_export, dpi=_fdpi)
#        plt.close()

def axplot_DWT(ax, dwt, vmin, vmax, wt_num, land=None, wt_color=None):
    'axes plot EOFs 2d map'

    cmap = copy.deepcopy(cm.get_cmap('RdBu_r'))

    # EOF pcolormesh
    pc = ax.pcolormesh(
        dwt,
        cmap = cmap, shading = 'gouraud',
        clim = (vmin, vmax),
    )

    # optional mask land
    if type(land).__module__ == np.__name__:
        landc = land.copy()
        landc[np.isnan(land)]=1
        landc[land==1]=np.nan
        ax.pcolormesh(
            np.flipud(landc),
            cmap=colors.ListedColormap([wt_color]), shading='gouraud',
        )

    # wt text
    ax.text(0.87, 0.85, wt_num, transform=ax.transAxes, fontweight='bold')

    # customize axis
    ax.set_xticks([])
    ax.set_yticks([])

    return pc

def GetBestRowsCols(n):
    'try to square number n, used at gridspec plots'
    from math import sqrt

    sqrt_n = sqrt(n)
    if sqrt_n.is_integer():
        n_r = int(sqrt_n)
        n_c = int(sqrt_n)
    else:
        l_div = GetDivisors(n)
        n_c = l_div[int(len(l_div)/2)]
        n_r = int(n/n_c)

    return n_r, n_c

def GetDivisors(x):
    l_div = []
    i = 1
    while i<x:
        if x%i == 0:
            l_div.append(i)
        i = i + 1
    return l_div

def colors_dwt(num_clusters):

    # 42 DWT colors
    l_colors_dwt = [
        (1.0000, 0.1344, 0.0021),
        (1.0000, 0.2669, 0.0022),
        (1.0000, 0.5317, 0.0024),
        (1.0000, 0.6641, 0.0025),
        (1.0000, 0.9287, 0.0028),
        (0.9430, 1.0000, 0.0029),
        (0.6785, 1.0000, 0.0031),
        (0.5463, 1.0000, 0.0032),
        (0.2821, 1.0000, 0.0035),
        (0.1500, 1.0000, 0.0036),
        (0.0038, 1.0000, 0.1217),
        (0.0039, 1.0000, 0.2539),
        (0.0039, 1.0000, 0.4901),
        (0.0039, 1.0000, 0.6082),
        (0.0039, 1.0000, 0.8444),
        (0.0039, 1.0000, 0.9625),
        (0.0039, 0.8052, 1.0000),
        (0.0039, 0.6872, 1.0000),
        (0.0040, 0.4510, 1.0000),
        (0.0040, 0.3329, 1.0000),
        (0.0040, 0.0967, 1.0000),
        (0.1474, 0.0040, 1.0000),
        (0.2655, 0.0040, 1.0000),
        (0.5017, 0.0040, 1.0000),
        (0.6198, 0.0040, 1.0000),
        (0.7965, 0.0040, 1.0000),
        (0.8848, 0.0040, 1.0000),
        (1.0000, 0.0040, 0.9424),
        (1.0000, 0.0040, 0.8541),
        (1.0000, 0.0040, 0.6774),
        (1.0000, 0.0040, 0.5890),
        (1.0000, 0.0040, 0.4124),
        (1.0000, 0.0040, 0.3240),
        (1.0000, 0.0040, 0.1473),
        (0.9190, 0.1564, 0.2476),
        (0.7529, 0.3782, 0.4051),
        (0.6699, 0.4477, 0.4584),
        (0.5200, 0.5200, 0.5200),
        (0.4595, 0.4595, 0.4595),
        (0.4100, 0.4100, 0.4100),
        (0.3706, 0.3706, 0.3706),
        (0.2000, 0.2000, 0.2000),
        (     0, 0, 0),
    ]

    # get first N colors
    np_colors_base = np.array(l_colors_dwt)
    np_colors_rgb = np_colors_base[:num_clusters]

    return np_colors_rgb


from future import standard_library
standard_library.install_aliases()
import sys
import threading
import time

from nose.tools import assert_true
from desktop.lib.thread_util import dump_traceback

if sys.version_info[0] > 2:
  from io import StringIO as string_io
else:
  from cStringIO import StringIO as string_io

def test_dump_traceback():
  started = threading.Event()
  stopped = threading.Event()

  class Thread(threading.Thread):
    def run(self):
      started.set()
      stopped.wait(10.0)
      assert_true(stopped.is_set())

  thread = Thread(name='thread_util_test thread')
  thread.start()
  thread_ident = str(thread.ident)

  header = 'Thread thread_util_test thread %s' % thread_ident

  try:
    started.wait(10.0)
    assert_true(started.is_set())

    out = string_io()
    dump_traceback(file=out)

    assert_true(header in out.getvalue())

    out = string_io()
    dump_traceback(file=out, all_threads=False)

    assert_true(header not in out.getvalue())
  finally:
    stopped.set()
    thread.join()