import numpy as np
import pandas as pd
import streamlit as st
import plotly.graph_objects as go
import plotly.express as px
import seaborn as sns
import os

def data_visualization(side_bar_input):
    data = side_bar_input["data"]

    if data is not None:
        # Web App Title
        st.subheader("I. Dataset Visualization")
        st.set_option('deprecation.showPyplotGlobalUse', False)

        # EDA:

        # Part 1:
        st.markdown(f"
Dataset Information
", unsafe_allow_html=True)

        # Show Dataset:
        if st.checkbox("Preview Data", value=True):
            st.text("Head of dataset: ")
            st.write(data.head())
            st.text("Tail of dataset:")
            st.write(data.tail())

        # Dimensions:
        data_dim = st.radio("What dimension do you want to show ?", ('Rows', 'Columns'))
        if data_dim == 'Rows':
            st.write("The Length of Rows is:", len(data))
        if data_dim == 'Columns':
            st.write("The Length of Columns is :", data.shape[1])
        if st.checkbox("Show All Data"):
            st.dataframe(data)

        st.markdown("
", unsafe_allow_html=True)

        st.markdown(f"
Unidimensional analysis of variables
",
                    unsafe_allow_html=True)

        # Type of variable:
        # numeric_data = data.select_dtypes(include=[np.number])
        # categorical_data = data.select_dtypes(exclude=[np.number])

        numeric_data = side_bar_input["quantitative_variables"]
        categorical_data = side_bar_input["qualitative_variables"]

        col_quan, col_qual = st.columns(2)

        with col_quan:
            st.markdown("- Quantitative Variables:")
            st.write("Count : ", len(numeric_data))
            if st.button("Show Quantitative Features", disabled=False):
                st.write(numeric_data)

        with col_qual:
            st.markdown("- Qualitative Variables:")
            st.write("Count :", len(categorical_data))
            if st.button("Show Qualitative Features", disabled=False):
                st.write(categorical_data)

        # percentage of pie chart:
        labels = ['Quantitative variables', 'Qualitative variables']
        if numeric_data and categorical_data is not None:
            values = [len(numeric_data), len(categorical_data)]
        else :
            values = [1,1]
        fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.3, pull=[0, 0, 0.2, 0])])
        # st.header('Percentage')
        st.plotly_chart(fig)

        # Summary
        if st.checkbox("Show Summary of Quantitative Variables"):
            if numeric_data and categorical_data is not None:
                st.write(data.describe())
                option = st.selectbox(
                "Which column do you want to visualise as a boxplot?",
                numeric_data)
                fig = px.box(data, y=option)
                st.plotly_chart(fig, use_container_width=True)
            else:
                st.markdown(f"
Please select the quantitative variables in the left sidebar
",
                            unsafe_allow_html=True)

        """
       if st.checkbox("Show Summary of Qualitative Variables"):
           option = st.selectbox(
               "Which column would you like to visualise?",
               categorical_data)

           fig = px.bar(categorical_data,x=0)
           st.plotly_chart(fig,use_container_width=True)

           st.markdown("
", unsafe_allow_html=True)
        """

        # Two-dimension
        st.markdown("
", unsafe_allow_html=True)
        st.markdown(f"
 Two-dimensional analysis of variables
",
                    unsafe_allow_html=True)
        col_x, col_y = st.columns(2)
        with col_x:
            option_x = st.multiselect(
                'Select X axis',
                data.columns)
        with col_y:
            option_y = st.selectbox(
                'Select Y axis',
                data.columns)

        # scatter plot:
        if st.checkbox("Show the sactter plot"):
            fig = px.scatter(data, x=option_x, y=option_y)
            st.plotly_chart(fig)

        st.markdown("
", unsafe_allow_html=True)
        st.markdown(f"
 Missing Data Detection
",
                    unsafe_allow_html=True)

        st.markdown(f"
 The number of missing values in each column is :
", unsafe_allow_html=True)
        series_missing = (data == '?').astype(int).sum()
        st.write(series_missing.sum())
        if st.checkbox("Show the heatmap"):
            st.write(sns.heatmap(data == '?', cbar=False), weight=150, height=200)
            st.pyplot()


import torch
from torch import nn

class ConvBlock(nn.Module):
    def __init__(self, in_channels: int, out_channels: int, kernel_size: int = 3, stride: int = 1, padding: int = 0, pool_size:int = 2, bias: bool = True):
        super().__init__()

        self.c1 = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias)
        self.act = nn.GELU()
        self.pool = nn.MaxPool2d(kernel_size=pool_size)

    def forward(self, x):
        x = self.c1(x)
        x = self.act(x)
        x = self.pool(x)

        return x


"""
Useful kernels:
---------------
https://naif.jpl.nasa.gov/pub/naif/generic_kernels/lsk/naif0012.tls
https://naif.jpl.nasa.gov/pub/naif/generic_kernels/pck/earth_200101_990628_predict.bpc
https://naif.jpl.nasa.gov/pub/naif/generic_kernels/pck/pck00010.tpc

https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp

#https://naif.jpl.nasa.gov/pub/naif/generic_kernels/pck/earth_latest_high_prec.bpc
#https://naif.jpl.nasa.gov/pub/naif/generic_kernels/fk/planets/earth_assoc_itrf93.tf
"""

import spiceypy as sp
import requests
from glob import glob

d2r = 3.141592653589793/180
r2d = 180/3.141592653589793

def download_file(url, path=''):
    filename = url.rsplit('/', 1)[-1]
    r = requests.get(url, allow_redirects=True)
    open(path+filename, 'wb').write(r.content)

def lonlat_to_cartesian(obs_loc):
    """
    obs_loc : (lon (deg), lat (deg), alt (m))
    """
    lon, lat, alt = obs_loc
    lon = lon * d2r
    lat = lat * d2r
    alt = alt / 1000
    radii = [6378.1366, 6378.1366, 6356.7519]
    re = radii[0]
    rp = radii[2]
    f = (re-rp)/re
    obspos = sp.pgrrec(body='earth', lon=lon, lat=lat, alt=alt, re=re, f=f)
    return obspos

def get_apparent(body, t, obs_loc, kernels, abcorr='LT+S'):

    if isinstance(body, int):
        body = str(body)

    for k in kernels:
        sp.furnsh(k)

    et = sp.str2et(str(t))

    state, lt  = sp.azlcpo(
        method='ELLIPSOID',
        target=body,
        et=et,
        abcorr=abcorr,
        azccw=False,
        elplsz=True,
        obspos=lonlat_to_cartesian(obs_loc),
        obsctr='earth',
        obsref='ITRF93')

    r, az, alt = state[:3]

    sp.kclear()

    return r, az*r2d, alt*r2d


import unittest
from dateparser import future_datetime as future, Especial
from datetime import datetime

class TestFutureDate(unittest.TestCase):

    def test_especial_later(self):
        base_date = datetime(2023,10,27,11,30,0)
        self.assertEqual(future(especial=Especial.LATER, base_date=base_date), datetime(2023,10,27,15,0,0),
            "It should be 4 hours later without minutes and seconds")

    def test_especial_weekend(self):
        # laborable day
        base_date = datetime(2023,10,25,11,30,0) # Wednesday
        self.assertEqual(future(base_date=base_date, especial=Especial.WEEKEND), datetime(2023,10,28,8,0), "Should be next Saturday")

        # Saturday
        base_date = datetime(2023,10,28,11,30,0) # Saturday
        self.assertEqual(future(base_date=base_date, especial=Especial.WEEKEND), datetime(2023,11,4,8,0), "Should be Saturday next week")

    def test_tonight(self):
        # morning
        base_date = datetime(2023,10,25,11,30,0) # morning
        self.assertEqual(future(base_date=base_date, especial=Especial.TONIGHT), datetime(2023,10,25,20,0), "Should be today at 20:00")

        # night
        base_date = datetime(2023,10,25,20,30,0) # night
        self.assertEqual(future(base_date=base_date, especial=Especial.TONIGHT), None, "Should be None because it is already tonight")

    def test_just_wednesday(self):
        WEDNESDAY = 2
        # it's Wednesday, so returns 7 days ahead
        base_date = datetime(2023,10,25)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,1,8,0), "Should be next Wednesday")

        # it's Friday, so returns 5 days ahead
        base_date = datetime(2023,11,3)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,8,8,0), "Should be next Wednesday")

        # it's Monday, so returns 2 days ahead
        base_date = datetime(2023,11,13)
        self.assertEqual(future(base_date=base_date, weekday=WEDNESDAY), datetime(2023,11,15,8,0), "Should be next Wednesday")

    def test_just_18th(self):
        # it's 18th, so returns 18th next month
        base_date = datetime(2023,10,18,9)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,11,18,8,0), "Should be 18th next month")

        # it's 19th, so returns 18th next month
        base_date = datetime(2023,10,19)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,11,18,8,0), "Should be 18th next month")

        # it's 17th, so returns 17th same month
        base_date = datetime(2023,10,17)
        self.assertEqual(future(base_date=base_date, day_number=18), datetime(2023,10,18,8,0), "Should be 18th same month")

    def test_edge_cases(self):
        # test 31st in June 5th, should return 31st July
        base_date = datetime(2023,6,5)
        self.assertEqual(future(base_date=base_date, day_number=31), datetime(2023,7,31,8,0), "Should be 31st July")

        # test 8am in January 31st 9 am, should return 8am on February 1st
        base_date = datetime(2023,1,31,9)
        self.assertEqual(future(base_date=base_date, hour=8), datetime(2023,2,1,8,0), "Should be 8am in February 1st")

        # test 8am in Dicember 31st 9 am, should return 8am on January 1st
        base_date = datetime(2023,12,31,9)
        self.assertEqual(future(base_date=base_date, hour=8), datetime(2024,1,1,8,0), "Should be 8am in January 1st")

        # test February 29th in April 2023, should return February 29th in 2024
        base_date = datetime(2023,4,1,9)
        self.assertEqual(future(base_date=base_date, month=2, day_number=29), datetime(2024,2,29,8,0), "Should be February 29th 2024")

        # test February 29th in April 2022, should return February 29th in 2023
        base_date = datetime(2022,4,1,9)
        self.assertEqual(future(base_date=base_date, month=2, day_number=29), datetime(2023,3,1,8,0), "Should be March 1st 2023")

    def test_just_april(self):
        APRIL = 4
        # it's June, so returns April next year (2024-04-01 08:00)
        base_date = datetime(2023,6,5)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2024,4,1,8,0), "Should be April next year")

        # it's April, so returns April next year
        base_date = datetime(2023,4,10)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2024,4,1,8), "Should be April next year")

        # it's February, so returns April this year (2023-04-01 08:00)
        base_date = datetime(2023,2,15)
        self.assertEqual(future(base_date=base_date, month=APRIL), datetime(2023,4,1,8,0), "Should be April same year")

    def test_just_2023(self):
        # it's 2023, so returns None
        base_date = datetime(2023,5,1)
        self.assertEqual(future(base_date=base_date, year=2023), None, "Should be None (same year)")

        # it's 2022, so returns 2023-01-01 08:00
        base_date = datetime(2022,6,5)
        self.assertEqual(future(base_date=base_date, year=2023), datetime(2023,1,1,8,0), "Should be first day of 2023")

        # it's 2024, so returns None
        base_date = datetime(2024,6,5)
        self.assertEqual(future(base_date=base_date, year=2023), None, "Should be None (previous year)")

    def test_2nd_may(self):
        # it's 2nd May 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,5,2,9)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # it's 1st May 2023, so returns 2023-05-02 08:00
        base_date = datetime(2023,5,1)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2023,5,2,8,0), "Should be 2nd May 2023")

        # it's 3rd May 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,5,3)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # it's 15th April 2023, so returns 2023-05-02 08:00
        base_date = datetime(2023,4,15)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2023,5,2,8,0), "Should be 2nd May 2023")

        # it's 10th June 2023, so returns 2024-05-02 08:00
        base_date = datetime(2023,6,10)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5), datetime(2024,5,2,8,0), "Should be 2nd May 2024")

        # try with hour
        base_date = datetime(2023,6,10,8)
        self.assertEqual(future(base_date=base_date, day_number=2, month=5, hour=10, minute=30), datetime(2024,5,2,10,30), "Should be 2nd May 2024 10:30")

    def test_april_2023(self):
        # # it's February 2023, so returns 2023-04-01 08:00
        base_date = datetime(2023,2,1)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), datetime(2023,4,1,8,0), "Should be April 2023")

        # # it's April 2023, so returns None
        base_date = datetime(2023,4,1,8)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), None, "Should be None (same month)")

        # # it's June 2023, so returns None
        base_date = datetime(2023,6,1)
        self.assertEqual(future(base_date=base_date, month=4, year=2023), None, "Should be None (previous month)")

        # try with hour 11:15
        base_date = datetime(2023,2,1,8,0)
        self.assertEqual(future(base_date=base_date, month=4, year=2023, hour=11, minute=15), datetime(2023,4,1,11,15), "Should be April 1st 2023 11:15")

    def test_16th_2024(self):
        # it's 16th Jan 2024, so returns next month
        base_date = datetime(2024,1,16, 9)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,2,16,8,0), "Should be next month")

        # it's 18th Jan 2024, so returns 16th Feb 2024
        base_date = datetime(2024,1,18)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,2,16,8,0), "Should be 16th next month")

        # it's 1st January 2023 so returns 2024-01-16 08:00
        base_date = datetime(2023,1,1)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024), datetime(2024,1,16,8,0), "Should be 16th January 2024")

        # try 4 pm
        base_date = datetime(2023,1,1,8)
        self.assertEqual(future(base_date=base_date, day_number=16, year=2024, hour=16), datetime(2024,1,16,16,0), "Should be 16th January 2024 16:00")

    def test_april_15th_2023(self):
        # it's April 1st 2023 so returns 2023-04-15 08:00
        # print("First case")
        base_date = datetime(2023,1,4)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), datetime(2023,4,15,8,0), "Should be April 15th 2023")

        # print("Second case")
        # it's April 15th 2023 so returns None
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), None, "Should be None (same day)")

        # print("Third case")
        # it's April 30th 2023 so returns None
        base_date = datetime(2023,4,30)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023), None, "Should be None (previous day)")

    def test_april_15th_2023_9am(self):
        # it's April 15th 2023 8am so returns 2023-04-15 09:00
        base_date = datetime(2023,4,15,8)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=9), datetime(2023,4,15,9,0), "Should be April 15th 2023 9am")

        # it's April 15th 2023 9am so returns None
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=9), None, "Should be None (same date and time)")

        # it's April 15th 2023 10am so returns None
        base_date = datetime(2023,4,15,10)
        self.assertEqual(future(base_date=base_date, day_number=15, month=4, year=2023, hour=10), None, "Should be None (previous time)")

    def test_just_9am(self):
        # it's 8am so returns today 9am
        base_date = datetime(2023,4,15,8)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,15,9,0), "Should be today 9am")

        # it's 9am so returns tomorrow 9am
        base_date = datetime(2023,4,15,9)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,16,9,0), "Should be tomorrow 9am")

        # it's 10am so returns tomorrow 9am
        base_date = datetime(2023,4,15,10)
        self.assertEqual(future(base_date=base_date, hour=9), datetime(2023,4,16,9,0), "Should be tomorrow 9am")

    def test_tuesday_april(self):
        # last Tuesday in April 2023 is April 25th
        # it's Wednesday after last Tuesday in April, so returns first Tuesday in April next year
        base_date = datetime(2023,4,26,8)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2024,4,2,8,0), "Should be first Tuesday in April next year")

        # it's March same year, so returns first Tuesday in April this year
        base_date = datetime(2023,1,15)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2023,4,4,8,0), "Should be first Tuesday in April this year")

        # it's May same year, so returns first Tuesday in April next year
        base_date = datetime(2023,5,26)
        self.assertEqual(future(base_date=base_date, weekday=1, month=4), datetime(2024,4,2,8,0), "Should be first Tuesday in April next year")

    def test_monday_month_starts_on_monday(self):
        base_date = datetime(2023,11,6,8)
        self.assertEqual(future(base_date=base_date, weekday=0, month=1), datetime(2024,1,1,8,0,0), "Should be first Monday in January 2024, which is 1st")

    def test_thursday_2024(self):
        # it's Friday after last Thursday in 2024, so returns None
        base_date = datetime(2024,12,27)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024), None, "Should be None")

        # it's Thursday same year, so returns next Thursday 08:00
        base_date = datetime(2024,12,12,11)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024, hour=8), datetime(2024,12,19,11), "Should be next Thursday")

        # it's Friday same year, so returns next Thursday 08:00
        base_date = datetime(2024,12,13)
        self.assertEqual(future(base_date=base_date, weekday=3, year=2024), datetime(2024,12,19), "Should be next Thursday")

    def test_wednesday_2nd(self):
        # it's April 2nd 2023, first 2nd Wedneesday is in August
        base_date = datetime(2023,4,2,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2023,8,2,8,0), "Should be first 2nd that is Wednesday since next month")

        # it's Tuesday 1st, so returns tomorrow
        base_date = datetime(2022,11,1,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2022,11,2,8,0), "Should be next day")

        # it's 3rd, so returns first 2nd that is Wednesnday since next month
        base_date = datetime(2023,9,3,9)
        self.assertEqual(future(base_date=base_date, day_number=2, weekday=2), datetime(2024,10,2,8,0), "Should be first 2nd that is Wednesday since next month")

    def test_thursday_3rd_january(self):
        # it's 3rd January 2023, first 3rd Thursday is in February
        base_date = datetime(2023,1,3,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2030,1,3,8,0), "Should be first 3rd that is Thursday since next year")

        # it's 1st January 2019, so returns 3rd January 209
        base_date = datetime(2019,1,1,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2019,1,3,8,0), "Should be 3rd January 2019")

        # it's 4th January 2030, so returns first 3rd that is Thursday since next month
        base_date = datetime(2030,1,4,9)
        self.assertEqual(future(base_date=base_date, day_number=3, month=1, weekday=3), datetime(2036,1,3,8,0), "Should be first 3rd that is Thursday since next month")

    def test_friday_4th_2024(self):
        # it's 4th January 2024, first 4th Friday is in October
        base_date = datetime(2024,1,4,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), datetime(2024,10,4,8,0), "Should be first 4th that is Friday since next month")

        # it's 3rd October 2024, so returns 4th October 2024
        base_date = datetime(2024,10,3,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), datetime(2024,10,4,8,0), "Should be 4th October 2024")

        # it's 5th October 2024, so returns None
        base_date = datetime(2024,10,5,9)
        self.assertEqual(future(base_date=base_date, day_number=4, year=2024, weekday=4), None, "There is no 4th Friday in 2024 after October")

    def test_saturday_march_2024(self):
        # it's 5th January 2024, first Saturday March is March 2nd
        base_date = datetime(2024,1,5,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=5), datetime(2024,3,2,8,0), "Should be first Saturday in March 2024")

        # it's 4th March 2024, so returns 9th March 2024
        base_date = datetime(2024,3,4,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=5), datetime(2024,3,9,9,0), "Should be second Saturday in March 2024")

    def test_friday_march_2024(self):
        # it's Saturday 30th March 2024, so returns None
        base_date = datetime(2024,3,30,9)
        self.assertEqual(future(base_date=base_date, month=3, year=2024, weekday=4), None, "There is no Saturday in March after March 30th 2024")

    def test_just_q2(self):
        # it's q1_2023, so returns 2023-04-01
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2023,4,1,8), "Should be April 1st 2023")

        # it's q2_2023, so returns 2024-04-01 08:00
        base_date = datetime(2023,4,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2024,4,1,8), "Should be April 1st 2024")

        # it's q4_2023, so returns 2024-04-01 08:00
        base_date = datetime(2023,12,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2), datetime(2024,4,1,8), "Should be April 1st 2024")

    def test_q1_2023(self):
        # it's q1_2023, so returns None
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), None, "Should be None")

        # it's q3_2022, so returns 2023-01-01 08:00
        base_date = datetime(2022,12,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), datetime(2023,1,1,8), "Should be January 1st 2023")

        # it's q4_2023, so returns None
        base_date = datetime(2023,10,1,9)
        self.assertEqual(future(base_date=base_date, quarter=1, year=2023), None, "Should be None")

    def test_q2_9am(self):
        # it's q1_2023, so returns 2023-04-01
        base_date = datetime(2023,1,1,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2023,4,1,9), "Should be April 1st 2023 09am")

        # it's q2_2023, so returns 2024-04-01 09:00
        base_date = datetime(2023,4,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2024,4,1,9), "Should be April 1st 2024 09am")

        # it's q4_2023, so returns 2024-04-01 09:00
        base_date = datetime(2023,12,5,9)
        self.assertEqual(future(base_date=base_date, quarter=2, hour=9), datetime(2024,4,1,9), "Should be April 1st 2024")

    def test_inconsistent_weekday(self):
        # ask for Friday, October 26th 2023, 26th is Thursday so returns Thursday, October 26th 2023
        base_date = datetime(2023,2,26,7)
        self.assertEqual(future(base_date=base_date, month=10, day_number=26, year=2023, weekday=4), None, "Should be None because 2023-26-10 is Thursday not Friday")

        base_date = datetime(2023,1,26,7)
        self.assertEqual(future(base_date=base_date, month=10, day_number=26, year=2023, weekday=3), datetime(2023,10,26,8), "Should be 2023-26-10 because it is Thursday")

    def test_in_200_seconds(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, seconds=200), datetime(2023,4,1,8,18,20), "Should be 200 seconds later")

    def test_in_90_minutes(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, minutes=90), datetime(2023,4,1,9,45,0), "Should be 90 minutes later")

    def test_in_3_hours(self):
        base_date = datetime(2023,4,1,8,15)
        self.assertEqual(future(base_date=base_date, hours=3), datetime(2023,4,1,11,15,0), "Should be 3 hours later")

    def test_in_4_days(self):
        base_date = datetime(2023,4,1,16,25)
        self.assertEqual(future(base_date=base_date, days=4), datetime(2023,4,5,16,25,0), "Should be 4 days later")

        # in 4 days 9 am
        self.assertEqual(future(base_date=base_date, days=4, hour=9, minute=0, second=0), datetime(2023,4,5,9,0,0), "Should be 4 days later 9 am")

    def test_in_5_weeks(self):
        base_date = datetime(2023,4,1,17,0)
        self.assertEqual(future(base_date=base_date, weeks=5), datetime(2023,5,6,17,0,0), "Should be 5 weeks later")

    def test_in_6_months(self):
        base_date = datetime(2023,7,2,18,10)
        self.assertEqual(future(base_date=base_date, months=6), datetime(2024,1,2,18,10,0), "Should be 6 months later")

    def test_in_1_year(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, years=1), datetime(2024,8,3,19,5,0), "Should be 1 year later")

    def test_in_1_month_and_3_days(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, months=1, days=3), datetime(2023,9,6,19,5,0), "Should be 1 month and 3 days later")

    def test_in_3_weeks_and_2_days(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, weeks=3, days=2), datetime(2023,8,26,19,5,0), "Should be 23 days later")

    def test_in_5_hours_and_40_minutes(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, hours=5, minutes=40), datetime(2023,8,4,0,45,0), "Should be 5 hours and 40 minutes later")

    def test_in_2_years_and_5_months(self):
        base_date = datetime(2023,8,3,19,5)
        self.assertEqual(future(base_date=base_date, years=2, months=5), datetime(2026,1,3,19,5,0), "Should be 28 months later")

if __name__ == '__main__':
    unittest.main()


import folium
import pandas

data = pandas.read_csv("Volcanoes.txt")
lat = list(data["LAT"])
lon = list(data["LON"])
elev = list(data["ELEV"])
name = list(data["NAME"])

html = """
Volcano name:

%s

Height: %s m
"""

def volcano_color(el):
    if el < 1000:
        return 'blue'
    elif el < 2000:
        return 'orange'
    else:
        return 'red'

map = folium.Map(location=[40,-100], zoom_start=6, tiles="Stamen Terrain")
fgv = folium.FeatureGroup(name="Volcanoes")

for lt, ln, el, nm in zip(lat, lon, elev, name):
    iframe = folium.IFrame(html=html % (nm, nm, el), width=200, height=100)
    #fg.add_child(folium.Marker(location=[lt,ln], popup=str(el), icon=folium.Icon(color='red')))
    #fg.add_child(folium.Marker(location=[lt, ln], popup=folium.Popup(iframe), icon = folium.Icon(color = volcano_color(el))))
    fgv.add_child(folium.CircleMarker(location=[lt, ln], radius=5, popup=folium.Popup(iframe),
    fill_color = volcano_color(el), fill_opacity=0.8, color='gray'))

fgp = folium.FeatureGroup(name="Population")

fgp.add_child(folium.GeoJson(data=open('world.json', 'r', encoding='utf-8-sig').read(),
style_function = lambda x: {'fillColor' : 'green' if x['properties']['POP2005'] < 10000000
else 'orange' if x['properties']['POP2005'] < 20000000 else 'red'}))  #to form polygons

map.add_child(fgv)
map.add_child(fgp)
map.add_child(folium.LayerControl())
map.save("Map1.html")


# -*- coding: utf-8 -*-
"""该案例展示了一个利用单目标进化算法实现句子匹配的应用实例。问题的定义详见MyProblem.py."""
from MyProblem import MyProblem  # 导入自定义问题接口

import geatpy as ea  # import geatpy

if __name__ == '__main__':
    # 实例化问题对象
    problem = MyProblem()
    # 快速构建算法
    algorithm = ea.soea_DE_rand_1_L_templet(
        problem,
        ea.Population(Encoding='RI', NIND=50),
        MAXGEN=2000,  # 最大进化代数
        logTras=1)  # 表示每隔多少代记录一次日志信息，0表示不记录。
    # 求解
    res = ea.optimize(algorithm,
                      verbose=True,
                      drawing=1,
                      outputMsg=False,
                      drawLog=False,
                      saveFlag=True)
    print('最佳目标函数值：%s' % res['ObjV'][0][0])
    print('搜索到的句子为：')
    for num in res['Vars'][0, :]:
        print(chr(int(num)), end='')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sample

if __name__ == "__main__":
    sample_string = (
        "Уважаемый %F% %N%! Вы делаете работу по замыканиям функции."
    )
    name, surname = input("Введите имя и фамилию: ").split()
    print(sample.sample(sample_string, name, surname))


from __future__ import print_function
import logging
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
from scapy.all import *

ARP_REQUEST_THRESHOLD=1
ARP_RESPONSE_THRESHOLD=3

HELP="""
pip3 install scapy-python3
"""

class ArpFaker(object):
    @staticmethod
    def arp_request_trigger(pkt):
        msg = 'Request [who-has]: {} ({}) is asking ({}) about {} '.format(
                                                                pkt[ARP].psrc,
                                                                pkt[ARP].hwsrc,
                                                                pkt[ARP].hwdst,
                                                                pkt[ARP].pdst,
                                                                )
        print(msg)

    @staticmethod
    def arp_response_trigger(pkt):
        msg = '*Response [is-at]: {} ({}) has address {} ({})'.format(
                                                               pkt[ARP].pdst,
                                                               pkt[ARP].hwsrc,
                                                               pkt[ARP].psrc,
                                                               pkt[ARP].hwdst,
                                                               )
        print(msg)

    @classmethod
    def arp_filter(cls, pkt):
        if not pkt[ARP]: return False
        #~ print(pkt[ARP].__dict__)
        #~ for x,y in pkt[ARP].fields.items():
            #~ print(x, y)
        if pkt[ARP].op == 1:  # who-has (request)
            cls.arp_request_trigger(pkt)
        elif pkt[ARP].op == 2:  # is-at (response)
            cls.arp_response_trigger(pkt)

    @classmethod
    def arp_listen(cls, stype):
        if stype == 'request':
            req = sniff(prn=cls.arp_filter, filter='arp', store=0, count=ARP_REQUEST_THRESHOLD)
        elif stype == 'response':
            res = sniff(prn=cls.arp_filter, filter='arp', store=0, count=ARP_RESPONSE_THRESHOLD)

    @staticmethod
    def arp_reply(pkt):
        packet = Ether()/ARP(op="who-has",hwsrc=my_mac,psrc=sys.argv[2],pdst=sys.argv[1])

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()

    # By default it will fail with multiple arguments.
    parser.add_argument('--default')

    # This is the correct way to handle accepting multiple arguments.
    # '+' == 1 or more.
    # '*' == 0 or more.
    # '?' == 0 or 1.
    # An int is an explicit number of arguments to accept.
    parser.add_argument('-ip', nargs='+')
    parser.add_argument('-req', action="store_true", help="listen only for requests")
    parser.add_argument('-res', action="store_true", help="listen for requests and responses")
    #~ parser.add_argument('-h', action="store_true", required=0)
    args = parser.parse_args()

    #~ if args.h:
        #~ print(HELP)

    if args.ip:
        print(args.ip)

    if args.req:
        while 1:
            #~ try:
            res = ArpFaker.arp_listen('request')
            #~ except Exception as e: print(e)
    elif args.res:
        while 1:
            #~ try:
            res = ArpFaker.arp_listen('response')
            #~ except Exception as e: print(e)


import sys

from loguru import logger
from web3 import Web3
from web3.middleware import geth_poa_middleware
from web3.gas_strategies.time_based import fast_gas_price_strategy

bsc_Urls = [
    "https://bsc-dataseed.binance.org/",
    "https://bsc-dataseed1.defibit.io/",
    "https://bsc-dataseed1.ninicoin.io/",
    "https://bsc.nodereal.io"
]
bsc = bsc_Urls[0]

VBNB_address = "0xA07c5b74C9B40447a954e1466938b865b6BBea36"

myAddress = "0xC61eb43D6A01a512491Ba08de74dB2D573f56c0b"
myKey = None
if len(sys.argv) > 1:
    myAddress = sys.argv[1]
if len(sys.argv) > 2:
    myKey = sys.argv[2]

def main():
    w3 = Web3(Web3.HTTPProvider(bsc))
    w3.middleware_onion.inject(geth_poa_middleware, layer=0)
    w3.eth.set_gas_price_strategy(fast_gas_price_strategy)

    abi = open("Venus_VBNB.abi").read()

    contractAddress = w3.toChecksumAddress(VBNB_address)
    contract = w3.eth.contract(address=contractAddress, abi=abi)

    # https://web3py.readthedocs.io/en/stable/web3.eth.account.html#sign-a-contract-transaction

    logger.debug(w3.eth.get_transaction_count(myAddress))

    def deposit(amount):
        nonce = w3.eth.get_transaction_count(myAddress)
        contract_txn = contract.functions.mint().buildTransaction({
            'value': amount,
            'chainId': 56,
            'gas': 1000000,
            'gasPrice': w3.toWei('5', 'gwei'),
            'nonce': nonce
        })
        recommended_gasPrice = w3.eth.generate_gas_price()
        contract_txn['gasPrice'] = recommended_gasPrice
        logger.debug(contract_txn) # data: 0x1249c58b
        if myKey:
            signed_txn = w3.eth.account.sign_transaction(contract_txn, myKey)
            logger.debug(signed_txn)
            tx_hash = w3.eth.send_raw_transaction(signed_txn.rawTransaction)
            logger.debug(tx_hash)

    def withdraw(amount):
        nonce = w3.eth.get_transaction_count(myAddress)
        contract_txn = contract.functions.redeemUnderlying(amount).buildTransaction({
            'value': 0,
            'chainId': 56,
            'gas': 1000000,
            'gasPrice': w3.toWei('5', 'gwei'),
            'nonce': nonce
        })
        recommended_gasPrice = w3.eth.generate_gas_price()
        contract_txn['gasPrice'] = recommended_gasPrice
        logger.debug(contract_txn) # data: 0x852a12e3000000000000000000000000000000000000000000000000016345785d8a0000
        if myKey:
            signed_txn = w3.eth.account.sign_transaction(contract_txn, myKey)
            logger.debug(signed_txn)
            tx_hash = w3.eth.send_raw_transaction(signed_txn.rawTransaction)
            logger.debug(tx_hash)

    amount = 100 * 1000 * 1000 * 1000 * 1000 * 1000
    deposit(amount) # https://bscscan.com/tx/0x39994af0328c3a00c221b229b3cd3c235176f088ed358d3ec8ac3b16092856f8
    withdraw(amount) # https://bscscan.com/tx/0xe99a2d22a6ba09d359a8cee76a31532835b07f8bd3675abafc27ea12edfb313b

if __name__ == '__main__':
    main()


import requests
from twilio.rest import Client

STOCK = "TSLA"
COMPANY_NAME = "Tesla Inc"
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

# set twilio application (you must have to register first and use your individual identification)
phone_number = "TWILIO PHONE NUMBER"
account_sid = "YOUR ACCOUNT SID"
auth_token = "YOUR AUTH TOKEN"
client = Client(account_sid, auth_token)

# formatting the date
y_day = datetime.now() - timedelta(days=1)
yesterday = y_day.strftime('%Y-%m-%d')
y2_day = datetime.now() - timedelta(days=2)
before_yesterday = y2_day.strftime('%Y-%m-%d')

# tesla parameters
parameters = {
    "function": "TIME_SERIES_DAILY",
    "symbol": "TSLA",
    "apikey": "YOUR API HERE",
}

# query the data from yesterday and before yesterday
response = requests.get("https://www.alphavantage.co/query", params=parameters)
data = response.json()
yesterday_data = data['Time Series (Daily)'][yesterday]
before_yesterday_data = data['Time Series (Daily)'][before_yesterday]

# query the final data
stock_yesyerday = yesterday_data['4. close']
stock_before_yesyerday = before_yesterday_data['4. close']

# count the percentage
price_inc_or_dec = float(stock_before_yesyerday) * 100 / float(stock_yesyerday) - 100
percentage = (round(price_inc_or_dec))

# newsapi parameters
parameters_tesla = {
    "q": "tesla",
    "from": yesterday,
    "sortBy": "relevancy",
    "apiKey": "YOUR API KEY",
}

# query the information from newsapi website
response2 = requests.get("https://newsapi.org/v2/everything", params=parameters_tesla)
response2.raise_for_status()
tesla_news = response2.json()
title_list = []
description_list = []
#  the 3 most relecant news
for i in range(3):
    articles = tesla_news["articles"][i]["title"]
    title_list.append(articles)
    description = tesla_news["articles"][i]["description"]
    description_list.append(description)

# formatting
def remove_tags(html):
    # parse html content
    soup = BeautifulSoup(html, "html.parser")
    for data in soup(['style', 'script']):
        # Remove tags
        data.decompose()
    # return data by retrieving the tag content
    return ' '.join(soup.stripped_strings)

# examining the growth or fall
# sending sms about changes
if stock_yesyerday < stock_before_yesyerday:
    for i in range(3):
        client = Client(account_sid, auth_token)
        message = client.messages \
            .create(
            body=f"TSLA: ðŸ”»{percentage} %\nHeadline: {remove_tags(title_list[i])}\nBrief: {remove_tags(description_list[i])}",
            from_='TWILIO PHONE NUMBER',
            to='YOUR PHONE NUMBER'
        )
        print(message.status)
else:
    for i in range(3):
        client = Client(account_sid, auth_token)
        message = client.messages \
            .create(
            body=f"TSLA: ðŸ”º{percentage} %\nHeadline: {remove_tags(title_list[i])}\nBrief: {remove_tags(description_list[i])}",
            from_='TWILIO PHONE NUMBER',
            to='YOUR PHONE NUMBER'
        )
        print(message.status)


from django.contrib.auth.models import User
from accounts.models import Profile
from datetime import datetime

print('cron page running......')

def registration_expiry_check():
    today = datetime.today()
    profiles = Profile.objects.filter(signup_confirmation=False)
    for p in profiles:
        print(p)
        print(p.user)
        expdate = p.expiration_date
        if expdate < today:
            p.user.delete()


from flask import current_app, render_template, flash, redirect, url_for, request, jsonify, send_from_directory
from flask_login import current_user, login_required
from werkzeug.urls import url_parse
from werkzeug.utils import secure_filename
from app import db, s3
from app.main.forms import EmptyForm, ActorForm, ArtForm, AuthorForm, BookForm, CharacterForm, SeriesForm, UniverseForm, UploadForm
from app.models import User, Actor, Art, Author, Book, Character, Series, Universe
from app.main import bp
from sqlalchemy import exc
import os

@bp.route('/')
@bp.route('/index')
@login_required
def index():
    posts = [
        {
            'author': {'username': 'John'},
            'body': 'Beautiful day in Portland!'
        },
        {
            'author': {'username': 'Susan'},
            'body': 'The Avengers movie was so cool!'
        }
    ]
    return render_template('index.html', title='Home', posts=posts)

@bp.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(current_app.root_path, 'static'), 'favicon.ico', mimetype='image/vnd.microsoft.icon')

@bp.route('/search')
def search():
    return render_template('search.html')

@bp.route('/explore')
def explore():
    return render_template('explore.html')

@bp.route('/submit')
def submit():
    return render_template('submit.html')

# Actor Endpoints

@bp.route('/actors/add', methods=['GET', 'POST'])
def add_actor():
    return add_resource(Actor, ActorForm, 'resource.html', url_for('main.submit'), url_for('main.get_actors'))

@bp.route('/actors/<int:id>/edit', methods=['GET', 'POST'])
def edit_actor(id):
    return edit_resource(Actor, ActorForm, id, 'resource.html', url_for('main.get_actors'), url_for('main.get_actors'))

@bp.route('/actors')
def get_actors():
    columns = ['first_name', 'middle_name', 'last_name', 'suffix']
    return get_resources(Actor, 'resource.html', url_for('main.explore'), 'main.get_actor', 'main.edit_actor', columns)

@bp.route('/actors/<int:id>')
def get_actor(id):
    columns = ['first_name', 'middle_name', 'last_name', 'suffix']
    return get_resource(Actor, id, 'resource.html', url_for('main.get_actors'), 'main.get_actor', 'main.edit_actor', columns)

@bp.route('/actors/<int:id>', methods=['DELETE'])
def delete_actor(id):
    return delete_resource(Actor, id)

# Art Endpoints

@bp.route('/art/add', methods=['GET', 'POST'])
def add_art():
    return add_resource(Art, ArtForm, 'resource.html', url_for('main.submit'), url_for('main.get_art'))

@bp.route('/art/<int:id>/edit', methods=['GET', 'POST'])
def edit_art(id):
    return edit_resource(Art, ArtForm, id, 'resource.html', url_for('main.get_art'), url_for('main.get_art'))

@bp.route('/art')
def get_art():
    columns = ['artist', 'title', 'description', 'source']
    return get_resources(Art, 'resource.html', url_for('main.explore'), 'main.get_art_id', 'main.edit_art', columns)

@bp.route('/art/<int:id>')
def get_art_id(id):
    columns = ['artist', 'title', 'description', 'source']
    return get_resource(Art, id, 'resource.html', url_for('main.get_art'), 'main.get_art_id', 'main.edit_art', columns)

@bp.route('/art/<int:id>', methods=['DELETE'])
def delete_art(id):
    return delete_resource(Art, id)

# Author Endpoints

@bp.route('/authors/add', methods=['GET', 'POST'])
def add_author():
    return add_resource(Author, AuthorForm, 'resource.html', url_for('main.submit'), url_for('main.get_authors'))

@bp.route('/authors/<int:id>/edit', methods=['GET', 'POST'])
def edit_author(id):
    return edit_resource(Author, AuthorForm, id, 'resource.html', url_for('main.get_authors'), url_for('main.get_authors'))

@bp.route('/authors')
def get_authors():
    columns = ['first_name', 'middle_name', 'last_name', 'suffix']
    return get_resources(Author, 'resource.html', url_for('main.explore'), 'main.get_author', 'main.edit_author', columns)

@bp.route('/authors/<int:id>')
def get_author(id):
    columns = ['first_name', 'middle_name', 'last_name', 'suffix']
    return get_resource(Author, id, 'resource.html', url_for('main.get_authors'), 'main.get_author', 'main.edit_author', columns)

@bp.route('/authors/<int:id>', methods=['DELETE'])
def delete_author(id):
    return delete_resource(Author, id)

# Book Endpoints

@bp.route('/books/add', methods=['GET', 'POST'])
def add_book():
    return add_resource(Book, BookForm, 'resource.html', url_for('main.submit'), url_for('main.get_books'))

@bp.route('/books/<int:id>/edit', methods=['GET', 'POST'])
def edit_book(id):
    return edit_resource(Book, BookForm, id, 'resource.html', url_for('main.get_books'), url_for('main.get_books'))

@bp.route('/books')
def get_books():
    columns = ['universe', 'series', 'author', 'title', 'series_number']
    return get_resources(Book, 'resource.html', url_for('main.explore'), 'main.get_book', 'main.edit_book', columns)

@bp.route('/books/<int:id>')
def get_book(id):
    columns = ['universe', 'series', 'author', 'title', 'series_number']
    return get_resource(Book, id, 'resource.html', url_for('main.get_books'), 'main.get_book', 'main.edit_book', columns)

@bp.route('/books/<int:id>', methods=['DELETE'])
def delete_book(id):
    return delete_resource(Book, id)

# Character Endpoints

@bp.route('/characters/add', methods=['GET', 'POST'])
def add_character():
    return add_resource(Character, CharacterForm, 'resource.html', url_for('main.submit'), url_for('main.get_characters'))

@bp.route('/characters/<int:id>/edit', methods=['GET', 'POST'])
def edit_character(id):
    return edit_resource(Character, CharacterForm, id, 'resource.html', url_for('main.get_characters'), url_for('main.get_characters'))

@bp.route('/characters')
def get_characters():
    columns = ['universe', 'series', 'first_name', 'last_name', 'suffix', 'description']
    return get_resources(Character, 'resource.html', url_for('main.explore'), 'main.get_character', 'main.edit_character', columns)

@bp.route('/characters/<int:id>')
def get_character(id):
    columns = ['universe', 'series', 'first_name', 'last_name', 'suffix', 'description']
    return get_resource(Character, id, 'resource.html', url_for('main.get_characters'), 'main.get_character', 'main.edit_character', columns)

@bp.route('/characters/<int:id>', methods=['DELETE'])
def delete_character(id):
    return delete_resource(Character, id)

# Series Endpoints

@bp.route('/series/add', methods=['GET', 'POST'])
def add_series():
    return add_resource(Series, SeriesForm, 'resource.html', url_for('main.submit'), url_for('main.get_series'))

@bp.route('/series/<int:id>/edit', methods=['GET', 'POST'])
def edit_series(id):
    return edit_resource(Series, SeriesForm, id, 'resource.html', url_for('main.get_series'), url_for('main.get_series'))

@bp.route('/series')
def get_series():
    columns = ['universe', 'title']
    return get_resources(Series, 'resource.html', url_for('main.explore'), 'main.get_series_id', 'main.edit_series', columns)

@bp.route('/series/<int:id>')
def get_series_id(id):
    columns = ['universe', 'title']
    return get_resource(Series, id, 'resource.html', url_for('main.get_series'), 'main.get_series_id', 'main.edit_series', columns)

@bp.route('/series/<int:id>', methods=['DELETE'])
def delete_series(id):
    return delete_resource(Series, id)

# Universe Endpoints

@bp.route('/universes/add', methods=['GET', 'POST'])
def add_universe():
    return add_resource(Universe, UniverseForm, 'resource.html', url_for('main.submit'), url_for('main.get_universes'))

@bp.route('/universes/<int:id>/edit', methods=['GET', 'POST'])
def edit_universe(id):
    return edit_resource(Universe, UniverseForm, id, 'resource.html', url_for('main.get_universes'), url_for('main.get_universes'))

@bp.route('/universes')
def get_universes():
    columns = ['title']
    return get_resources(Universe, 'resource.html', url_for('main.explore'), 'main.get_universe', 'main.edit_universe', columns)

@bp.route('/universes/<int:id>')
def get_universe(id):
    columns = ['title']
    return get_resource(Universe, id, 'resource.html', url_for('main.get_universes'), 'main.get_universe', 'main.edit_universe', columns)

@bp.route('/universes/<int:id>', methods=['DELETE'])
def delete_universe(id):
    return delete_resource(Universe, id)

# CRUD functions for basic resource management

def add_resource(ResourceClass, FormClass, default_template, redirect_url, back_url):
    form = FormClass()
    if form.validate_on_submit():
        resource = ResourceClass()
        form.populate_obj(resource)

        try:
            db.session.add(resource)
            db.session.commit()
            flash('Congratulations, you added a {}!'.format(ResourceClass.__name__))
        except exc.IntegrityError:
            db.session.rollback()
            flash('Duplicate {} entry detected'.format(ResourceClass.__name__), 'error')
            return render_template('errors/500.html'), 500
        except:
            db.session.rollback()
            flash('Unknown error occured', 'error')
            return render_template('errors/500.html'), 500

        return redirect(redirect_url)

    return render_template(default_template, title='Add to {} Collection'.format(ResourceClass.__name__), form=form, back_url=back_url)

def edit_resource(ResourceClass, FormClass, id, default_template, redirect_url, back_url):
    resource = ResourceClass.query.get(id)
    if resource:
        form = FormClass(obj=resource)
        if form.validate_on_submit():
            try:
                form.populate_obj(resource)
                db.session.commit()
                flash('{} record update success'.format(ResourceClass.__name__))
            except exc.IntegrityError:
                db.session.rollback()
                flash('Duplicate {} entry detected'.format(ResourceClass.__name__), 'error')
                return render_template('errors/500.html'), 500
            except:
                db.session.rollback()
                flash('Unknown error occured', 'error')
                return render_template('errors/500.html'), 500

            return redirect(redirect_url)

    return render_template(default_template, title='Edit Resource', form=form, back_url=back_url)

def get_resources(ResourceClass, default_template, back_url, get_uri, edit_uri, columns):
    resources = ResourceClass.query.all()
    return render_template(default_template, results=resources, back_url=back_url, get_uri=get_uri, edit_uri=edit_uri, columns=columns)

def get_resource(ResourceClass, id, default_template, back_url, get_uri, edit_uri, columns):
    resource = ResourceClass.query.get(id)

    if resource:
        return render_template(default_template, results=[resource], back_url=back_url, get_uri=get_uri, edit_uri=edit_uri, columns=columns)
    else:
        return render_template('errors/404.html'), 404

def delete_resource(ResourceClass, id):
    try:
        result = ResourceClass.query.filter_by(id=id).delete()
        db.session.commit()

        if result:
            flash('Successfully deleted {} record'.format(ResourceClass.__name__))
            return jsonify(success=True)
        else:
            flash('Error deleting {} id: {}'.format(ResourceClass.__name__, id), 'error')
            return jsonify(success=False)
    except:
        db.session.rollback()
        flash('Error deleting {} id: {}'.format(ResourceClass.__name__, id), 'error')
        return jsonify(success=False)

# CRUD functions for S3

@bp.route('/upload', methods=['GET', 'POST'])
def upload_file():
    form = UploadForm()

    if form.validate_on_submit():
        filename = secure_filename(form.file.data.filename)
        print("ATTEMPTING FILE UPLOAD")
        output = send_to_s3(form.file.data)
        flash(output)
        print("ATTEMPTING FILE DELETE")
        deleteput = delete_from_s3('Reyna.png')
        print(deleteput)
        return redirect(url_for('main.upload_file'))

    return render_template('upload.html', form=form)

def send_to_s3(file):
    """
    Docs: http://boto3.readthedocs.io/en/latest/guide/s3.html
    """
    try:
        s3.upload_fileobj(
            file,
            current_app.config["S3_BUCKET"],
            file.filename,
            ExtraArgs={
                "ContentType": file.content_type
            }
        )
    except Exception as e:
        print("Something Happened: ", e)
        return e
    return "{}{}".format(current_app.config["S3_LOCATION"], file.filename)

def delete_from_s3(key):
    try:
        result = s3.delete_object(Bucket=current_app.config["S3_BUCKET"], Key=key)
        print(result)
        return result
    except Exception as e:
        print("Something Happened: ", e)
        return e


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelBinarizer

train = pd.read_csv("/Users/fadouabadr/Library/Mobile Documents/com~apple~CloudDocs/OBJECTIF CDI/IT stuff/PYTHON/Titanic/Datas/Train.csv")
test = pd.read_csv("/Users/fadouabadr/Library/Mobile Documents/com~apple~CloudDocs/OBJECTIF CDI/IT stuff/PYTHON/Titanic/Datas/Test.csv")

train_copy = train.copy()
test_copy = test.copy()

survived_train = train['Survived'].copy() # Serues object
PassengerId_test = test['PassengerId']

# ***************************************************************************************************
# ***************************************************************************************************
#print first 3 rows of train and test data
train.head(3)
test.head(3)

#print the total number of columns in the train data
print("The total number of rows in the training data is", format(train.shape[0]))
print("the total number of rows in the test data is", format(test.shape[0]))
print(60*'-')
print("The total number of columns in the training data is", format(train.shape[1]))
print("the total number of columns in the test data is", format(test.shape[1]))

print("The different attributes in the train set are", format(train.columns))

#let's remove the PassengerId/Cabin attribute from the test and train set as they are literally useless for our models
train = train.drop(columns=['PassengerId', 'Cabin'], axis=1)
test = test.drop(columns=['PassengerId','Cabin'], axis=1)
# ***************************************************************************************************
# ***************************************************************************************************

# *************************************             ******************************************
# ==================================== DATA ANALYSIS ==========================================
# *************************************             ******************************************
print((train.groupby(['Sex','Survived']).Survived.count() * 100) / train.groupby('Sex').Survived.count())

print((train.groupby(['Pclass','Survived']).Survived.count() * 100) / train.groupby('Pclass').Survived.count())

print((train.groupby(['Embarked','Survived']).Survived.count() * 100) / train.groupby('Embarked').Survived.count())

print(train.groupby(by=["Survived"]).mean()["Age"])

#let's use the info() method to get a better view of our train set
info_train = train.info()
info_test = test.info()

# ************************************                                 ******************************************
# ==================================== DEALING WITH NAN/MISSING VALUES ==========================================
# ************************************                                 ******************************************

datas = [train,test]

#train dataset : 177 values are missing in the 'Age' attribute, 2 are missing in the 'Embarked' attribute
#test dataset : 86 values are missing in the 'Age' attrivute, 1 is missing in the 'Fare' attribute
#Let's visualise the number of null-values/missing values in the training set using matplotlib's bar() method
for data in datas:
    print(data.isna().sum()) #here isna and isnull is the same (nan values)
    plt.figure(figsize=(10,3))
    plt.bar(data.columns, data.isnull().sum())
    plt.xticks(rotation=90)
    plt.xlabel("Attributes"); plt.ylabel("# of missing values in the dataset")
    print(20 * '*')

print('Survived/Dead\n', train['Survived'].value_counts())
print(50*'-')
#print('Class\n', train["Pclass"].value_counts())
print(50*'-')
print('Sex\n', train["Sex"].value_counts())
print(50*'-')
print('Siblings/Spouses\n', train['SibSp'].value_counts())
print(50*'-')
print('Parents/Children\n', train['Parch'].value_counts())
print(50*'-')
print('Embarked\n', train['Embarked'].value_counts())

# ***************************************************************************************************
# ***************************************************************************************************
#*******************************************************************************************************
#The 'Pclass' attribute does not have any missing values

#----------------------------------------------------- Plot ----------------------------------------------------------------------
# Visualising whether Pclass affects the survival rate or not
plt.figure(figsize=(10,5))
sns.countplot(x = 'Pclass', hue = 'Survived', data = train, palette = 'Paired')
plt.show()
# We notice that the 3rd class is the class that had the most deadly rate followed by the 2dn class and the 1st.
# This will be confirmed when looking to the Correlation matrix

#Let's plot several conditional relationship related the the Pclass attributes
grid = sns.FacetGrid(train, col='Survived', row='Pclass', palette='Paired')
grid.map_dataframe(sns.histplot, x='Age')
plt.legend()
plt.show()

fg = sns.FacetGrid(train, hue='Pclass', aspect=3)
fg.map(sns.kdeplot, 'Age', shade=True)
plt.legend()
fg.set(xlim=(0, 80))

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
encoder = LabelBinarizer()
train_pclass = encoder.fit_transform(train["Pclass"])
test_pclass = encoder.fit_transform(test["Pclass"])

#use of the classes_ attribute to see which index corresponds to which class
#Rq: do not confuse the class (which is an indication of the socio economical background) and the class (which is .....)
print('The different classes in the Pclass attribute are', encoder.classes_) #class 0 = 1st class / class 1 = 2nd Class / class 2 = 3rd Class

#concatenate with the new dummies variables and delete the 'Embarked' attribute
#Rq: the train_pclass output is an array and we have to convert it to a df
train.drop(columns=["Pclass"], axis=1, inplace=True)
test.drop(columns=["Pclass"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_pclass)], axis=1)
test = pd.concat([test, pd.DataFrame(test_pclass)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'1st class',1:'2nd class',2:'3rd class'})
test = test.rename(columns={0:'1st class',1:'2nd class',2:'3rd class'})
#*******************************************************************************************************

############################################## SEX ATTRIBUTE #######################################
#The 'Sex' attrivute does not have any missing value or nan value
plt.figure(figsize=(10,5))
sns.countplot(x = 'Sex', hue = 'Survived', data = train, palette = 'Paired')
plt.show()

#Let's plot several conditional relationship related the the Pclass attributes
grid = sns.FacetGrid(train, col='Survived', row='Sex', palette='Paired')
grid.map_dataframe(sns.histplot, x='Age')
plt.show()

fg = sns.FacetGrid(train, hue="Sex", aspect=3)
fg.map(sns.kdeplot, "Age", shade=True)
plt.legend()
fg.set(xlim=(0, 80))

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
# Let's convert the numerical attributes 'male' and 'female' into indicator variables using the LabelBinarizer function
encoder = LabelBinarizer()
train_sex = encoder.fit_transform(train["Sex"])
test_sex = encoder.fit_transform(test["Sex"])

#drop the 'Sex' attrivute for both the data and the test set
#concatenate the new variables into our training and test datasets
#Rq: we have only added the column female because this columns gives us all the necessary information we need
train.drop(columns=["Sex"], axis=1, inplace=True)
test.drop(columns=["Sex"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_sex)], axis=1)
test = pd.concat([test, pd.DataFrame(test_sex)], axis=1)

#rename the columns 'female' into 'Sex' #female = 1, male = 0
train = train.rename(columns={0:'female=1/male=0'})
test = test.rename(columns={0:'female=1/male=0'})

##############################################  AGE ATTRIBUTE #######################################
#the 'Age' attribute has 714 non-null values which gives us 177 null/nan values

#plotting histogram of the Age
plt.hist(train['Age'], bins= 10)
plt.xlabel("Age"); plt.ylabel("Frequence")
plt.show()

#breakdown by age and sex

fg = sns.FacetGrid(train_copy, col='Sex', row='Pclass', hue='Pclass', height=2.5, aspect=2.5)
fg.map(sns.kdeplot, 'Age', shade=True)
fg.map(sns.rugplot, 'Age')
sns.despine(left=True)
fg.set(xlim=(0, 80))

#let's get a better view of statistical properties of the 'Age' attrivute in the train dataset
#and fill the 'nan' values for 'Age' with the mean
for data in datas:
    print(round(data['Age'].describe(),3))
    print(30*'-')

#Let's compute the missing values by replacing them by the median.
train['Age'].fillna(train['Age'].median(), inplace=True)
test['Age'].fillna(test['Age'].median(), inplace=True)

##############################################  FARE ATTRIBUTE #######################################
#there is 1 missing value in the test dataset for the 'Fare' attribute.

#plotting histogram of the Fare price pa
f, axes = plt.subplots(1,1, figsize=(10,5))
g1 = sns.histplot(x=train.Fare, color='red', ax= axes)
plt.title("Fare price paid by passengers")
plt.xlabel("Fare"); plt.ylabel("Frequence")
plt.show()

fg = sns.FacetGrid(train, hue="Fare", aspect=3)
fg.map(sns.kdeplot, "Age", shade=True)
plt.legend()
fg.set(xlim=(0, 80))

fg = sns.FacetGrid(train_copy, hue="Sex", aspect=3)
fg.map(sns.kdeplot, "Fare", shade=True)
plt.legend()
fg.set(xlim=(-100, 100))

fg = sns.FacetGrid(train_copy, col='Survived', height=2.5, aspect=2.5)
fg.map(sns.kdeplot, 'Fare', shade=True)
fg.map(sns.rugplot, 'Fare')
sns.despine(left=True)
fg.set(xlim=(-100, 100))

'''
#creating of a sublist of Fare price paid by people that died/survived
Fare_Died = []
Fare_Survived = []
for i in range(0,len(train)):
    if train.Survived[i] == 0: #did not survived
        Fare_Died.append(train.Fare[i])
    else:
        Fare_Survived.append(train.Fare[i])

f, axes = plt.subplots(nrows=1, ncols=1, figsize=(20,10))
g1 = sns.histplot(Fare_Died, color="orange",ax = axes)
plt.title("Fare distribution for the people who did not survive")
plt.show()

f, axes = plt.subplots(1,1, figsize = (20, 10))
g1 = sns.histplot(Fare_Survived, color="blue",ax = axes)
plt.title("Fare distribution for the people who survived")
plt.show()
'''

#since there is 1 missing value in the 'Fare' attrovute in the test dataset, we simply fillna by the mean
test["Fare"].fillna(test['Fare'].mean(), inplace = True)

'''
# here, we want to break it up into discrete bins by breaking down the data into several ranges
# let's breal k up our Fare features into multiple thresolds (the Q1, Q2, Q3)
train['Fare'] = np.digitize(train["Fare"], bins=[7.910400,14.454200,31.000000], right=True)
test['Fare'] = np.digitize(test["Fare"], bins=[7.895800,14.454200, 31.500000], right=True)

#use of the LabelBinarizer function that gives numerical attributes to text values
encoder = LabelBinarizer()
train_fare = encoder.fit_transform(train["Fare"])
test_fare = encoder.fit_transform(test["Fare"])

#use of the classes_ attrivute to see which index is which class
print(encoder.classes_)

train = pd.concat([train, pd.DataFrame(train_fare)], axis=1)
train = train.drop(columns=["Fare"], axis=1)
test = pd.concat([test, pd.DataFrame(test_fare)], axis=1)
test = test.drop(columns=["Fare"], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'Fare range 1',1:'Fare range 2',2:'Fare range 3',3:'Fare range 4'})
test = test.rename(columns={0:'Fare range 1',1:'Fare range 2',2:'Fare range 3',3:'Fare range 4'})
'''

##############################################  EMBARKED ATTRIBUTE #######################################
sns.countplot(y='Embarked', hue='Survived', data=train, orient='h',palette='Set3')
plt.show()

#from datavisualisation, we can see that Southhampton is the most frequent embarked port
#so, filling the missing values with 'S' (2 missing values in the train dataset)
train['Embarked'].fillna('S', inplace=True)
test['Embarked'].fillna('S', inplace=True)

fg = sns.FacetGrid(train_copy, col='Survived', row='Embarked', height=2.5, aspect=2.5, palette='Set2')
fg.map(sns.kdeplot, 'Fare', shade=True)
fg.map(sns.rugplot, 'Fare')
sns.despine(left=True)
fg.set(xlim=(-100, 100))

grid = sns.FacetGrid(train_copy, col='Survived', row='Embarked', palette='Set2')
grid.map(sns.barplot, 'Sex', 'Fare', ci=None)
grid.add_legend()
plt.show()

#use of the LabelBinarizer function that gives numerical attributes to text values
# here C, S, Q
encoder = LabelBinarizer()
train_embarked = encoder.fit_transform(train["Embarked"]) #print the classes (C = classe 0, Q = Classe 1, S = Classe 2)
test_embarked = encoder.fit_transform(test["Embarked"])

#use of the classes_ attrivute to see which index is which class
print('The different classes in the Pclass attribute are', encoder.classes_) #class 0 = C / class 1 = Q / class 2 = S

#concatenate with the new dummies variables and delete the 'Embarked' attribute
#Rq: the train_embarked output is an array and we have to convert it to a df
train.drop(columns=["Embarked"], axis=1, inplace=True)
test.drop(columns=["Embarked"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_embarked)], axis=1)
test = pd.concat([test, pd.DataFrame(test_embarked)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'C',1:'Q',2:'S'})
test = test.rename(columns={0:'C',1:'Q',2:'S'})

##############################################  SIBSP & PARCH ATTRIBUTES #######################################
#When looking at the correlation matrix, we see that the corr betwee SibSp and parch = 41%
# so we  create a new attribute 'alone'

def accompanied(x):
        if (x['SibSp'] + x['Parch']) > 0:
            return 1
        else:
            return 0

##for data in datas:
#    data['family'] = data.apply(fam, axis=1)
train['Accompanied'] = train.apply(accompanied, axis = 1)
test['Accompanied'] = test.apply(accompanied, axis = 1)

#drop both the columns Parch and Sibsp
train.drop(columns=['Parch','SibSp'], axis=1, inplace=True)
test.drop(columns=['Parch','SibSp'], axis=1, inplace=True)

#let's plot the histogram of the new attribute vs Survived/Not survuved
plt.figure(figsize=(10,6))
sns.countplot(x='Accompanied', hue='Survived', data = train, palette="YlOrBr")
plt.show()
#those having accompaned have better chance to survive according to the data

##############################################  Name ATTRIBUTES #######################################
# ************************************                                          ******************************************
# ==================================== CREATING NEW FEATURE TITLE EXTRACTING FROM ==========================================
#                                              EXISTING FEATURE NAME
# ************************************                                          ************************************
#we create a 'Title' feature which contains the title of each passenger and drop the Name column

print(train['Name'].str.extract(' ([A-Za-z]+)\.', expand=False).unique().size)
print(test['Name'].str.extract(' ([A-Za-z]+)\.', expand=False).unique().size)

train['Title'] = train['Name'].str.extract(' ([A-Za-z]+)\.', expand=False)
test['Title'] = test['Name'].str.extract(' ([A-Za-z]+)\.', expand=False)
train.drop(columns=['Name'], axis=1, inplace=True)
test.drop(columns=['Name'], axis=1, inplace=True)

#number of titles in the train set
#for data in datas:
#    print(data.Title.value_counts()) #Mr / Miss / Mrs / Master / Dr
#    print(30*'-')

print('All the Titles are: \n', set(train.Title) | set(test.Title))

#Let's replace least occuring (1 time) title in the dataset with rare
least_occuring =  ['Countess', 'Capt', 'Dona', 'Dr', 'Ms', 'Major', 'Mme',
                   'Jonkheer', 'Col', 'Don', 'Lady', 'Sir', 'Mlle', 'Rev']

train['Title'] = train['Title'].replace(least_occuring, "rare")
test['Title'] = test['Title'].replace(least_occuring, "rare")

#Let's take a look at our Titles
print(train["Title"].value_counts(), test["Title"].value_counts())

#----------------------------------------------------- LabelBinarizer ----------------------------------------------------------------------
encoder = LabelBinarizer() #class 0 = Master / Class 1 = Miss / class 2 = Mr / Class 3 = Mrs / Class 4 = rare
train_title = encoder.fit_transform(train['Title'])
test_title = encoder.fit_transform(test['Title'])
print('The different classes in the Pclass attribute are', encoder.classes_)

#concatenate with the new dummies variables and delete the 'Title' attribute
train.drop(columns=["Title"], axis=1, inplace=True)
test.drop(columns=["Title"], axis=1, inplace=True)
train = pd.concat([train, pd.DataFrame(train_title)], axis=1)
test = pd.concat([test, pd.DataFrame(test_title)], axis=1)

# rename the columns of the new train df
train = train.rename(columns={0:'Master',1:'Miss',2:'Mr',3:'Mrs',4:'rare'})
test = test.rename(columns={0:'Master',1:'Miss',2:'Mr',3:'Mrs',4:'rare'})

##############################################  Ticket ATTRIBUTES #######################################
#I don't see any accurate relation between this variable and the others, so we'll delete it for the moment
train.drop(columns=['Ticket'], axis=1, inplace=True)
test.drop(columns=['Ticket'], axis=1, inplace=True)

########################################## CHECKING FOR MISSING VALUES ##########################
#no missing values in both the datasets
print(train.info())
print(train.isna().sum())
print(test.isna().sum())

# *************************************                         ******************************************
# ==================================== LOOKING FOR CORRELATIONS ==========================================
# *************************************                         ******************************************

#Pearson Correlation matrix
plt.figure(figsize=(15,10))
corr = train.corr()
sns.heatmap(corr, annot=True)
plt.show()

# look at how much each attrivute correlates with the "survived" instance
print(corr["Survived"].sort_values(ascending=False))
# there is a positive linear correlation between the number of person survived and the Sex (female in this case)
# there is a negative linear correlation between the number of person that survived and the Class

# ************************************  FEATURE SCALING WITH STANDARDIZATION                                   ******************************************
# ====================================  ==========================================
from sklearn.preprocessing import StandardScaler

standard_scaler = StandardScaler()

train.drop(['Survived'], axis=1, inplace=True)
X_train = standard_scaler.fit_transform(train)
y_train = survived_train
X_test = standard_scaler.fit_transform(test)

###############################################################################################
from sklearn.linear_model import SGDClassifier, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.ensemble import VotingClassifier

classifiers = ['LogisticRegression','SGDClassifier','DecisionTreeClassifier',
               'RandomForestClassifier', 'AdaBoostClassifier', 'KNeighborsClassifier']

models = [LogisticRegression(), SGDClassifier(random_state=42), DecisionTreeClassifier(random_state=42),
          RandomForestClassifier(), AdaBoostClassifier(DecisionTreeClassifier(random_state=42)), KNeighborsClassifier()]

#voting_clf = VotingClassifier(estimators=[('lr', LogisticRegression()), ('sgd', SGDClassifier(random_state=42)),
#                                                                 ('dtc', DecisionTreeClassifier(random_state=42)),
#                                                                 ('rfc', RandomForestClassifier())], voting='hard')

#from sklearn.metrics import accuracy_score

#for clf in (LogisticRegression(), SGDClassifier(random_state=42), DecisionTreeClassifier(random_state=42), RandomForestClassifier(), voting_clf):
#    clf.fit(X_train, y_train)
#    y_pred = clf.predict(X_train)
#    print(clf.__class__.__name__, accuracy_score(y_train, y_pred))

#Let's use cross_val_score() method to evaluate our model using K-fold cross-validation
from sklearn.model_selection import cross_val_score, StratifiedKFold, cross_validate
################################################################################################

##########################################################################################
#Use of cross_validate function that allows us to use multiple scores / It is best using this instead of what is up
j =0
results = {}
cross_validate_ = {}
for i in models:
    model = i
    scoring = ['accuracy','precision','f1']
    cross_validate_[classifiers[j]] = cross_validate(model, X_train, y_train, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1), scoring=scoring, n_jobs=-1)
    results[j] = pd.DataFrame(cross_validate_[classifiers[j]])
    j += 1

j =0
#test_precision_LG, test_precision_SGD, test_precision_DTC, test_precision_RFC, test_precision_ADA, test_precision_KNN = []
for i in models:
    test_precision_ = {}
    model = i
    for index in range(0,6):
        test_precision_[classifiers[j]] = pd.DataFrame(results[0]['test_precision']).T
    j += 1

test_precision_LG = pd.DataFrame(results[0]['test_precision']).T
test_precision_SGD = pd.DataFrame(results[1]['test_precision']).T
test_precision_DTC = pd.DataFrame(results[2]['test_precision']).T
test_precision_RFC = pd.DataFrame(results[3]['test_precision']).T
test_precision_ADA = pd.DataFrame(results[4]['test_precision']).T
test_precision_KNN = pd.DataFrame(results[5]['test_precision']).T
test_precision_all = pd.concat([test_precision_LG, test_precision_SGD, test_precision_DTC, test_precision_RFC, test_precision_ADA, test_precision_KNN], axis=0)
test_precision_all.index = classifiers
test_precision_mean = test_precision_all.mean(axis=1)
test_precision_all= pd.concat([test_precision_all, test_precision_mean], axis=1)
test_precision_all.columns = ['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5', 'Fold 6', 'Fold 7', 'Fold 8', 'Fold 9', 'Fold 10','Mean']

test_accuracy_LG = pd.DataFrame(results[0]['test_accuracy']).T
test_accuracy_SGD = pd.DataFrame(results[1]['test_accuracy']).T
test_accuracy_DTC = pd.DataFrame(results[2]['test_accuracy']).T
test_accuracy_RFC = pd.DataFrame(results[3]['test_accuracy']).T
test_accuracy_ADA = pd.DataFrame(results[4]['test_accuracy']).T
test_accuracy_KNN = pd.DataFrame(results[5]['test_accuracy']).T
test_accuracy_all = pd.concat([test_accuracy_LG, test_accuracy_SGD, test_accuracy_DTC, test_accuracy_RFC, test_accuracy_ADA, test_accuracy_KNN], axis=0)
test_accuracy_all.index = classifiers
test_accuracy_mean = test_accuracy_all.mean(axis=1)
test_accuracy_all= pd.concat([test_accuracy_all, test_accuracy_mean], axis=1)
test_accuracy_all.columns = ['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5', 'Fold 6', 'Fold 7', 'Fold 8', 'Fold 9', 'Fold 10','Mean']

##########################################################################################

##########################################################################################
# A much better way to evaluate the performance is to look at the confusion matrix
from sklearn.model_selection import cross_val_predict #performs K folds cross validation but instead of returning the evaluation scores, it returns the predictions made on each test fold
from sklearn.metrics import confusion_matrix, roc_auc_score

j =0
cross_val_predictions = {}
conf_matrix = {}
roc_auc_scores = {}
for i in models:
    model = i
    cross_val_predictions[classifiers[j]] = cross_val_predict(model, X_train, y_train, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1))
    conf_matrix[classifiers[j]] = confusion_matrix(y_train, cross_val_predictions[classifiers[j]])
    plt.show()
    j += 1

##########################################################################################

# ************************************  FINE TUNE YOUR MODEL ******************************************
# ====================================  Grid Search ==========================================
#let search for the best possible combinations of hyperparameters values for RandomFirestClassifier
from sklearn.model_selection import GridSearchCV
import numpy as np

param_grid = {'n_estimators': [int(x) for x in np.linspace(200, 2000, num = 10)], 'max_features': ['auto', 'sqrt'],
              'max_depth' : [int(x) for x in np.linspace(10, 110, num = 11)],
              'criterion' :['gini', 'entropy']}

grid_search = GridSearchCV(RandomForestClassifier(), param_grid=param_grid, cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=1))

best_model = grid_search.fit(X_train, y_train)
print(best_model.best_params_)

classifier_best = RandomForestClassifier(random_state=42, n_estimators=300, criterion='entropy',max_depth=6, max_features='sqrt')
classifier_best.fit(X_train, y_train)

X_test_pred = classifier_best.predict(X_test)
final_results = pd.DataFrame({'PassengerId': PassengerId_test, 'Survived': X_test_pred})
final_results


# Module 07
# Q 10 (complete)

# tutorial :
"""
Given a string ‘s’ you need to find the words that are in list ‘a’ and use the next words on string ‘s’ to create a new string. Save it inside a file called ‘out.txt’. Remember to close the file after writing.

Write a function named create_new_string() and write your code inside this function.

a = ['oh', 'Emelia', 'to']

s = "Oh, I got two tickets for Dhaka. I and Emelia love to visit different places very much. This time we are going to Bangladesh."

output = "I love Bangladesh" (inside a file)

"""

# function
def create_new_string(s,a):

    # print(type(s))
    s = s.replace(',', '') # remove comma from oh,
    s = s.replace('.', '') # remove . from the end,
    # print(s)

    # for i in range(len(a)):
    #     a[i]=a[i].lower()

    # s=s.lower() # lowering
    s=s.split() # make a list
    # print(type(s))
    # print(a)
    # print(s)

    new_list=[]
    for i in range(len(s)):
        # print(s[i])
        for l in range(len(a)):
            if s[i].lower()==a[l].lower():
                # print(s[i],a[l])
                new_list.append(s[i+1])
                break

    # print(new_list)

    new=""
    for i in range(len(new_list)):
        if i==2:
            continue
        else:
            new+=new_list[i]+" "
    print(new)

    # file write
    file=open("out.txt",'w')
    file.write(new)

    # file read and close
    file=open("out.txt")
    # print(file.read())
    file.close()

# main
a = ['oh', 'Emelia', 'to']
s = "Oh, I got two tickets for Dhaka. I and Emelia love to visit different places very much. This time we are going to Bangladesh."

create_new_string(s,a)


class SerialEntrepreneur:
    def __init__(self, id, full_name, last_name, first_name,
                 primary_position, primary_company, board_seats,
                 roles, deal_roles, phone, email, location,
                 address_line1, address_line2, city, state,
                 post_code, country, fax, biography, pitchbook_link):
        self.ID = id
        self.full_name = full_name
        self.last_name = last_name
        self.first_name = first_name
        self.primary_position = primary_position
        self.primary_company = primary_company
        self.board_seats = board_seats
        self.roles = roles
        self.deal_roles = deal_roles
        self.phone = phone
        self.email = email
        self.location = location
        self.address_line1 = address_line1
        self.address_line2 = address_line2
        self.city = city
        self.state = state
        self.post_code = post_code
        self.country = country
        self.full_address = self.get_full_address()
        self.fax = fax
        self.biography = biography
        self.pitchbook_link = pitchbook_link

    def get_full_address(self):
        full_address = self.address_line1 + ", " + self.city + \
                       ", " + self.state + " " + self.post_code +\
                       ", " + self.country
        return full_address


# 对通用代码封装
from selenium import webdriver
from selenium.webdriver.remote.webdriver  import WebDriver
import time

class BasePage:
    def __init__(self, driver: WebDriver=None):
        if driver is None:
            # index页面使用这个方法
            self._driver = webdriver.Chrome()
            self._driver.implicitly_wait(3)
            self._driver.get(self._base_url)
        else:
            # 是因为有的页面需要引用其他的driver
            # login和register等页面需要这个方法
            self._driver = driver

    def close(self):
        time.sleep(20)
        self._driver.quit()


'''
Created on 2013-4-1

@author: zhangbin
'''

import os
import logging
import logging.handlers
import traceback

from util.config import options, define

define('--log-dir', name='logdir', type='string')

def setup(product_name):
    try:
        log_dir = options.logdir
        if not log_dir:
            log_dir = os.path.normpath(os.path.join(__file__,
                                                    os.pardir,
                                                    os.pardir,
                                                    os.pardir,
                                                    'log'))
        logger=logging.getLogger()
        logger.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(module)s.%(funcName)s:%(lineno)d - %(message)s', '')
        _console=logging.StreamHandler();
        _file=logging.FileHandler(log_dir+'/'+product_name+'.log')
        _file.setFormatter(formatter)
        _console.setFormatter(formatter)
        logger.addHandler(_console)
        logger.addHandler(_file)
    except Exception:
        traceback.print_exc()
        raise

class WritableLogger(object):
    '''
    classdocs
    '''

    def __init__(self, logger, level=logging.INFO):
        '''
        Constructor
        '''
        self.logger = logger
        self.level = level

    def write(self, msg):
        self.logger.log(self.level, msg)

LOG = logging.getLogger()


import discord
import asyncio
from datetime import datetime

from discord.ext import commands

class FeedBack(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command(aliases=["fd"])
    @commands.cooldown(1, 7200, commands.BucketType.user)
    async def feedback(self, ctx, *, args=None):
        if args is None:
            await ctx.send(f"{ctx.message.author.mention} bitte geb ein Feedback zum Bot.")
        elif len(args) < 100:
            await ctx.send(f"{ctx.message.author.mention} dein Feedback muss mindestens 100 Zeiche lang sein. Länge des Feedbacks {len(args)} Zeichen.")
        elif len(args) > 2048:
            await ctx,send(f"{ctx.message.author.mention} dein Feedback darf nur 2048 Zeichen lang sein. Länge des Feedbacks {len(args)} Zeichen.")
        else:
            embed = discord.Embed(title="**Feedback**", color=discord.Color.green())
            embed.add_field(name='Nachricht', value=args, inline=False)
            embed.set_footer(text=f"{ctx.message.author.name} | {ctx.guild.name}", icon_url=ctx.guild.icon_url_as(size=512))

            msg = await ctx.send(embed=embed)
            await asyncio.sleep(3)
            await msg.delete()
            await ctx.send(f"{ctx.message.author.mention} dein Feedback wurde gesendet!")

            for guild in self.client.guilds:
                if guild.name == "➤ Xan Bot | Support":
                    for channel in guild.text_channels:
                        if channel.id == 738067878552535054:
                            await channel.send(embed=embed)

    @feedback.error
    async def feedback_error(self, ctx, error):
        if isinstance(error, commands.CommandOnCooldown):

            cooldown = datetime.fromtimestamp(int(error.retry_after)).strftime("%I:%M:%S")

            await ctx.send(f"Du hast ein Cooldown von _**{cooldown}m**_ !")

def setup(client):
    client.add_cog(FeedBack(client))


import logging
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from concurrent.futures import CancelledError as FutureCancelledError
from threading import Lock as ThreadLock
from queue import Queue as ThreadQueue
from queue import Empty

from newio import spawn
from newio.socket import socketpair

from .asyncio_executor import AsyncioExecutor

LOG = logging.getLogger(__name__)

class ExecutorFuture:
    def __init__(self, handler, task, future):
        self._handler = handler
        self._task = task
        self._fut = future
        self._is_expired = False
        future.add_done_callback(self._on_fn_done)

    def cancel(self):
        if not self._is_expired:
            LOG.debug('task %r cancel executor', self._task)
            self._fut.cancel()
            self._is_expired = True

    def state(self):
        return 'wait_executor'

    def clean(self):
        self.cancel()

    def _on_fn_done(self, fut):
        if self._is_expired:
            return
        self._is_expired = True
        result = error = None
        try:
            error = fut.exception()
            if error is None:
                result = fut.result()
        except FutureCancelledError:
            return  # ignore
        if error:
            LOG.debug('executor for task %r crashed:',
                      self._task, exc_info=error)
        else:
            LOG.debug('executor for task %r finished', self._task)
        self._handler(self._task, result, error)

class Executor:
    def __init__(self, handler, max_num_thread=None, max_num_process=None):
        self._handler = handler
        self._thread_executor = ThreadPoolExecutor(max_num_thread)
        self._process_executor = ProcessPoolExecutor(max_num_process)
        self._asyncio_executor = AsyncioExecutor()
        self._notify, self._wakeup = socketpair()
        self._queue = ThreadQueue(1024)
        self.agent_task = None
        self._is_exiting = False
        self._notify_lock = ThreadLock()

    def run_in_thread(self, task, fn, args, kwargs):
        LOG.debug('task %r run %r in thread executor', task, fn)
        fut = self._thread_executor.submit(fn, *args, **kwargs)
        return ExecutorFuture(self.handler, task, fut)

    def run_in_process(self, task, fn, args, kwargs):
        LOG.debug('task %r run %r in process executor', task, fn)
        fut = self._process_executor.submit(fn, *args, **kwargs)
        return ExecutorFuture(self.handler, task, fut)

    def run_in_asyncio(self, task, coro):
        LOG.debug('task %r run %r in asyncio executor', task, coro)
        fut = self._asyncio_executor.submit(coro)
        return ExecutorFuture(self.handler, task, fut)

    async def executor_agent(self):
        is_exiting = False
        while True:
            try:
                task, result, error = self._queue.get_nowait()
            except Empty:
                if is_exiting:
                    break
                nbytes = await self._wakeup.recv(128)
                if not nbytes or b'x' in nbytes:
                    LOG.debug('executor agent exiting')
                    is_exiting = True
                    await self._wakeup.close()
            else:
                self._handler(task, result, error)

    async def start(self):
        self.agent_task = await spawn(self.executor_agent())

    async def stop(self):
        LOG.debug('executor exiting')
        self._is_exiting = True
        with self._notify_lock:
            await self._notify.sendall(b'x')
            await self._notify.close()
        await self.agent_task.join()

    def shutdown(self, wait=True):
        self._thread_executor.shutdown(wait=wait)
        self._process_executor.shutdown(wait=wait)
        self._asyncio_executor.shutdown(wait=wait)

    def handler(self, task, result, error):
        self._queue.put_nowait((task, result, error))
        if self._is_exiting:
            return
        with self._notify_lock:
            with self._notify.blocking() as notify:
                notify.sendall(b'1')


# Gregory Rosenblatt
# 3/21/06

import renderer
import texture
from OpenGL.GL import *
from OpenGL.GLU import *

# immediate mode projections
class PerspectiveProjection:
	def __init__(self, rect, fovy=45, near=2.0, far=25.0):
		# what should the near/far values be?
		PushPerspectiveView(rect, fovy, near, far)
	def __del__(self):
		PopView()

class OrthogonalProjection:
	def __init__(self, rect, near=-1.0, far=1.0):
		PushOrthogonalView(rect, near, far)
	def __del__(self):
		PopView()

class SceneTextureSurface:	# re-render the scene when context invalidated
	def __init__(self, drawScene):
		self.drawScene = drawScene

def CopyToTexture(rect):
	x,y,w,h
	ww = texture.NextPowerOfTwo(w)
	hh = texture.NextPowerOfTwo(h)
	widthRatio = 1.0/ww
	heightRatio = 1.0/hh
	# create and bind tex id here
	glCopyTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA
					,x, renderer.CorrectedY(y,h), ww, hh, 0)
	glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
	glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)

def PushPerspectiveProjection(rect, fovy, near, far):
	x,y,w,h = rect
	if h == 0:
		h = 1
	glViewport(x, renderer.CorrectedY(y,h), w, h)
	glMatrixMode(GL_PROJECTION)
	glPushMatrix()
	glLoadIdentity()
	gluPerspective(fovy, w/h, near, far)
#	glFrustum(-1.0, 1.0, -1.0, 1.0, near, far)
	glMatrixMode(GL_MODELVIEW)

def PushOrthogonalProjection(rect, near, far):
	x,y,w,h = rect
	if h == 0:
		h = 1
	y = renderer.CorrectedY(y,h)
	glViewport(x, y, w, h)
	glMatrixMode(GL_PROJECTION)
	glPushMatrix()
	glLoadIdentity()
	glOrtho(x, x+w, y+h, y, near, far)
	glMatrixMode(GL_MODELVIEW)

def PopProjection():
	w,h = renderer.screen.get_size()
	if h == 0:
		h = 1
	glViewport(0,0,w,h)
	glMatrixMode(GL_PROJECTION)
	glPopMatrix()
	glMatrixMode(GL_MODELVIEW)