###################################
# SFCN_MangroveMarsh_Tables_Figures.py
###################################
# Description:  Routine to Summarize Mangrove Marsh Monitoring data including output tables and figures for annual reporting.

# Code performs the following routines:
# 1) Summaries the Event Average Distance from Ground Truth values.  This includes Averge, Standard Error, Confidence Interval for defined %, and Maximum and Minimum values
# by event/segment replicating the 'Mangrove-Marsh Ecotone Monitoring' SOP8-1 summary table. Confidence Intervals are defined using a Student T Distribution with Student T defiend as: np.abs(t.ppf((1 - confidence) / 2, dof)).

# 2) Summarize via a CrossTab/Pivot Table the Absolute Vegetation by Region, Location Name (i.e. Point on Segment), by Community Type, and by Taxon (Scale is Point - single value).
# Routine replicates the 'Mangrove-Marsh Ecotone Monitoring' SOP8-2 summary table.

# 3) Calculates the Absolute Cover By Region, By Community, By Strata - data is from table  'tbl_MarkerData'. Output replicates the 'Mangrove-Marsh Ecotone Monitoring' SOP8-3 summary Figure.

# Output:
# An excel spreadsheet with Tables SOP8-1 (i.e. Average Distance from Ground Truth Values), Tables SOP8-2 By region (three total) with the the Absolute Vegetation by Region, Location Name (i.e. Point on Segment),
# by Community Type, and by Taxon, and a pdf file withe Figures SOP8-3 by region (three total) with the Absolute Cover By Region, By Community, By Strata.

# Dependencies:
# Python version 3.9
# Pandas
# Scipy

# Python/Conda environment - py39
# Created by: Kirk Sherrill - Data Manager South Florida Caribbean Network (Detail) - Inventory and Monitoring Division - National Park Service
# Date Created: March 2023

#######################################
# Start of Parameters requiring set up.
#######################################
#Define Inpurt Parameters:

#Mangrove Marsh Access Database and location
inDB = r'C:\SFCN\Monitoring\Mangrove_Marsh_Ecotone\data\SFCN_Mangrove_Marsh_Ecotone_tabular_20230318.mdb'

#Confidence Interval
confidence = 0.95

#Directory Information
outputDir = r'C:\SFCN\Monitoring\Mangrove_Marsh_Ecotone\analysis\Python\2020'  #Output Directory
workspace = r'C:\SFCN\Monitoring\Periphyton\Data\HY2021\Tp\workspace'  # Workspace Folder
monitoringYear = 2020  #Monitoring Year of Mangrove Marsh data being processing

#Get Current Date
from datetime import date
dateString = date.today().strftime("%Y%m%d")

#Define Output Name for log file
outName = "MangroveMarsh_AnnualTablesFigs_" + str(monitoringYear) + "_" + dateString  # Name given to the exported pre-processed
outPDF = outputDir + "\\" + outName + ".pdf"

#Workspace Location
workspace = r'C:\SFCN\Monitoring\Mangrove_Marsh_Ecotone\analysis\Python\workspace'

#Logifile name
logFileName = workspace + "\\" + outName + "_logfile.txt"

#######################################
## Below are paths which are hard coded:
#######################################
#Import Required Libraries
import pandas as pd
import sys
from datetime import date

import os

import traceback
import numpy as np
from scipy.stats import t

import matplotlib as mp
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages #Imort PdfPages from MatplotLib

import pyodbc
pyodbc.pooling = False  #So you can close the pydobx connection

##################################

##################################
# Checking for directories and create Logfile
##################################
if os.path.exists(workspace):
    pass
else:
    os.makedirs(workspace)

# Check for logfile
if os.path.exists(logFileName):
    pass
else:
    logFile = open(logFileName, "w")  # Creating index file if it doesn't exist
    logFile.close()
#################################################

# Function to Get the Date/Time
def timeFun():
    from datetime import datetime
    b = datetime.now()
    messageTime = b.isoformat()
    return messageTime

def main():
    try:

        ########################
        #Functions for Table 8-1
        ########################
        #Pull Data from Database for table 'tbl_MarkerData'
        outVal = defineRecords_MarkerData()
        if outVal[0].lower() != "success function":
            print("WARNING - Function defineRecords_MarkerData - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function defineRecords_MarkerData")
            outDF = outVal[1]

        #Summarize Data from tbl_MarkerData into format for Table 8-1 in SFCN Mangrove Marsh SOP.
        outVal = SummarizeFigure8_1(outDF)
        if outVal[0].lower() != "success function":
            print("WARNING - Function SummarizeFigure8_1 - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function SummarizeFigure8_1")

        ########################
        #Functions for Table 8-2  - Absolute Cover Species Data by Transect and Point By Region
        ########################

        #Summarize via a CrossTab/Pivot Table the Absolute Vegetation by Location Name (i.e. Point on Segment), by Community Type and Vegetation Type (Scale is Point - single value)
        outVal = defineRecords_VegCoverByPointAbsolute()
        if outVal[0].lower() != "success function":
            print("WARNING - Function defineRecords_VegCoverByPointAbsolute - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function defineRecords_VegCoverByPointAbsolute")

        messageTime = timeFun()
        scriptMsg = "Successfully Finished Processing - SFCN_MangroveMash_Tables_Figures - " + messageTime
        print(scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")
        logFile.close()

        ########################
        # Functions for Figure 8-3  - Calculate the Absolute Cover By Region, By Community, By Strata - data is from table  'tbl_MarkerData'
        ########################

        #Pull Stratum Cover Data by point in 'tbl_MarkerData'
        outVal = defineRecords_CoverByStratum()
        if outVal[0].lower() != "success function":
            print("WARNING - Function defineRecords_CoverByStratum - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function defineRecords_CoverByStratum")
            outDF = outVal[1]

        #Figures for Marker Point Stratum By Region - Stacked Top Marsh, Botom Mangrove
        outVal = figure_CoverByStratum(outDF)
        if outVal.lower() != "success function":
            print("WARNING - Function figure_CoverByStratum - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function figure_CoverByStratum")

        messageTime = timeFun()
        scriptMsg = "Successfully Finished Processing - SFCN_MangroveMash_Tables_Figures - " + messageTime
        print(scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")
        logFile.close()

    except:

        messageTime = timeFun()
        scriptMsg = "WARNING Script Failed - " + messageTime
        print (scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")
        traceback.print_exc(file=sys.stdout)
        logFile.close()

#Function to define the Student’s t distribution
def defineStudentT(dof):
    try:

        x = np.abs(t.ppf((1 - confidence) / 2, dof))

        return x
    except:
        scriptMsg = "Exiting Error - defineStudentT"
        print(scriptMsg)
        traceback.print_exc(file=sys.stdout)
        return "Failed"

#Summarize Mangrove Marsh Ecotone Values - Average Distance, Standard Error, Lower 95% Confidence Limit, Upper 95% Confidence Limit, Max and Min Values
#Output DataFrame with the summary values by Segment
def SummarizeFigure8_1(inDF):
    try:

        # Define Average Distance Meters
        out_GroupByMean = inDF.groupby(['Event_Group_ID', 'Region', 'Segment']).mean()
        outDf_GroupByMean = out_GroupByMean.reset_index()
        # Rename
        outDf_GroupByMean.rename(columns={'Distance': 'AverageDist_M'}, inplace=True)

        # Create output DataFrame - will be joining to this
        outDf_8pt1 = outDf_GroupByMean[['Event_Group_ID', 'Region', 'Segment', 'AverageDist_M']]

        # Define Standard Error
        out_GroupBySE = inDF.groupby(['Event_Group_ID', 'Segment']).sem()
        outDf_GroupBySE = out_GroupBySE.reset_index()
        # Rename
        outDf_GroupBySE.rename(columns={'Distance': 'StandardError'}, inplace=True)

        # Define the Event_Group_ID, Event_ID, Site_ID and Visit Type fields via a join on 'Site_Name and Site ID fields
        outDf_8pt1_j1 = pd.merge(outDf_8pt1, outDf_GroupBySE, how='inner', left_on=('Event_Group_ID', 'Segment'), right_on=('Event_Group_ID', 'Segment'), suffixes=(None, "_Right"))

        # Delete fields 'Segment_Right' and Assesment
        outDf_8pt1_j1.drop(columns=['Assessment'], inplace=True)

        # Add a Count Field
        out_GroupByCount = inDF.groupby(['Event_Group_ID', 'Segment'])['Distance'].count()
        outDf_GroupByCount = out_GroupByCount.reset_index()
        #Rename
        outDf_GroupByCount.rename(columns={'Distance': 'RecCount'}, inplace=True)

        # Join Count to Mean and Standard Error
        outDf_8pt1_j2 = pd.merge(outDf_8pt1_j1, outDf_GroupByCount, how='inner', left_on=('Event_Group_ID', 'Segment'), right_on=('Event_Group_ID', 'Segment'), suffixes=(None, "_Right"))

        #Create Degree of Freedom field (N-1)
        outDf_8pt1_j2['DOF'] = outDf_8pt1_j2['RecCount'] - 1

        ##################################
        #Calculate the Confidence Interval
        ##################################
        outVal = calc_CI(outDf_8pt1_j2, confidence)
        if outVal[0].lower() != "success function":
            print("WARNING - Function calc_CI95 - Failed - Exiting Script")
            exit()
        else:
            print("Success - Function calc_CI95")
            outDF2 = outVal[1]

        # Calculate the Minimum Difference
        out_GroupByMin = inDF.groupby(['Event_Group_ID', 'Segment'])['Distance'].min()
        outDf_GroupByMin = out_GroupByMin.reset_index()
        outDf_GroupByMin.rename(columns={'Distance': 'MinDifference'}, inplace=True)
        # Join with working Data Frame
        outDf_8pt1_j3= pd.merge(outDF2, outDf_GroupByMin, how='inner', left_on=('Event_Group_ID', 'Segment'), right_on=('Event_Group_ID', 'Segment'), suffixes=(None, "_Right"))
        del outDF2

        #Calculate the Maximum Difference
        out_GroupByMax = inDF.groupby(['Event_Group_ID', 'Segment'])['Distance'].max()
        outDf_GroupByMax = out_GroupByMax.reset_index()
        outDf_GroupByMax.rename(columns={'Distance': 'MaxDifference'}, inplace=True)
        # Join with working Data Frame
        outDf_8pt1_j4 = pd.merge(outDf_8pt1_j3, outDf_GroupByMax, how='inner', left_on=('Event_Group_ID', 'Segment'), right_on=('Event_Group_ID', 'Segment'), suffixes=(None, "_Right"))
        del outDf_8pt1_j3

        #Sort by Numeric Segment Number
        outDf_8pt1_j4['SortField'] = outDf_8pt1_j4['Segment'].str.replace('Segment_', '')
        #Convert Sort Field to Integer
        outDf_8pt1_j4['SortField'] = pd.to_numeric(outDf_8pt1_j4['SortField'], errors='coerce', downcast='integer')
        #Sort on Segment Number
        outDf_8pt1_j4.sort_values(by=['SortField'], inplace = True)
        #Set index Field to Sort Field
        outDf_8pt1_j4.set_index('SortField', inplace=True)
        #Drop 'DOF' and t_crit fields
        outDf_8pt1_j4.drop(columns=['DOF','t_crit'], inplace=True)

        #Export Table to Excel
        dateString = date.today().strftime("%Y%m%d")
        # Define Export .csv file
        outFull = outputDir + "\MangroveMarsh_Export_" + dateString + ".xlsx"

        #Export
        outDf_8pt1_j4.to_excel(outFull, sheet_name = 'SOP8-1', index=False)

        messageTime = timeFun()
        scriptMsg=  ("Successfully Exported Table 8-1 to: " + outFull + " - " + messageTime)
        print(scriptMsg)

        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")
        logFile.close()

        return "success function", outDf_8pt1_j4

    except:
        messageTime = timeFun()
        print("Error on SummarizeFigure8_1 Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'SummarizeFigure8_1'"

#Calculate the Confidence Interval Upper 95% and Lower 95% usinga Sutdents T Distribution
#Student T Distribution is defined as t_crit = np.abs(t.ppf((1-confidence)/2,dof))
#CI Upper and lower is: (Mean - Standard Deviation *t_crit/np.sqrt(n))  and  (Mean + Standard Deviation *t_crit/np.sqrt(n))   where n = number of records
def calc_CI(inDF, confidence):

    try:
        #Calculate the Student T's Distribution
        inDF['t_crit'] = inDF.apply(lambda x: defineStudentT(x['DOF']), axis=1)

        #inDF['t_crit'] = np.abs(t.ppf((1 - confidence) / 2, ['dof']))
        #Convert CI to Str
        confidenceStr = str(confidence)

        #Calculate Lower Confidence Interval
        inDF['LowerCI_' + str(confidenceStr)] = inDF['AverageDist_M'] - inDF['StandardError'] * inDF['t_crit'] / np.sqrt(inDF['RecCount'])

        # Calculate Upper Confidence Interval
        inDF['UpperCI_' + str(confidenceStr)] = inDF['AverageDist_M'] + inDF['StandardError'] * inDF['t_crit'] / np.sqrt(inDF['RecCount'])

        return "success function", inDF

    except:
        messageTime = timeFun()
        print("Error on calc_CI Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'calc_CI95'"

# Summarize via a CrossTab/Pivot Table the Absolute Cover By Region, Community, Strata and Taxon across point locations
#Export By Region
def defineRecords_VegCoverByPointAbsolute():
    try:

        dateString = date.today().strftime("%Y%m%d")
        # Process By Region
        regionlList = ['Turner River', 'Shark Slough', 'Taylor Slough']
        for count, region in enumerate(regionlList):

            inQuery = "TRANSFORM Sum(IIf([VegetationType]='Tree' And [CommunityType]='Mangrove',([MangroveSide_Cover_Overall])*([MangroveSide_Cover_Tree]/100)*([PercentCover]/100),IIf([VegetationType]='Shrub' And"\
                " [CommunityType]='Mangrove',([MangroveSide_Cover_Overall])*([MangroveSide_Cover_Shrub]/100)*([PercentCover]/100),IIf([VegetationType]='Herb' And"\
                " [CommunityType]='Mangrove',([MangroveSide_Cover_Overall])*([MangroveSide_Cover_Herb]/100)*([PercentCover]/100),IIf([VegetationType]='Tree' And"\
                " [CommunityType]='Marsh',([MarshSide_Cover_Overall])*([MarshSide_Cover_Tree]/100)*([PercentCover]/100),IIf([VegetationType]='Shrub' And"\
                " [CommunityType]='Marsh',([MarshSide_Cover_Overall])*([MarshSide_Cover_Shrub]/100)*([PercentCover]/100),IIf([VegetationType]='Herb' And"\
                " [CommunityType]='Marsh',([MarshSide_Cover_Overall])*([MarshSide_Cover_Herb]/100)*([PercentCover]/100),-999))))))) AS AbsolutePercCover"\
                " SELECT tbl_Locations.Region, tbl_MarkerData_Vegetation.CommunityType, tbl_MarkerData_Vegetation.VegetationType, tlu_Vegetation.ScientificName"\
                " FROM tbl_Locations INNER JOIN (((tbl_Event_Group INNER JOIN tbl_Events ON (tbl_Event_Group.Event_Group_ID = tbl_Events.Event_Group_ID) AND"\
                " (tbl_Event_Group.Event_Group_ID = tbl_Events.Event_Group_ID)) INNER JOIN tbl_MarkerData ON (tbl_Events.Event_ID = tbl_MarkerData.Event_ID) AND"\
                " (tbl_Events.Event_ID = tbl_MarkerData.Event_ID)) INNER JOIN (tbl_MarkerData_Vegetation LEFT JOIN tlu_Vegetation ON tbl_MarkerData_Vegetation.SpeciesCode = tlu_Vegetation.SpeciesCode)"\
                " ON (tbl_MarkerData.Point_ID = tbl_MarkerData_Vegetation.Point_ID) AND (tbl_MarkerData.Point_ID = tbl_MarkerData_Vegetation.Point_ID)) ON tbl_Locations.Location_ID"\
                " = tbl_Events.Location_ID WHERE ((Not (tbl_MarkerData_Vegetation.PercentCover) Is Null) AND ((tbl_Events.Event_Type)='Marker Visit') AND ((tbl_Locations.Region)= '" + region + "'))"\
                " GROUP BY tbl_Locations.Region, tbl_MarkerData_Vegetation.CommunityType, tbl_MarkerData_Vegetation.VegetationType, tlu_Vegetation.ScientificName"\
                " ORDER BY tbl_MarkerData_Vegetation.CommunityType, tbl_MarkerData_Vegetation.VegetationType, tlu_Vegetation.ScientificName"\
                " PIVOT tbl_Locations.Location_Name;"

            outVal = connect_to_AcessDB(inQuery, inDB)
            if outVal[0].lower() != "success function":
                messageTime = timeFun()
                print("WARNING - Function defineRecords_VegCoverBySegment - " + messageTime + " - Failed - Exiting Script")
                exit()
            else:

                outDF = outVal[1]
                messageTime = timeFun()
                scriptMsg = "Success:  defineRecords_VegCoverBySegment" + messageTime
                print(scriptMsg)

                #Define Export .csv file
                outFull = outputDir + "\MangroveMarsh_Export_" + dateString + ".xlsx"

                #Append DataFrame to existing excel file
                with pd.ExcelWriter(outFull, mode='a', engine="openpyxl") as writer:
                    outDF.to_excel(writer, sheet_name='SOP8-2-AbsCov-' + region, index=False)

                messageTime = timeFun()
                scriptMsg = ("Successfully Exported Table 8-2-AbsCov - " + region + " - to: " + outFull + " - " + messageTime)
                print(scriptMsg)

                logFile = open(logFileName, "a")
                logFile.write(scriptMsg + "\n")
                logFile.close()

        return "success function", outDF

    except:
        messageTime = timeFun()
        print("Error on defineRecords_VegCoverByPointAbsolute Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'defineRecords_VegCoverByPointAbsolute'"

#Create  Figures - Absolute Cover By Region, By Community, By Strata
def figure_CoverByStratum(inDF):
    try:

        #Open PDF to be copied to
        pdf = PdfPages(outPDF)

        # Process By Region
        regionlList = ['Turner River', 'Shark Slough', 'Taylor Slough']
        for count, region in enumerate(regionlList):

            #Subset By Region
            outDFSub = inDF[inDF['Region'] == region]

            ###########################
            #Marsh DataFrame and Figure
            ###########################
            #Subset to marshDF fields
            marshDF = outDFSub.loc[:,('Location_Name', 'MarshSide_Cover_Overall', 'AbsCover_Marsh_Tree', 'AbsCover_Marsh_Shrub','AbsCover_Marsh_Herb')]
            #Rename Fields
            marshDF.rename(columns={"AbsCover_Marsh_Tree": "Tree", "AbsCover_Marsh_Shrub": "Shrub", "AbsCover_Marsh_Herb": "Herb"}, inplace=True)

            #Set Index
            marshDF.set_index('Location_Name', inplace=True)

            #Subset to Mangrove fields
            mangroveDF = outDFSub.loc[:, ('Location_Name', 'MangroveSide_Cover_Overall', 'AbsCover_Mangrove_Tree','AbsCover_Mangrove_Shrub', 'AbsCover_Mangrove_Herb')]
            # Rename Fields
            mangroveDF.rename(columns={"AbsCover_Mangrove_Tree": "Tree", "AbsCover_Mangrove_Shrub": "Shrub", "AbsCover_Mangrove_Herb": "Herb"}, inplace=True)

            #Set Index
            mangroveDF.set_index('Location_Name', inplace=True)

            ####################
            #Create the Figures:
            ####################
            plt.figure(figsize=(8, 6))
            ax1 = plt.subplot(2, 1, 1)
            marshDF.plot.bar(stacked=True, title="Marsh Side - " + region, xlabel="Marker Points", ylabel="Absolute Percent Cover (%)", color={'Shrub': 'red', 'Herb': 'turquoise', 'Tree': 'orange'}, ax=ax1)
            lgd = plt.legend(['Tree', 'Shrub', 'Herb'], loc='center left', bbox_to_anchor=(1, 0.5))
            plt.grid(axis='y')
            plt.ylim(0, 100)
            plt.tight_layout(pad=0.4)

            ax2 = plt.subplot(2, 1, 2)
            mangroveDF.plot.bar(stacked=True, title="Mangrove Side - " + region, xlabel="Marker Points", ylabel="Absolute Percent Cover (%)", color={'Shrub': 'red', 'Herb': 'turquoise', 'Tree': 'orange'}, ax=ax2)
            lgd = plt.legend(['Tree', 'Shrub', 'Herb'], loc='center left', bbox_to_anchor=(1, 0.5))
            plt.grid(axis='y')
            plt.ylim(0, 100)
            plt.tight_layout(pad=0.4)

            figure = mp.pyplot.gcf()
            pdf.savefig(figure)

            messageTime = timeFun()
            scriptMsg = "Successfully Exported Figures Region:" + region + " - " + messageTime
            print(scriptMsg)
            logFile = open(logFileName, "a")
            logFile.write(scriptMsg + "\n")
            logFile.close()

        pdf.close()

        messageTime = timeFun()
        scriptMsg = "Success:  figure_CoverByStratum" + messageTime
        print(scriptMsg)

        return "success function"

    except:
        messageTime = timeFun()
        print("Error on figure_CoverByStratum Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'defineRecords_CoverByStratum'"

#Calculate Absolute Cover By Stratum and Community type in table 'tbl_MarkerData'
def defineRecords_CoverByStratum():
    try:

        inQuery = "SELECT tbl_Locations.Location_ID, tbl_Locations.Order_ID,  tbl_Locations.Region, tbl_Locations.Location_Name,  tbl_Events.Event_ID, tbl_Event_Group.Start_Date, tbl_MarkerData.MangroveSide_Cover_Overall,"\
                " tbl_MarkerData.MangroveSide_Cover_Tree, tbl_MarkerData.MangroveSide_Cover_Shrub, tbl_MarkerData.MangroveSide_Cover_Herb, [MangroveSide_Cover_Overall]*([MangroveSide_Cover_Tree]/100)"\
                " AS AbsCover_Mangrove_Tree, [MangroveSide_Cover_Overall]*([MangroveSide_Cover_Shrub]/100) AS AbsCover_Mangrove_Shrub, [MangroveSide_Cover_Overall]*([MangroveSide_Cover_Herb]/100)"\
                " AS AbsCover_Mangrove_Herb, tbl_MarkerData.MarshSide_Cover_Overall, tbl_MarkerData.MarshSide_Cover_Tree, tbl_MarkerData.MarshSide_Cover_Shrub, tbl_MarkerData.MarshSide_Cover_Herb,"\
                " [MarshSide_Cover_Overall]*([MarshSide_Cover_Tree]/100) AS AbsCover_Marsh_Tree, [MarshSide_Cover_Overall]*([MarshSide_Cover_Shrub]/100) AS AbsCover_Marsh_Shrub,"\
                " [MarshSide_Cover_Overall]*([MarshSide_Cover_Herb]/100) AS AbsCover_Marsh_Herb"\
                " FROM tbl_Locations INNER JOIN ((tbl_Event_Group INNER JOIN tbl_Events ON (tbl_Event_Group.Event_Group_ID = tbl_Events.Event_Group_ID) AND (tbl_Event_Group.Event_Group_ID"\
                " = tbl_Events.Event_Group_ID)) INNER JOIN tbl_MarkerData ON (tbl_Events.Event_ID = tbl_MarkerData.Event_ID) AND (tbl_Events.Event_ID = tbl_MarkerData.Event_ID))"\
                " ON tbl_Locations.Location_ID = tbl_Events.Location_ID WHERE (((tbl_Events.Event_Type)='Marker Visit')) ORDER BY tbl_Locations.Order_ID, tbl_Locations.Location_Name,"\
                " tbl_Event_Group.Start_Date;"

        outVal = connect_to_AcessDB(inQuery, inDB)
        if outVal[0].lower() != "success function":
            messageTime = timeFun()
            print("WARNING - Function defineRecords_CoverByStratum - " + messageTime + " - Failed - Exiting Script")
            exit()
        else:

            outDF = outVal[1]
            messageTime = timeFun()
            scriptMsg = "Success:  defineRecords_CoverByStratum" + messageTime
            print(scriptMsg)

            return "success function", outDF

    except:
        messageTime = timeFun()
        print("Error on defineRecords_CoverByStratum Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'defineRecords_CoverByStratum'"

#Extract Mangrove Marsh Distance Records table 'tbl_MarkerData' where Event Type = 'Marker Visit'
def defineRecords_MarkerData():
    try:
        inQuery = "SELECT tbl_Event_Group.Event_Group_ID, tbl_Event_Group.Event_Group_Name, tbl_Event_Group.Start_Date, tbl_Event_Group.End_Date, tbl_Event_Group.Assessment, tbl_Events.Event_Type,"\
                " tbl_Events.Location_ID, tbl_Locations.Region,tbl_Locations.Segment, tbl_Locations.Location_Name, tbl_MarkerData.Distance, tbl_MarkerData.Method"\
                " FROM tbl_Locations INNER JOIN ((tbl_Event_Group INNER JOIN tbl_Events ON (tbl_Event_Group.Event_Group_ID = tbl_Events.Event_Group_ID) AND (tbl_Event_Group.Event_Group_ID = tbl_Events.Event_Group_ID))"\
                " INNER JOIN tbl_MarkerData ON (tbl_Events.Event_ID = tbl_MarkerData.Event_ID) AND (tbl_Events.Event_ID = tbl_MarkerData.Event_ID)) ON tbl_Locations.Location_ID = tbl_Events.Location_ID"\
                " WHERE tbl_Events.Event_Type = 'Marker Visit' ORDER BY tbl_Locations.Segment, tbl_Locations.Location_Name, tbl_Events.Event_Type;"\

        outVal = connect_to_AcessDB(inQuery, inDB)
        if outVal[0].lower() != "success function":
            messageTime = timeFun()
            print("WARNING - Function defineRecords_MarkerData - " + messageTime + " - Failed - Exiting Script")
            exit()
        else:

            outDF = outVal[1]
            messageTime = timeFun()
            scriptMsg = "Success:  defineRecords_MarkerData" + messageTime
            print(scriptMsg)

            return "success function", outDF

    except:
        messageTime = timeFun()
        print("Error on defineRecords_MarkderData Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'defineRecords'"

#Connect to Access DB and perform defined query - return query in a dataframe
def connect_to_AcessDB(query, inDB):

    try:
        connStr = (r"DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=" + inDB + ";")
        cnxn = pyodbc.connect(connStr)
        queryDf = pd.read_sql(query, cnxn)
        cnxn.close()

        return "success function", queryDf

    except:
        messageTime = timeFun()
        scriptMsg = "Error function:  connect_to_AcessDB - " +  messageTime
        print(scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")

        traceback.print_exc(file=sys.stdout)
        logFile.close()
        return "failed function"

if __name__ == '__main__':

    # Write parameters to log file ---------------------------------------------
    ##################################
    # Checking for working directories
    ##################################

    if os.path.exists(workspace):
        pass
    else:
        os.makedirs(workspace)

    #Check for logfile

    if os.path.exists(logFileName):
        pass
    else:
        logFile = open(logFileName, "w")    #Creating index file if it doesn't exist
        logFile.close()

    # Analyses routine ---------------------------------------------------------
    main()


# -*- encoding: utf-8 -*-
'''
@File    :   ApiTestGui.py
@Time    :   2020/02/02 16:05:17
@Author  :   chenjiusi
'''

from tkinter import *
from tkinter import ttk
from tkinter import filedialog
import hashlib
import time
import json
from BearSki.utils.hartool import HarTool
import sys
import time
import threading
LOG_LINE_NUM = 0

class ApiTest_GUI():
    def __init__(self,init_window_name):

        self.init_window_name = init_window_name
        self.row_1=1
        self.row_2=2
        self.row_3=3
        self.row_4=7
        self.row_5=8

        self.row_rowspan_1=2
        self.row_rowspan_2=2

        self.column_1=0
        self.column_2=4
        self.column_row1_2=1
        self.column_row1_3=7
        self.column_row1_4=8
        self.column_row1_5=9

        self.row3_columnspan_1=4
        self.row3_columnspan_2=6
        self.column_2_columnspan=5

        self.pdx=5
        self.pady=5

        self.text_height=20

        self.HarTool= None

    #设置窗口
    def set_init_window(self):
        self.init_window_name.title("BearSki 用例生成工具 V 0.0.1")   #窗口名
        self.init_window_name.geometry('960x680')  #窗口大小设置
        self.init_window_name["bg"] = "#F5F5F5"
        self.init_window_name.grid_columnconfigure(0,weight=1)
        self.init_window_name.grid_rowconfigure(0,weight=1)

        #row 1
        self.init_data_label = Label(self.init_window_name, text="Har文件地址：")
        self.init_data_label.grid(row=self.row_1, column=self.column_1,sticky=N+W,padx=self.pdx,pady=self.pady)
        self.har_path_input=Entry(self.init_window_name,width=25)
        self.har_path_input.grid(row=self.row_1,column=self.column_row1_2,sticky=N+E,padx=self.pdx,pady=self.pady)

        self.get_har_text = ttk.Button(self.init_window_name, text="读取文件",command=self.openfiles)
        self.get_har_text.grid(row=self.row_1, column=self.column_2,sticky=N+W,padx=self.pdx,pady=self.pady)
        self.har_path_input2=Entry(self.init_window_name,width=15)
        self.har_path_input2.grid(row=self.row_1,column=self.column_row1_3,sticky=N+E,padx=self.pdx,pady=self.pady)

        self.put_one_request = ttk.Button(self.init_window_name, text="生成用例",command=self.createOneCase)  # 调用内部方法  加()为直接调用
        self.put_one_request.grid(row=self.row_1, column=self.column_row1_4,sticky=N+W,padx=self.pdx,pady=self.pady)
        self.put_all_request = ttk.Button(self.init_window_name, text="批量生成用例",command=self.createAllCase)  # 调用内部方法加()为直接调用
        self.put_all_request.grid(row=self.row_1, column=self.column_row1_5,sticky=N+W,padx=self.pdx,pady=self.pady)
        # self.put_all_request = ttk.Button(self.init_window_name, text="测试退出",command=self.win_exit)  # 调用内部方法加()为直接调用
        # self.put_all_request.grid(row=self.row_1, column=self.column_row1_5,sticky=N+W,padx=self.pdx,pady=self.pady)

        #row 2
        self.result_data_label = Label(self.init_window_name, text="请求目录:")
        self.result_data_label.grid(row=self.row_2, column=self.column_1, columnspan=self.row3_columnspan_1,sticky=N+W,padx=self.pdx,pady=self.pady)
        self.result_data_label1 = Label(self.init_window_name, text="发送的消息请求信息：")
        self.result_data_label1.grid(row=self.row_2, column=self.column_2,columnspan=self.row3_columnspan_2,sticky=N+W,padx=self.pdx,pady=self.pady)

        #row 3
        # # tree
        self.init_tree = ttk.Treeview(self.init_window_name, show="headings", columns=('col1'))
        self.init_tree.heading('col1', text='modelname')
        self.init_tree.column('col1', width=350, anchor='nw')
        self.init_tree.grid(row=self.row_3, column=self.column_1, rowspan=self.row_rowspan_1, columnspan=self.row3_columnspan_1,sticky=N+W,padx=self.pdx)

        self.result_data_Text = Text(self.init_window_name,height=self.text_height)  #处理结果展示
        self.result_data_Text.grid(row=self.row_3, column=self.column_2, rowspan=self.row_rowspan_1, columnspan=self.row3_columnspan_2,sticky=N+W,padx=self.pdx)

        #row 4
        self.log_label = Label(self.init_window_name, text="日志:")
        self.log_label.grid(row=self.row_4, column=self.column_1,sticky=N+W,padx=self.pdx)
        self.result_data_label1 = Label(self.init_window_name, text="获取的返回值:")
        self.result_data_label1.grid(row=self.row_4, column=self.column_2,sticky=N+W,padx=self.pdx)

        #row 5
        self.log_data_Text = Text(self.init_window_name,height=self.text_height)  # 日志框
        self.log_data_Text.grid(row=self.row_5, column=self.column_1, columnspan=self.row3_columnspan_1,rowspan=self.row_rowspan_2,sticky=N+W,padx=self.pdx)
        self.result_data_Text1 = Text(self.init_window_name,height=self.text_height)#处理结果展示
        self.result_data_Text1.grid(row=self.row_5, column=self.column_2, rowspan=self.row_rowspan_2, columnspan=self.row3_columnspan_2,sticky=N+W,padx=self.pdx)

        #事件件绑定
        self.init_tree.bind("<Double-1>", self.onDBClick)
        self.init_window_name.bind("<Control-d>", self.win_exit)

    def onDBClick(self,event):
        try:
            item = self.init_tree.selection()[0]
            modelname=self.init_tree.item(item, "values")
            print ("you clicked on name is :{0} ".format(modelname[0]))
            self.getOneMessage(modelname[0])
            self.write_log_to_Text("INFO: 查看接口 【{0}】的信息".format(modelname[0]))
            self.har_path_input2.insert(0,modelname[0])
        except Exception as e:
            self.write_log_to_Text("ERROR: {0}".format(e))

    def openfiles(self):
      try:
            harfilename = filedialog.askopenfilename(title='打开Har文件', filetypes=[('har', '*.har'), ('All Files', '*')],initialdir='./')
            self.har_path_input.insert(0,harfilename)
            self.write_log_to_Text("INFO: 打开文件: {0}".format(harfilename))
            self.HarTool=HarTool(harfilename)
            #清空表格
            table_items=self.init_tree.get_children()
            for item in table_items:
                self.init_tree.delete(item)
            i=0
            for name in self.HarTool.harfile.keys():
                self.init_tree.insert('',i,values=(name))
      except Exception as e:
            self.write_log_to_Text("ERROR: {0}".format(e))

    def getOneMessage(self,name):
        try:
            self.write_log_to_Text("INFO: 获取一条返回值模型信息")
            result,res=self.HarTool.getMessage(name)
            self.result_data_Text.delete(1.0,END)
            self.result_data_Text.insert(1.0,json.dumps(result, sort_keys=True, indent=4, separators=(',', ': '),ensure_ascii=False))
            self.result_data_Text1.delete(1.0,END)
            self.result_data_Text1.insert(1.0,json.dumps(res, sort_keys=True, indent=4, separators=(',', ': '),ensure_ascii=False))

        except Exception as e:
            self.write_log_to_Text("ERROR: {0}".format(e))

    def createAllCase(self):
        try:
            self.write_log_to_Text("INFO: 开始为全部接口信息创建测试用例")
            self.HarTool.createALLCase()
            self.write_log_to_Text("INFO: 全部用例创建结束")
        except Exception as e:
            self.write_log_to_Text("ERROR: {0}".format(e))

    #获取当前时间
    def get_current_time(self):
        current_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
        return current_time

    def createOneCase(self):
        try:
            req_str=json.loads(self.result_data_Text.get(1.0,END))
            self.write_log_to_Text("INFO: 开始生成单条测试用例 {0}".format(req_str['name']))
            res_str=json.loads(self.result_data_Text1.get(1.0,END))
            self.HarTool.createTestCase(req_str,res_str)
            self.write_log_to_Text("INFO: 成功生成用例")
        except Exception as e:
            self.write_log_to_Text("ERROR: {0}".format(e))

    #日志动态打印
    def write_log_to_Text(self,logmsg):
        global LOG_LINE_NUM
        current_time = self.get_current_time()
        logmsg_in = str(current_time) +" " + str(logmsg) + "\n"      #换行
        if LOG_LINE_NUM <= 7:
            self.log_data_Text.insert(END, logmsg_in)
            LOG_LINE_NUM = LOG_LINE_NUM + 1
        else:
            self.log_data_Text.delete(1.0,2.0)
            self.log_data_Text.insert(END, logmsg_in)

    def win_exit(self,event):
        print("in exit")
        self.init_window_name.destroy()

def gui_start(mode):
    init_window = Tk()              #实例化出一个父窗口
    ZMJ_PORTAL = ApiTest_GUI(init_window)
    # 设置根窗口默认属性
    ZMJ_PORTAL.set_init_window()
    print("guitools startup ...")
    if mode == "test":
        init_window.mainloop(2)
    else:
        init_window.mainloop()
    # # ZMJ_PORTAL.initData()
    # init_window.mainloop(3)          #父窗口进入事件循环，可以理解为保持窗口运行，否则界面不展示
    print("guitools exit.")

# if __name__ == "__main__":

#     gui_start()


import seaborn as sns
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
from scipy.stats import spearmanr, rankdata,fisher_exact
from utils import *
from sklearn.metrics import roc_auc_score, roc_curve
import matplotlib
plt.style.use('ggplot')
matplotlib.rc('font', family='sans-serif')

#mira
db = 'mira'
cdr3_col = 'beta_sequences'
epitope_col = 'peptide'
categories = ['orf_name','peptide']

file = '../../Data/ImmuneCODE/mira_data_parsed_ci.csv'
df = pd.read_csv(file,dtype=object,engine='python')
df = Process_Seq(df,cdr3_col)
df.drop_duplicates(subset=[cdr3_col,epitope_col],inplace=True)
df['cell_type'] = 'CD8'
df_cd8 = df

file = '../../Data/ImmuneCODE/mira_data_parsed_cii.csv'
df = pd.read_csv(file,dtype=object,engine='python')
df = Process_Seq(df,cdr3_col)
df.drop_duplicates(subset=[cdr3_col,epitope_col],inplace=True)
df['cell_type'] = 'CD4'
df_cd4 = df

df = pd.concat([df_cd8,df_cd4])

#isb
with open('../supervised/isb_ft_pred.pkl','rb') as f:
    _,_,df_isb = pickle.load(f)
df_isb['cohort'] = 'COVID-19-ISB'
df_isb['covid+'] = None
df_isb['covid+'][df_isb['beta_sequences'].isin(df['beta_sequences'])] = 'COVID (+)'
df_isb['covid+'][~df_isb['beta_sequences'].isin(df['beta_sequences'])] = 'COVID (-)'
df_isb = pd.merge(df_isb,df,how='left')

#df_isb = df_isb[df_isb['cell_type']!='CD8']
l = 'covid+'
sns.violinplot(data=df_isb,x=l,y='pred',cut=0,order=['COVID (-)','COVID (+)'])
plt.gca().set_facecolor('white')
plt.gca().spines['left'].set_color('black')
plt.gca().spines['bottom'].set_color('black')
plt.xlabel('')
plt.ylabel('Prediction',fontsize=18)
plt.xticks(fontsize=18)
plt.gca().axhline(.90,linestyle='--',color='black')
plt.tight_layout()
plt.savefig('figures/covid_preds_isb.eps')

l = 'orf_name'
diff = []
for o in np.unique(df_isb[l].dropna()):
    df_temp = df_isb[df_isb[l]==o]
    df_temp = df_temp.groupby(['cell_type']).agg({'pred':'mean'})
    try:
        val_cd8 = df_temp.loc['CD8']['pred']
    except:
        val_cd8 = 0

    try:
        val_cd4 = df_temp.loc['CD4']['pred']
    except:
        val_cd4 = 0

    diff.append(np.abs(val_cd8-val_cd4))

order = np.unique(df_isb[l].dropna())[np.argsort(diff)]
fig,ax = plt.subplots(figsize=(5,7))
sns.violinplot(data=df_isb,x='pred',y=l,cut=0,order=order,hue='cell_type',palette=['m','g'],orient='h')
plt.gca().set_facecolor('white')
plt.gca().spines['left'].set_color('black')
plt.gca().spines['bottom'].set_color('black')
plt.xticks(rotation=90)
plt.ylabel('',fontsize=18)
plt.tight_layout()
plt.xlabel('Prediction')
plt.legend(loc='lower right')
labels = [item.get_text() for item in plt.gca().get_yticklabels()]
labels_new = []
for l in labels:
    labels_new.append(l.replace(' ', '\n'))
plt.gca().set_yticklabels(labels_new)
plt.tight_layout()
plt.savefig('figures_ant/covid_preds_isb_cd84.eps')

#niaid
with open('../supervised/niaid_ft_pred.pkl','rb') as f:
    _,_,df_niaid = pickle.load(f)
df_niaid['cohort'] = 'COVID-19-NIH/NIAID'
df_niaid['covid+'] = None
df_niaid['covid+'][df_niaid['beta_sequences'].isin(df['beta_sequences'])] = 'COVID (+)'
df_niaid['covid+'][~df_niaid['beta_sequences'].isin(df['beta_sequences'])] = 'COVID (-)'
df_niaid = pd.merge(df_niaid,df,how='left')

l = 'covid+'
sns.violinplot(data=df_niaid,x=l,y='pred',cut=0,order=['COVID (-)','COVID (+)'])
plt.gca().set_facecolor('white')
plt.gca().spines['left'].set_color('black')
plt.gca().spines['bottom'].set_color('black')
plt.tight_layout()
plt.xlabel('')
plt.ylabel('Prediction',fontsize=18)
plt.xticks(fontsize=18)
plt.gca().axhline(.90,linestyle='--',color='black')
plt.savefig('figures/covid_preds_niaid.eps')

l = 'orf_name'
fig,ax = plt.subplots(figsize=(5,7))
sns.violinplot(data=df_niaid,x='pred',y=l,cut=0,order=order,hue='cell_type',palette=['m','g'],orient='h')
plt.gca().set_facecolor('white')
plt.gca().spines['left'].set_color('black')
plt.gca().spines['bottom'].set_color('black')
plt.xticks(rotation=90)
plt.ylabel('',fontsize=18)
plt.tight_layout()
plt.xlabel('Prediction')
plt.legend(loc='lower right')
labels = [item.get_text() for item in plt.gca().get_yticklabels()]
labels_new = []
for l in labels:
    labels_new.append(l.replace(' ', '\n'))
plt.gca().set_yticklabels(labels_new)
plt.tight_layout()
plt.savefig('figures_ant/covid_preds_niaid_cd84.eps')

df_isb['call'] = df_isb['pred'] > 0.90
df_isb['count'] = 1
df_isb_cont = pd.pivot_table(data=df_isb,index='covid+',columns=['call'],values=['count'],aggfunc='sum')
df_isb_cont = df_isb_cont.loc[['COVID (-)','COVID (+)']]
OR, p_val = fisher_exact(df_isb_cont)

df_niaid['call'] = df_niaid['pred'] > 0.90
df_niaid['count'] = 1
df_niaid_cont = pd.pivot_table(data=df_niaid,index='covid+',columns=['call'],values=['count'],aggfunc='sum')
df_niaid_cont = df_niaid_cont.loc[['COVID (-)','COVID (+)']]
OR, p_val = fisher_exact(df_niaid_cont)

df_isb_covid = df_isb[df_isb['covid+']=='COVID (+)']
df_isb_covid.sort_values(by='pred',ascending=False,inplace=True)
df_isb_covid.to_csv('isb_covid_seq.csv',index=False)

df_niaid_covid = df_niaid[df_niaid['covid+']=='COVID (+)']
df_niaid_covid.sort_values(by='pred',ascending=False,inplace=True)
df_niaid_covid.to_csv('niaid_covid_seq.csv',index=False)


import os
import time

import settings
import kubectl
import clickhouse
import util

from testflows.core import fail, Given, Then

current_dir = os.path.dirname(os.path.abspath(__file__))
operator_label = "-l app=clickhouse-operator"

def get_full_path(test_file):
    return os.path.join(current_dir, f"{test_file}")

def set_operator_version(version, ns=settings.operator_namespace, timeout=60):
    operator_image = f"{settings.operator_docker_repo}:{version}"
    metrics_exporter_image = f"{settings.metrics_exporter_docker_repo}:{version}"
    kubectl.launch(f"set image deployment.v1.apps/clickhouse-operator clickhouse-operator={operator_image}", ns=ns)
    kubectl.launch(f"set image deployment.v1.apps/clickhouse-operator metrics-exporter={metrics_exporter_image}", ns=ns)
    kubectl.launch("rollout status deployment.v1.apps/clickhouse-operator", ns=ns, timeout=timeout)
    if kubectl.get_count("pod", ns=ns, label=operator_label) == 0:
        fail("invalid clickhouse-operator pod count")

def set_metrics_exporter_version(version, ns=settings.operator_namespace):
    kubectl.launch(f"set image deployment.v1.apps/clickhouse-operator metrics-exporter=radondb/metrics-exporter:{version}", ns=ns)
    kubectl.launch("rollout status deployment.v1.apps/clickhouse-operator", ns=ns)

def restart_operator(ns=settings.operator_namespace, timeout=600):
    pod_name = kubectl.get("pod", name="", ns=ns, label=operator_label)["items"][0]["metadata"]["name"]
    kubectl.launch(f"delete pod {pod_name}", ns=ns, timeout=timeout)
    kubectl.wait_object("pod", name="", ns=ns, label=operator_label)
    pod_name = kubectl.get("pod", name="", ns=ns, label=operator_label)["items"][0]["metadata"]["name"]
    kubectl.wait_pod_status(pod_name, "Running", ns=ns)

def require_zookeeper(manifest='zookeeper-1-node-1GB-for-tests-only.yaml', force_install=False):
    with Given("Install Zookeeper if missing"):
        if force_install or kubectl.get_count("service", name="zookeeper") == 0:
            config = util.get_full_path(f"../deploy/zookeeper/quick-start-persistent-volume/{manifest}")
            kubectl.apply(config)
            kubectl.wait_object("pod", "zookeeper-0")
            kubectl.wait_pod_status("zookeeper-0", "Running")

def wait_clickhouse_cluster_ready(chi):
    with Given("All expected pods present in system.clusters"):
        all_pods_ready = False
        while all_pods_ready is False:
            all_pods_ready = True
            for pod in chi['status']['pods']:
                cluster_response = clickhouse.query(
                    chi["metadata"]["name"],
                    "SYSTEM RELOAD CONFIG; SELECT host_name FROM system.clusters WHERE cluster='all-sharded'",
                    pod=pod
                )
                for host in chi['status']['fqdns']:
                    svc_short_name = host.replace(f'.{settings.test_namespace}.svc.cluster.local', '')
                    if svc_short_name not in cluster_response:
                        with Then("Not ready, sleep 5 seconds"):
                            all_pods_ready = False
                            time.sleep(5)

def install_clickhouse_and_zookeeper(chi_file, chi_template_file, chi_name):
    with Given("install zookeeper+clickhouse"):
        kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600)
        kubectl.create_ns(settings.test_namespace)
        util.require_zookeeper()
        kubectl.create_and_check(
            config=chi_file,
            check={
                "apply_templates": [
                    chi_template_file,
                    "templates/tpl-persistent-volume-100Mi.yaml"
                ],
                "object_counts": {
                    "statefulset": 2,
                    "pod": 2,
                    "service": 3,
                },
                "do_not_delete": 1
            }
        )
        clickhouse_operator_spec = kubectl.get(
            "pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator"
        )
        chi = kubectl.get("chi", ns=settings.test_namespace, name=chi_name)
        return clickhouse_operator_spec, chi


from rest_framework import serializers
from lsdb.models import WorkOrder
from lsdb.serializers import UnitListSerializer
from lsdb.serializers import TestSequenceDefinitionSerializer

class WorkOrderDetailSerializer(serializers.HyperlinkedModelSerializer):
    units = UnitListSerializer(many=True, read_only=True)
    test_sequence_definitions = TestSequenceDefinitionSerializer(many=True, read_only=True)
    disposition_name = serializers.ReadOnlyField(source='disposition.name')

    class Meta:
        model = WorkOrder
        depth = 1
        fields = [
            'id',
            'url',
            'name',
            'description',
            'project',
            'start_datetime',
            'disposition',
            'disposition_name',
            'test_sequence_definitions',
            'units',
        ]


#2468
#안전 영역
from collections import deque
import sys
input = sys.stdin.readline

def water_BFS(max, i, j):
    queue = deque()
    queue.append((i, j))
    visited[i][j] = True

    while(queue):
        x, y = queue.popleft()

        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]

            if 0 <= nx < n and 0 <= ny < n:
                if graph[nx][ny] <= max+1 and visited[nx][ny] == False:
                    visited[nx][ny] = True
                    queue.append((nx, ny))

def area_BFS(i, j):
    queue = deque()
    queue.append((0, 0))
    visited[i][j] = True

    while(queue):
        x, y = queue.popleft()

        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]

            if 0 <= nx < n and 0 <= ny < n:
                if visited[nx][ny] == False:
                    visited[nx][ny] = True
                    queue.append((nx, ny))

n = int(input())
graph = [list(map(int, input().split())) for _ in range(n)]
max_height = 0
for i in range(n):
    max_height = max(max_height, max(graph[i]))

dx = [1, 0, -1, 0]
dy = [0, 1, 0, -1]

area_list = [[0] for _ in range(max_height)]
for i in range(max_height):
    visited = [[False] * n for _ in range(n)]

    # 물 잠기기
    for j in range(n):
        for k in range(n):
            water_BFS(i, j, k)

    # 구역 세기
    cnt = 0
    for j in range(n):
        for k in range(n):
            if visited[j][k] == False:
                area_BFS(j, k)
                cnt += 1

    area_list[i] = cnt

print(max(area_list))

# 높이별로 BFS 돌려서 구역개수 max값
# 물 적시는 BFS 한번
# 구역 개수 세는 BFS 한번..?


from lenet import LeNet5
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision.datasets.mnist import MNIST
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import random

from torchvision.utils import save_image
import cv2
device = (torch.device('cuda') if torch.cuda.is_available()
            else torch.device('cpu'))

def add_noise_and_blur(data):
    """
    Add Gaussian noise and Gaussian blur to the images in data.

    data: The MNIST data

    """
    new_lst = []
    total_num = len(data)

    group_num = 5
    num_per_group = total_num // group_num

    kernel_lst = [(2 * k + 1, 2 * k + 1) for k in range(group_num)]
    std_lst = [0.01 * k for k in range(group_num)]

    for i in range(total_num):
        # if i == 3:
        #     break

        group_id = i // num_per_group
        kernel = kernel_lst[group_id]
        std = std_lst[group_id]

        img, label = data[i]
        img = cv2.GaussianBlur(img.detach().numpy(), kernel, cv2.BORDER_DEFAULT)  # add Gaussian blur
        img = torch.tensor(img)
        img = img + torch.normal(0, std, size=img.shape)  # add Gaussian noise
        new_lst.append((img, label))

    return new_lst

data_root = '/voyager/projects/aditya/CSC413/gan-vae-pretrained-pytorch/data/mnist'
random.seed(13)
torch.manual_seed(13)
data_train = MNIST(data_root,
                download=True,
                transform=transforms.Compose([
                    transforms.Resize((128, 128)),
                    transforms.ToTensor()]))
data_test = MNIST(data_root,
                train=False,
                download=True,
                transform=transforms.Compose([
                    transforms.Resize((128, 128)),
                    transforms.ToTensor()]))
data_train = add_noise_and_blur(data_train)
save_image(data_train[0][0], f'out_0.png')
save_image(data_train[12000][0], f'out_1.png')
save_image(data_train[24000][0], f'out_2.png')
save_image(data_train[36000][0], f'out_3.png')
save_image(data_train[48000][0], f'out_4.png')
data_test = add_noise_and_blur(data_test)

data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=8)
data_test_loader = DataLoader(data_test, batch_size=512, num_workers=8)

net = LeNet5().to(device=device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=2e-3)

def train(epoch):
    global cur_batch_win
    net.train()
    loss_list, batch_list = [], []
    for i, (images, labels) in enumerate(data_train_loader):
        images, labels = images.to(device=device), labels.to(device=device)
        optimizer.zero_grad()

        print(images.shape)
        output = net(images)

        loss = criterion(output, labels)

        loss_list.append(loss.detach().cpu().item())
        batch_list.append(i+1)

#         if i % 10 == 0:
#             print('Train - Epoch %d, Batch: %d, Loss: %f' % (epoch, i, loss.detach().cpu().item()))

        loss.backward()
        optimizer.step()

def test():
    net.eval()
    with torch.no_grad():
        total_correct = 0
        avg_loss = 0.0
        for i, (images, labels) in enumerate(data_test_loader):
            images, labels = images.to(device=device), labels.to(device=device)
            output = net(images)
            avg_loss += criterion(output, labels).sum()
            pred = output.detach().max(1)[1]
            total_correct += pred.eq(labels.view_as(pred)).sum()

    avg_loss /= len(data_test)
    acc = float(total_correct) / len(data_test)
    print('Test Avg. Loss: %f, Accuracy: %f' % (avg_loss.detach().cpu().item(), acc))
    return acc

def train_and_test(epoch):
    print('training...')
    train(epoch)
    acc = test()
    return acc

def main():
    for e in range(1, 13):
        acc = train_and_test(e)
        if e % 2 == 0:
            torch.save(net.state_dict(), f'./lenet_epoch={e}_test_acc={acc:0.3f}.pth')

if __name__ == '__main__':
    main()


import pygame
import sys
import random
import math
import numpy
pygame.init()
czas = pygame.time.Clock()

H = 500                                       # wysokość zbiornika
L = 800                                       # szerokość zbiornika
screen = pygame.display.set_mode((L, H))     # -------------

class Atom():
    def __init__(self, Rect, x, y, s, kolor):
        self.Rect = Rect
        self.speed_x = x
        self.speed_y = y
        self.x = Rect.center[0]
        self.y = Rect.center[1]

        self.r = s / 2
        self.col = kolor

def kolizja(j, atomy):  # zwraca indeks pierwszego atomu z którym wykryje zdarzenie
    atom = atomy[j]
    for i in range(len(atomy)):
        atom2 = atomy[i]
        # print(math.sqrt((atom.Rect.x - atom2.Rect.x) ** 2 + (atom.Rect.y - atom2.Rect.y) ** 2))
        # zamiast 56 2xpromień a x powinien r/10
        if 56 < math.sqrt((atom.Rect.center[0] - atom2.Rect.center[0])**2 + (atom.Rect.center[1] - atom2.Rect.center[1])**2) <= 60 + 6:
            print("kolizja")
            return i

        return -1

atomy = []  # lista atomów

s = 60  # średnica

# set 1 ukośne ten sam kierunek x
obiekt0 = pygame.Rect(200, 200, s, s)
atom0 = Atom(obiekt0, -3, -3, s, (100, 100, 250))
atomy.append(atom0)
obie1 = pygame.Rect(100, 100, s, s)
atom1 = Atom(obie1, 3, 3, s, (250, 100, 100))
atomy.append(atom1)

# set2 ukośne ten sam kierunek y
obiekt2 = pygame.Rect(650, 400, s, s)
atom2 = Atom(obiekt2, -3, -4, s, (255, 255, 255))
atomy.append(atom2)
obie3 = pygame.Rect(350, 300, s, s)
atom3 = Atom(obie3, 3, -2, s, (0, 0, 0))
atomy.append(atom3)

# set3    prawo lewo
obiekt4 = pygame.Rect(70, 430, s, s)
atom4 = Atom(obiekt4, 3, 0, s, (50, 150, 100))
atomy.append(atom4)
obie5 = pygame.Rect(200, 430, s, s)
atom5 = Atom(obie5, -3, 0, s, (250, 250, 100))
atomy.append(atom5)

# set4              góra dół
obiekt6 = pygame.Rect(80, 200, s, s)
atom6 = Atom(obiekt6, 0, 3, s, (250, 100, 250))
atomy.append(atom6)
obiekt7 = pygame.Rect(80, 360, s, s)
atom7 = Atom(obiekt7, 0, -3, s, (100, 250, 250))
atomy.append(atom7)

def ruch():
    for atom in atomy:
        # print(atom.col,atom.Rect.x,atom.Rect.y)
        atom.Rect.x += atom.speed_x
        atom.Rect.y += atom.speed_y
        # print(atom.col, atom.Rect.x, atom.Rect.y)
    tablica = [0] * len(atomy)
    '''print(
        f"pozycja atomu {atom.col} {atom.Rect.center[0]} i {atom.Rect.center[1]}")'''

    for i in range(len(atomy)):
        atom = atomy[i]
        kol = kolizja(i, atomy)

        # współrzędne wskazują na lewy górny róg kwadratu który wypełnia kulka
        if atom.Rect.center[0] - atom.r <= 0 or atom.Rect.center[0] + atom.r >= L:
            print("pyk róg")
            atom.speed_x *= -1

        elif atom.Rect.center[1] - atom.r <= 0 or atom.Rect.center[1] + atom.r >= H:
            print("pyk odbicie")
            atom.speed_y *= -1

        elif kol >= 0 and tablica[kol] == 0:
            atom1 = atomy[kol]
            tablica[kol] = 1
            tablica[i] = 1

            tupl = (atom.Rect.center[0] - atom1.Rect.center[0],  # (r1 - r2)
                    atom.Rect.center[1] - atom1.Rect.center[1])

            tupl2 = (atom1.Rect.center[0] - atom.Rect.center[0],  # (r2 - r1)
                     atom1.Rect.center[1] - atom.Rect.center[1])

            d = tupl[0]**2 + tupl[1]**2

            dot1 = ((atom.speed_x - atom1.speed_x) * tupl[0] + (
                atom.speed_y - atom1.speed_y) * tupl[1]) / d  # (dot/d)

            dot2 = ((atom1.speed_x - atom.speed_x) * tupl2[0] + (  # drugi atom
                atom1.speed_y - atom.speed_y) * tupl2[1]) / d

            tupl = (dot1 * tupl[0], dot1 * tupl[1])
            tupl2 = (dot2 * tupl2[0], dot2 * tupl2[1])
            atomy[i].speed_x = atom.speed_x - tupl[0]
            atomy[i].speed_y = atom.speed_y - tupl[1]
            atomy[kol].speed_x = atom1.speed_x - tupl2[0]
            atomy[kol].speed_y = atom1.speed_y - tupl2[1]

    # print("_____________________________",atom.Rect.colliderect(atom1.Rect))

        pygame.draw.ellipse(screen, atom.col, atom.Rect)

# Pętla programu
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:  # wyście iksem z okienka konczy program
            pygame.quit()
            sys.exit()

    screen.fill((80, 80, 80))  # zmienia kolor tła okna
    ruch()
    pygame.display.flip()  # wyświetla obiekty
    czas.tick(60)  # spowalnia, max 60 klatek na sekundę


#!/usr/bin/python
import socket
if __name__=='__main__':
    import sys
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    if 2 != len(sys.argv):
        print("usage: {} <hostname>".format(sys.argv[0]))
        sys.exit(-1)
    host = sys.argv[1]
    port = 13
    s.connect((host,port))
    data = bytearray()
    while True:
        recv_bytes = s.recv(100)
        if recv_bytes:
            data.extend(recv_bytes)
        else:
            break

    print(data)

    s.close()


from django.forms import ModelForm
from .models import ShortStory
from django import forms

class ShortStoryFrom(ModelForm):
    class Meta:
        model = ShortStory
        fields = ['title', 'context']

        widgets = {
            'title' : forms.TextInput(attrs={'class':'form-control',
                                              'placeholder' : 'Story Name'
                                             }),
            'context' : forms.Textarea(attrs={'class' : 'form-control',
                                              'placeholder' : 'Write From Here'
                                              }),
        }


import numpy as np
import re
from sklearn.model_selection import train_test_split
import pickle
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt

filename = 'smsspamcollection/SMSSpamCollection'
regex = re.compile('[^a-zA-Z0-9\s]')

# Read the data from the file and clean it
with open(filename, 'r', encoding='utf-8') as f:
    lines = f.readlines()

    data = []
    labels = []

    for line in lines:
        parts = line.split('\t')
        label = parts[0]
        message = parts[1]

        # clean the message
        message = regex.sub('', message)

        # separate the words
        words = message.split()
        data.append(words)
        labels.append(label)

# convert the data to numpy arrays
data = np.array(data)
labels = np.array(labels)

# divide the data into training and testing sets
train_data, test_data, train_labels, test_labels = train_test_split(data, labels, test_size=0.2, random_state=42)

class BayesianSpamClassifier:
    def __init__(self, load_data):
        self.epsilon = 1e-300
        self.load_data =load_data

    def train(self, train_data, train_labels):
        self.train_data = train_data
        self.train_labels = train_labels
        if self.load_data:
            # loading parameters
            with open("class_prob_dico.pkl", "rb") as file:
                self.class_prob_dico= pickle.load(file)
            with open("words_occ_dico.pkl", "rb") as file:
                self.words_occ_dico = pickle.load(file)
            with open("words_occ_dico_byClass.pkl", "rb") as file:
                self.words_occ_dico_byClass = pickle.load(file)
        else:
            # training parameters
            self.class_prob_dico = {}
            self.words_occ_dico = {}
            self.words_occ_dico_byClass = {}
            unique_labels, label_counts = np.unique(train_labels, return_counts=True)
            for lab in unique_labels:
                self.class_prob_dico[lab] = label_counts[unique_labels == lab][0] / len(train_labels)
            for i in range(len(train_data)):
                for word in train_data[i]:
                    if word not in self.words_occ_dico:
                        self.words_occ_dico[word] = 1
                    else:
                        self.words_occ_dico[word] += 1
                    if train_labels[i] not in self.words_occ_dico_byClass:
                        self.words_occ_dico_byClass[train_labels[i]] = {}
                    if word not in self.words_occ_dico_byClass[train_labels[i]]:
                        self.words_occ_dico_byClass[train_labels[i]][word] = 1
                    else:
                        self.words_occ_dico_byClass[train_labels[i]][word] += 1
            # saving parameters
            with open("class_prob_dico.pkl", "wb") as file:
                pickle.dump(self.class_prob_dico, file)
            with open("words_occ_dico.pkl", "wb") as file:
                pickle.dump(self.words_occ_dico, file)
            with open("words_occ_dico_byClass.pkl", "wb") as file:
                pickle.dump(self.words_occ_dico_byClass, file)

    def get_prob_byClass(self, label):
        # get the probability of a class
        return self.class_prob_dico[label]

    def get_prob_word_byClass(self, word, label):
        # get the probability of a word given a class
        if word in self.words_occ_dico_byClass[label]:
            return self.words_occ_dico_byClass[label][word]/sum(self.words_occ_dico_byClass[label].values())
        else:
            return self.epsilon

    def get_prob_word(self, word):
        # get the probability of a word
        if word in self.words_occ_dico:
            return self.words_occ_dico[word]/sum(self.words_occ_dico.values())
        else:
            return self.epsilon

    def predict(self, x):
        # predict the class of a message
        keys_list = list(self.class_prob_dico.keys())
        ret_prob_class = {}
        for k in keys_list:
            ret_prob_class[k] = 0
        for k in keys_list:
            prob_num = np.log(self.get_prob_byClass(k))
            prob_den = 0
            for xi in x:
                prob_num += np.log(self.get_prob_word_byClass(xi, k))
                prob_den += np.log(self.get_prob_word(xi))
            prob = prob_num - prob_den
            ret_prob_class[k] = prob
        return max(ret_prob_class, key=ret_prob_class.get)

    def accuracy(self, desc_set, label_set):
        # compute the accuracy of the model
        count=0
        for i in range(len(label_set)):
          if self.predict(desc_set[i]) == label_set[i]:
            count+=1
        return count/len(label_set)

bayes = BayesianSpamClassifier(True)
bayes.train(train_data, train_labels)
print("len test_data", len(test_data))
print(bayes.accuracy(test_data, test_labels))

print("Predicting...")
y_true = []
y_pred = []
for i in range(len(test_data)):
    y_true.append(test_labels[i])
    y_pred.append(bayes.predict(test_data[i]))
cm = confusion_matrix(y_true, y_pred)
plt.imshow(cm, cmap='binary')
plt.show()

# get the 5 most probable words for spam
spam_word_prob = {}
for word in bayes.words_occ_dico:
    spam_word_prob[word] = bayes.get_prob_word_byClass(word, 'spam')
sorted_spam_word_prob = sorted(spam_word_prob.items(), key=lambda x: x[1], reverse=True)
for i in range(5):
    print(sorted_spam_word_prob[i][0], ":", sorted_spam_word_prob[i][1])


#!/usr/bin/env python
""" Yet Another Timeseris Model (YATSM) - classify with trained algorithm

Usage:
    classify_yatsm.py [options] <config_file> <trained_algorithm>
        <job_number> <total_jobs>

Options:
    --resume                    Resume classification (don't overwrite)
    -v --verbose                Show verbose debugging messages
    --version                   Print program version and exit
    -h --help                   Show help and exit

"""
from __future__ import division, print_function

import logging
import os
import sys
import time

from docopt import docopt

import numpy as np
import numpy.lib.recfunctions as nprfn

from sklearn.externals import joblib

# Handle runnin as installed module or not
try:
    from yatsm.version import __version__
except ImportError:
    # Try adding `pwd` to PYTHONPATH
    sys.path.append(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__))))
    from yatsm.version import __version__
from yatsm.config_parser import parse_config_file
from yatsm.utils import calculate_lines, get_output_name, csvfile_to_dataset
from yatsm.reader import get_image_attribute

FORMAT = '%(asctime)s:%(levelname)s:%(module)s.%(funcName)s:%(message)s'
logging.basicConfig(format=FORMAT, level=logging.INFO, datefmt='%H:%M:%S')
logger = logging.getLogger('yatsm')

# Helper function
def try_resume(filename):
    """ Return True/False if dataset has already been classified

    Args:
      filename (str): filename of the result to be checked

    Returns:
      bool: If the `npz` file exists and contains a file 'class', this test will
    return True, else False.

    """
    try:
        z = np.load(filename)
    except:
        return False

    if not z['record'].dtype or 'class' not in z['record'].dtype.names:
        return False

    return True

# Main classification function
def classify_line(filename, classifier):
    """ Use `classifier` to classify data stored in `filename`

    Args:
      filename (str): filename of stored results
      classifier (sklearn classifier): pre-trained classifier

    """
    z = np.load(filename)
    rec = z['record']

    if rec.shape == 0:
        logger.debug('No records in {f}. Continuing'.format(f=filename))

    # Rescale intercept term
    coef = rec['coef']
    coef[:, 0, :] = (coef[:, 0, :] + coef[:, 1, :] *
                     ((rec['start'] + rec['end']) / 2.0)[:, np.newaxis])

    # Include RMSE for full X matrix
    newdim = (coef.shape[0], coef.shape[1] * coef.shape[2])
    X = np.hstack((coef.reshape(newdim), rec['rmse']))

    # Create output and classify
    classes = classifier.classes_
    classified = np.zeros(rec.shape[0], dtype=[
        ('class', 'u2'),
        ('class_proba', 'float32', classes.size)
    ])
    classified['class'] = classifier.predict(X)
    classified['class_proba'] = classifier.predict_proba(X).max(axis=1)

    # Merge
    merged_rec = nprfn.merge_arrays((rec, classified), flatten=True)

    # Create dict for re-saving `npz` file (only way to append)
    out = {}
    for k, v in z.iteritems():
        out[k] = v
    out['classes'] = classes
    out['record'] = merged_rec

    np.savez(filename, **out)

# Main and parsing of arguments
def parse_args(args):
    """ Returns dictionary of parsed and validated command arguments

    Args:
      args (dict): Arguments from user

    Returns:
      dict: Parsed and validated arguments

    """
    parsed_args = {}
    # Required args
    parsed_args['config_file'] = args['<config_file>']

    parsed_args['algo'] = args['<trained_algorithm>']

    try:
        job_number = int(args['<job_number>'])
    except:
        print('Error - <job_number> must be an integer greater than 0')
        sys.exit(1)
    if job_number <= 0:
        print('Error - <job_number> cannot be less than or equal to 0')
        sys.exit(1)
    parsed_args['job_number'] = job_number

    try:
        parsed_args['total_jobs'] = int(args['<total_jobs>'])
    except:
        print('Error - <total_jobs> must be an integer')
        sys.exit(1)
    if parsed_args['job_number'] > parsed_args['total_jobs']:
        print('Error - <job_number> must be less than or equal to total jobs')
        sys.exit(1)

    parsed_args['resume'] = args['--resume']

    return parsed_args

def main(args):
    """ Classify dataset """
    # Parse config and file data
    dataset_config, yatsm_config = parse_config_file(args['config_file'])

    # Get some attributes about the dataset
    dates, images = csvfile_to_dataset(
        dataset_config['input_file'],
        date_format=dataset_config['date_format']
    )
    nrow, _, _, _ = get_image_attribute(images[0])

    # Read in the saved classification result
    try:
        f = open(args['algo'])
    except:
        logger.error('Could not open pickled classifier')
        sys.exit(1)

    classifier = joblib.load(f)

    # Split into lines and classify
    job_lines = calculate_lines(args['job_number'] - 1, args['total_jobs'],
                                nrow)
    logger.debug('Responsible for lines: {l}'.format(l=job_lines))

    start_time = time.time()
    logger.info('Starting to run lines')
    for job_line in job_lines:

        filename = get_output_name(dataset_config, job_line)
        if not os.path.exists(filename):
            logger.warning('No model result found for line {l} '
                           '(file {f})'.format(l=job_line, f=filename))
            pass

        if args['resume'] and try_resume(filename):
            logger.debug('Already processed line {l}'.format(l=job_line))
            continue

        logger.debug('Classifying line {l}'.format(l=job_line))
        classify_line(filename, classifier)

    logger.debug('Completed {n} lines in {m} minutes'.format(
        n=len(job_lines),
        m=round((time.time() - start_time) / 60.0, 2))
    )

if __name__ == '__main__':
    args = docopt(__doc__, version=__version__)

    if args['--verbose']:
        logger.setLevel(logging.DEBUG)

    args = parse_args(args)
    main(args)


import sys, time
import socket
import struct
import select
from threading import Thread
import os

servers = []
clients = []

def find_clients():
    message = "Looking for clients"
    encoded_message = message.encode()
    mcast_grp = ('224.7.7.7', 7070)
    client_mcast = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    client_mcast.settimeout(1)
    ttl = struct.pack('b',1)
    client_mcast.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)

    try:
        sent = client_mcast.sendto(encoded_message, mcast_grp)
        while True:
            data, addr = client_mcast.recvfrom(16)
            print(f"Chess client {addr[0]} available.")
            clients.append(addr[0])
    except:
        client_mcast.close()
    finally:
        client_mcast.close()

def find_servers():
    message = "Looking for servers"
    encoded_message = message.encode()
    mcast_grp = ('224.6.6.6', 6060)
    client_mcast = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    client_mcast.settimeout(1)
    ttl = struct.pack('b',1)
    client_mcast.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)

    try:
        sent = client_mcast.sendto(encoded_message, mcast_grp)
        while True:
            data, addr = client_mcast.recvfrom(16)
            print(f"Chess server {addr[0]} available.")
            servers.append(addr[0])
    except:
        client_mcast.close()
    finally:
        client_mcast.close()

def receive_message(sock):
    while True:
        sockets_list = [sys.stdin, sock]
        try:
            read_sockets,write_socket, error_socket = select.select(sockets_list,[],[])

            for socks in read_sockets:
                if socks == sock:
                    message = socks.recv(1024)
                    message = message.decode()
                    print (message)
                    if not message:
                        sock.close()

        except Exception as e:
            print(e)
            print("You were disconnected.")
            # exit()
            kill()

def show_servers():
    print(servers)

def show_clients():
    print(clients)

def kill():
    os._exit(0)

if __name__ == "__main__":

    # if ((len(sys.argv) != 2) or sys.argv[1]!='peer-to-peer' or sys.argv[1]!='client-server'):
    #     print("Usage: 'python3 peer-to-peer|client-server'")
    #     exit()

    mode = sys.argv[1]

    while True:
        if(mode == 'peer-to-peer'):
            print("Not done yet.")
        if(mode == 'client-server'):
            find_servers()
            if(len(servers)>0):
                server = servers[0]

                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((server, 6060))
                print(f"Connected to {server}")
                Thread(target=receive_message, args=(sock,)).start()
                while True:
                    command = input("command>>")
                    sock.send(command.encode())


from NewsParser import *
import re
from urllib import parse

class NewsParser_NYT(NewsParser):
    """Custom Parser for 'NYtimes' news website. Inherits 'NewsParser' class."""

    # All the methods in here are similar to 'NewsParser_CNN' class methods. Hence, limiting the comments.
    def handle_data(self, data):
        if "body" in self.prev_tags:
            if self.tagcount < 3:
                return
            if self.source == "nytimes":
                self.getheadlines(data)
            elif self.source == "nytimes-politics":
                self.getheadlines(data)
            elif self.source == "nytimes-business":
                self.getheadlines(data)
            elif self.source == "nytimes-world":
                self.getheadlines_world(data)

    def getheadlines(self, data):
        if self.tagcount>1 and self.prev_tags[-1] == "a" and self.prev_tags[-2] == "h2":
            #print(data)
            attr_dict = dict(self.prev_attrs[-1])
            self.extract_content(data, attr_dict)
        elif self.tagcount>1 and self.prev_tags[-1] == "h2" and self.prev_tags[-2] == "a":
            # print(data)
            attr_dict = dict(self.prev_attrs[-2])
            self.extract_content(data, attr_dict)
        return

    def getheadlines_world(self, data):
        if self.tagcount>1 and self.prev_tags[-1] == "a" and self.prev_tags[-2] == "h2":
            h2_class = dict(self.prev_attrs[-2]).get("class")
            if h2_class != "headline":
                return
            #print(data)
            attr_dict = dict(self.prev_attrs[-1])
            self.extract_content(data, attr_dict)
        return

    def extract_content(self, data, attr_dict):
        """Has common execution content for all 'get_headlines" methods."""
        self.content += data + "; nytimes; "
        # print(attr_dict)
        artcl_url = parse.urljoin(self.baseUrl, attr_dict.get('href'))
        artcl_flnm = "nytimes_" + str(self.filename_counter) + ".txt"
        self.content += artcl_url + "; "
        self.content += artcl_flnm
        self.content += "\n"

        self.newscount += 1
        self.filename_counter += 1
        self.article_urls.append(artcl_url)
        self.article_filenames.append(artcl_flnm)
        if self.newscount > self.maxcount - 1:
            raise DataParsedException(data)
        return

    def get_article_data(self, url, filename):
        article_content = ""
        response = urlopen(url)
        htmlstring = response.read().decode('utf-8')

        content1 = re.findall(r'
(.*?)
', htmlstring)
        for i in range(len(content1)):
            article_content += str(content1[i]) + "\n"
            if i == 2:
                break
        p = re.compile('<(.*?)>')
        article_content = p.sub("", article_content)
        p = re.compile(''|‘|’')
        article_content = p.sub("\'", article_content)
        p = re.compile(' ')
        article_content = p.sub(" ", article_content)
        p = re.compile('"|“|”')
        article_content = p.sub("\"", article_content)
        p = re.compile('—')
        article_content = p.sub("-", article_content)
        #print(article_content)
        fh = open("files/news_articles/nytimes/" + filename, 'w')
        fh.write(article_content)
        fh.close()
        return

    def exec_articles(self):
        print("Extracting articles content...")
        for i in range(0, len(self.article_urls)):
            url = self.article_urls[i]
            filename = self.article_filenames[i]
            self.get_article_data(url, filename)
        self.article_urls = []
        self.article_filenames=[]
        print("Success...\n")

    def execute(self):
        self.parseurl("https://www.nytimes.com/section/us", "nytimes")
        self.write_to_file("headlines.txt")
        self.reset_content()
        self.parseurl("https://www.nytimes.com/section/politics", "nytimes-politics")
        self.write_to_file("headlines_politics.txt")
        self.reset_content()
        self.parseurl("https://www.nytimes.com/section/business", "nytimes-business")
        self.write_to_file("headlines_business.txt")
        self.reset_content()
        self.parseurl("https://www.nytimes.com/section/world", "nytimes-world")
        self.write_to_file("headlines_world.txt")
        self.reset_content()
        self.exec_articles()


"""MongoDB models for Social Auth"""
import base64
import six
from sqlalchemy.exc import IntegrityError

from social.exceptions import NotAllowedToDisconnect
from social.storage.base import UserMixin, AssociationMixin, NonceMixin, \
    BaseStorage

class MongoengineMixin(object):

    @classmethod
    def _new_instance(cls, model, *args, **kwargs):
        return cls._save_instance(model(*args, **kwargs))

    @classmethod
    def _save_instance(cls, instance):
        return instance.save()

    @classmethod
    def _del_session(cls, pk):
        raise NotImplementedError('Implement in subclass')

class MongoengineUserMixin(MongoengineMixin, UserMixin):

    """Social Auth association model"""
    @classmethod
    def changed(cls, user):
        cls._save_instance(user)

    def set_extra_data(self, extra_data=None):
        if super(MongoengineUserMixin, self).set_extra_data(extra_data):
            self._save_instance(self)

    @classmethod
    def disconnect(cls, name, user, association_id=None):
        if association_id is not None:
            qs = cls.user_model().objects(username=association_id)
        else:
            qs = cls.objects(provider__ne=backend_name)
        if qs.count() == 0:
            raise NotAllowedToDisconnect()
        cls._del_session(qs)

    @classmethod
    def user_query(cls):
        return cls._session().query(cls.user_model())

    @classmethod
    def user_exists(cls, username):
        """
        Return True/False if a User instance exists with the given arguments.
        Arguments are directly passed to filter() manager method.
        """
        return cls.user_model().objects.filter(username=username).count() > 0

    @classmethod
    def get_username(cls, user):
        return getattr(user, 'username', None)

    @classmethod
    def create_user(cls, username, email=None):
        return cls._new_instance(cls.user_model(), username=username,
                                 email=email)

    @classmethod
    def get_user(cls, pk):
        return cls._get_session(pk)

    @classmethod
    def get_name(cls, pk):
        return cls.user_model().objects(pk=pk)[0].username

    @classmethod
    def get_social_auth(cls, provider, uid):
        if not isinstance(uid, six.string_types):
            uid = str(uid)
        try:
            return cls.objects.filter(provider=provider,
                                      uid=uid).first()
        except IndexError:
            return None

    @classmethod
    def get_social_auth_for_user(cls, user):
        qs = cls.user_model().objects(username=user.username)
        user.provider = cls.objects(user__in=qs)[0].provider
        return user

    @classmethod
    def create_social_auth(cls, user, uid, provider):
        if not isinstance(uid, six.string_types):
            uid = str(uid)
        return cls._new_instance(cls, user=user, uid=uid, provider=provider)

class MongoengineNonceMixin(NonceMixin):

    @classmethod
    def use(cls, server_url, timestamp, salt):
        kwargs = {'server_url': server_url, 'timestamp': timestamp,
                  'salt': salt}
        try:
            return cls.objects.filter(**kwargs)
        except IndexError:
            return cls._new_instance(cls, **kwargs)

class MongoengineAssociationMixin(AssociationMixin):

    @classmethod
    def store(cls, server_url, association):
        # Don't use get_or_create because issued cannot be null
        try:
            assoc = cls.objects.filter(server_url=server_url,
                                       handle=association.handle)[0]
        except IndexError:
            assoc = cls(server_url=server_url,
                        handle=association.handle)
        assoc.secret = base64.encodestring(association.secret)
        assoc.issued = association.issued
        assoc.lifetime = association.lifetime
        assoc.assoc_type = association.assoc_type
        cls._save_instance(assoc)

    @classmethod
    def get(cls, *args, **kwargs):
        return cls.objects.filter(*args, **kwargs)

    @classmethod
    def remove(cls, ids_to_delete):
        cls.objects.filter(cls.id.in_(ids_to_delete)).delete()

class BaseMongoengineStorage(BaseStorage):
    user = MongoengineUserMixin
    nonce = MongoengineNonceMixin
    association = MongoengineAssociationMixin

    @classmethod
    def is_integrity_error(cls, exception):
        return exception.__class__ is IntegrityError


#
# Python implementation of the Crawford Lab @ Virginia Tech's c++ molecular
# programming project found at
#
# http://sirius.chem.vt.edu/wiki/doku.php?id=crawdad:programming:project1
# or
# https://github.com/CrawfordGroup/ProgrammingProjects/tree/master/Project%2301
#
# Licence originally under: http://creativecommons.org/licenses/by-nc-sa/4.0/
# Linear algebra implemented using numpy and some optional optimizations
# using the numba JIT-compiler.
#

import numpy as np
import numpy.linalg as la
from numba import jit, njit
from numba import int64, float64
import pandas as pd
import itertools as it
from my_elements import tbl as el_tbl
import scipy.constants as sc

br = sc.physical_constants['Bohr radius'][0]

@jit
def _tens_fo(masses, coords, ax1, ax2):
    arr = np.empty(coords.shape)
    arr[:, :2] = coords[:, [ax1, ax2]]
    arr[:, 2] = masses.ravel()
    element = np.prod(arr, axis=1).sum()
    return element

@jit
def _tens_fd(masses, coords, ax):
    axes = list(np.arange(3))
    axes.remove(ax)
    sqrs = coords**2
    element = (masses.T * (sqrs[:, axes[0]] + sqrs[:, axes[1]])).sum()
    return element

@jit
def _inertia_tensor(center_coords, tens_fo, tens_fd):
    center_coords()
    in_tens = np.frompyfunc(tens_fo, 2, 1).outer(np.arange(3), np.arange(3)).astype(np.float64)
    for i in np.arange(3):
        in_tens[i,i] = tens_fd(i)
    return in_tens

def _zvals_typecheck(zvals):
    if type(zvals[0]) == float:
        zvals = [int(z) for z in zvals]
    elif type(zvals[0]) == str:
        if any(chars.isdigit() for chars in zvals[0]):
            zvals = [int(float(z)) for z in zvals]
        else:
            zvals = [el_tbl[el_tbl.symbol == sym].atomicNumber.iloc[0] for sym in zvals]
    return zvals

def _xyz_input(txt_file, comment_line=0):
    with open(txt_file, 'r') as fin:
        lines = fin.read().splitlines()
        natom = int(lines[0])
        coords = np.zeros([natom, 3], dtype="float64")
        zvals = []
        for i, x in enumerate(coords):
            line = lines[i+1+comment_line].split()
            zvals.append(line[0])
            x[:] = list(map(float, line[1:4]))
        zvals = _zvals_typecheck(zvals)
    return natom, zvals, coords

def _dist_rep(self):
    self.perms = list(it.combinations(np.arange(0,self.natom), r=2))
    atom1 = [x+1 for x, y in self.perms]
    atom2 = [y+1 for x, y in self.perms]
    dist = [self.dist_matrix[pair] for pair in self.perms]
    return pd.DataFrame({'Atom_1': atom1, 'Atom_2': atom2, 'Distance': dist})

class Molecule(object):

    """
    Molecule class

    A simple Molecule class for computations, including calculations of bond lengths,
    bond angles, out-of-plane angles and dihedral angles. Also supports re-centering the
    coordinate system to the center-of-mass (translation). Calculates the principal moments
    inertia and rotational constants.

    Init either with normal initialisation through giving the number of atoms, the atomic
    number (zvals) as well as the coordinates of those atoms. Or, more conveniently, through
    either the .from_txt or .from_xyz methods.

    E.g.: h20 = Molecule(3, [8, 1, 1], np.array([[0, 0, -0.134503695264], [0, -1.684916670000, 1.067335684736], [1, 1.684916670000, 1.067335684736]]))

    or

    h20 = Molecule.from_txt('./h2o.txt')

    Use the .rep attribute to access a dataframe containg the atoms, their z-value as well as their
    cartesian coordinates.

    The distances between connected atoms are given in the form of a dataframe throught the
    .dist_rep attribute. The distance matrix can be accessed through the .dist_matrix attribute.

    Bond angles can be displayed through the .bond_angles() method, which returns a dataframe.

    Symmetry TBD.
    """

    def __init__(self, natom, zvals, coords):
        self.natom = natom
        self.charge = 0
        self.zvals = np.array(zvals)
        self.coords = coords
        self.point_group = ''
        self.rep = pd.DataFrame({'Sym': [el_tbl[el_tbl.atomicNumber == z].symbol.iloc[0] for z in zvals] , 'Z': zvals, 'x': coords[:,0], 'y': coords[:,1], 'z': coords[:,2]})
        self.dist_matrix = np.sum((coords[None,:] - coords[:, None])**2, -1)**0.5
        self.dist_rep = _dist_rep(self)
        self._bond_cond = (self.dist_matrix < 3) & (self.dist_matrix > 0)

    def __repr__(self):
        return "Molecule with following input:\n\n" + self.rep.to_string()

    @classmethod
    def from_txt(class_object, txt_file):
        natom, zvals, coords = _xyz_input(txt_file, comment_line=0)
        return class_object(natom, zvals, coords)

    @classmethod
    def from_xyz(class_object, txt_file):
        natom, zvals, coords = _xyz_input(txt_file, comment_line=1)
        return class_object(natom, zvals, coords)

    def print_geom(self):
        print(self.rep)

    def translate(self, xyz):
        trans = np.array(xyz)
        self.coords = self.coords + trans

    def print_dist(self):
        print('Distances between atoms (unit):')
        print(self.dist_rep)

    def bonds(self):
        return list({tuple(sorted(pair)) for pair in zip(np.where(self._bond_cond)[0], np.where(self._bond_cond)[1])})

    def bond_counts(self):
        num, cnt = np.unique(self.bonds(), return_counts=True)
        return dict(zip(num, cnt))

    def conn_atoms(self, rng):
        num, cnt = np.unique(self.bonds(), return_counts=True)
        bonds = np.array(self.bonds())
        adjacent_bonds = dict()
        for n in np.where(np.array((num, cnt))[1] > 1)[0]:
            adj_bnds = [bond for bond in self.bonds() if n in bond]
            adjacent_bonds[n] = adj_bnds
        if rng == 2:
            return adjacent_bonds

    def unit_vector(self, bond):
        atom1, atom2 = bond
        e = np.array(-(self.coords[atom1] - self.coords[atom2])) / self.dist_matrix[atom1][atom2]
        return e

    def bond_angle(self, bond1, bond2, deg=True):
        if deg:
            return np.rad2deg(np.arccos(np.dot(self.unit_vector((bond1)), self.unit_vector((bond2)))))
        else:
            return np.arccos(np.dot(self.unit_vector((bond1)), self.unit_vector((bond2))))

    def bond_angles(self):
        if self.natom <= 2:
            raise ValueError('The number of atoms in the molecule is not sufficient to calculate any bond angles')
        bnd_angls = pd.DataFrame()
        for key, vals in self.conn_atoms(2).items():
            for pair in list(it.combinations(vals, r=2)):
                bond1, bond2 = pair
                atom1, atom2 = list(pair[0]), list(pair[1])
                atom1.remove(key)
                atom2.remove(key)
                line = {'Atom_1': atom1[0], 'Conn_Atom': key, 'Atom_2': atom2[0], 'Angle': self.bond_angle(bond1, bond2)}
                line = pd.DataFrame(line, index=[0])
                bnd_angls = bnd_angls.append(line, ignore_index=True)
        return bnd_angls

    def oop_angle(self, bond_ki, bond_kj, bond_kl, deg=True):
        eki, ekj, ekl = self.unit_vector(bond_ki), self.unit_vector(bond_kj), self.unit_vector(bond_kl)
        theta = np.arcsin(np.dot(np.cross(ekj, ekl) / np.sin(self.bond_angle(bond_kj, bond_kl, deg=False)), eki))
        theta = abs(theta)
        if deg:
            return np.rad2deg(theta)
        else:
            return theta

    def oop_angles(self):
        if self.natom <= 3:
            raise ValueError('The number of atoms in the molecule is not sufficient to calculate any out-of-plane angles')
        oop_angls = pd.DataFrame()
        for key, vals in self.conn_atoms(2).items():
            for triple in list(it.combinations(vals, r=3)):
                for i, bond_ki in enumerate(triple):
                    bond_kj, bond_kl = (x for x in triple if x != triple[i])
                    atomi, atomj, atoml = list(bond_ki), list(bond_kj), list(bond_kl)
                    atomi.remove(key)
                    atomj.remove(key)
                    atoml.remove(key)
                    line = {'Atom_i': atomi[0], 'Atom_k': key, 'Atom_j': atomj[0], 'Atom_l': atoml[0], 'OOP_angle': self.oop_angle(bond_ki, bond_kj, bond_kl)}
                    line = pd.DataFrame(line, index=[0])
                    oop_angls = oop_angls.append(line, ignore_index=True)
        return oop_angls

    def tort_angle(self, bond_ij, bond_jk, bond_kl, deg=True):
        eij, ejk, ekl = self.unit_vector(bond_ij), self.unit_vector(bond_jk), self.unit_vector(bond_kl)
        cross_ijk = np.cross(eij, ejk)
        cross_jkl = np.cross(ejk, ekl)
        phi_ijk = self.bond_angle(bond_ij, bond_jk, deg=False)
        phi_jkl = self.bond_angle(bond_jk, bond_kl, deg=False)
        tau = np.arccos(np.dot(cross_ijk, cross_jkl) / (np.sin(phi_ijk)) * np.sin(phi_jkl))
        tau = abs(tau)
        if deg:
            return np.rad2deg(tau)
        else:
            return tau

    def middle_bonds(self):
        middle_bonds = [(x,y) for x,y in self.bonds() if (x in self.conn_atoms(2).keys()) and (y in self.conn_atoms(2).keys())]
        return middle_bonds

    def possible_tortion_angles(self):
        torsion_dict = {}
        for mb in self.middle_bonds():
            conn_bonds = []
            for atom in mb:
                atom_conn_bonds = []
                for bond in self.bonds():
                    if (bond != mb) and (any(e in bond for e in mb)) and (atom in bond):
                        atom_conn_bonds.append(bond)
                conn_bonds.append(atom_conn_bonds)
            torsion_dict[mb] = conn_bonds
        return torsion_dict

    def tort_angles(self):
        if self.natom <= 3:
            raise ValueError('The number of atoms in the molecule is not sufficient to calculate any tortion angles')
        tort_angls = pd.DataFrame()
        for mb, bond_list in self.possible_tortion_angles().items():
            for bond1 in bond_list[0]:
                for bond2 in bond_list[1]:
                    atom1, atom2 = list(bond1), list(bond2)
                    atom1.remove(mb[0])
                    atom2.remove(mb[1])
                    line = {'Atom_1': atom1[0], 'Atom_2': atom2[0], 'Middle_bond': f'{mb}', 'Torsion_angle': self.tort_angle(bond1, mb, bond2)}
                    line = pd.DataFrame(line, index=[0])
                    tort_angls = tort_angls.append(line, ignore_index=True)
        return tort_angls

    def masses(self):
        return np.array([el_tbl[el_tbl.atomicNumber == x].atomicMass.iloc[0] for x in self.zvals])[:, None]

    def mass_weighted_coords(self):
        mw_coords = self.coords * self.masses()
        return mw_coords

    def center_of_mass(self):
        com = self.mass_weighted_coords().sum(axis=0) / self.masses().sum()
        return com

    def center_coords(self):
        self.translate(-self.center_of_mass())

    def tens_fo(self, ax1, ax2):
        return _tens_fo(self.masses(), self.coords, ax1, ax2)

    def tens_fd(self, ax):
        return _tens_fd(self.masses(), self.coords, ax)

    def inertia_tensor(self):
        return _inertia_tensor(self.center_coords, self.tens_fo, self.tens_fd)

    def p_inertia_moments(self):
        vals = la.eigh(self.inertia_tensor())[0]
        keys = ['Ia', 'Ib', 'Ic']
        return dict(zip(keys, vals))

    def moment2const(self, I):
        # br = sc.physical_constants['Bohr radius'][0]
        br = 5.2917721067e-11
        return sc.Planck / (8 * np.pi**2 * br**2 * sc.atomic_mass * I)

    def rot_consts(self):
        const = np.empty((3,))
        for i, I in enumerate(self.p_inertia_moments().values()):
            const[i] = self.moment2const(I)
        return const

    def rot_const_mhz(self):
        return dict(zip(['A', 'B', 'C'], self.rot_consts()*1e-6))

    def rot_const_wn(self):
        return dict(zip(['A', 'B', 'C'], self.rot_consts() / (sc.speed_of_light * 1e+2)))

    def read_hessian(self, hessian_file, skiprows=1):
        with open(hessian_file) as hf:
            natom = int(hf.read().splitlines()[0].split()[0])
        if self.natom != natom:
            raise ValueError('The number of atoms in the molecule doesn\'t match those in the hessian matrix! Try another matrix.')
        self.hessian = np.loadtxt(hessian_file, skiprows=skiprows)
        self.hessian = self.hessian.reshape((self.natom*3, self.natom*3))

    def weight_hessian(self):
        masses = self.masses().ravel()
        wts = np.repeat(masses, 3)
        wt_matrix = np.outer(wts, wts)**.5
        self.wt_hessian = self.hessian / wt_matrix

    def hessian_evals(self):
        try:
            self.wt_hessian
        except AttributeError:
            print('The weighted hessian has\'t been defined for this molecule yet!')
            return
        eig = la.eig(self.wt_hessian)[0]
        eig.sort()
        return eig[::-1]

    def harm_freq_hz(self):
        hrm_frq = self.hessian_evals()
        for x in np.nditer(hrm_frq, op_flags=['readwrite']):
            if x <= 0:
                x[...] = 0
        const = (sc.physical_constants['Hartree energy'][0] / sc.physical_constants['Bohr radius'][0]**2 / sc.atomic_mass)
        hrm_frq = hrm_frq * const
        return hrm_frq**.5

    def harmonic_freq_mhz(self):
        return self.harm_freq_hz() * 1e-6

    def harmonic_freq_wn(self):
        return self.harm_freq_hz() / (2 * np.pi * sc.speed_of_light * 1e2)