# ---------------------------------------------------------------------------
# FireIgnitionRaw_GridMet_Historic.py
# Scripts Derives Fire Ignition Potential and categorizes By High, Medium, and Low Fire Ignition Potential rating pulling from the defined GridMet Stations
# Output time period is daily data 1991-2020.
# Process water balance data which is pull form the defiend Gird Met Station on Climate Analyzer via the serviceURL parameter
# for Forested and Non-Forested models counts number of occurrances (i.e. days) for the previous four years (i.e. annual summaries).
# Applies the Fire Ignition Potential model as defined in Thoma et. al. 2020 Management Applications Paper

#Input Parameter is Cumulative Water Defict - Water Balance parameter

#Updates:
#20230519 - Updated Fire Ignition Model to Steve Huysman Version 2.0 Southern Rockies first order https://huysman.net/research/fire/southern_rockies.html.
# Fire Ignition Model vesion 2.0 included used Monitoring Trends Burn Severity data from 1984-1-1 through 2021-12-31 (Note Version FI model 1.0 used MTBS data through 2015-12-31.

#20230707 - Updated script to dynamically process Gridmet Stations on Climate Analyzer via the Gridmet station name as defined in the 'siteName' variable.  Script is no longer hard coded for FLFO

#Dependicies:
#Python Version 3.9, Pandas, urllib, numpy

#Script Name: FireIgnitionRaw_GridMet_Historic.py  - was previously called 'FLFO_FireIgnitionRaw_GridMet_1991_2020v3.py' locally on KRS
#Created by Kirk Sherrill - Data Manager Rocky Mountain Network - I&M National Park Service

##Import Libraries
import pandas as pd, traceback, sys, os
import numpy as np
import urllib
from urllib.request import urlretrieve
import datetime
from datetime import date

###################################################
# Start of Parameters requiring set up.
###################################################

inFieldFieldNowCast = 'D (MM)'  #Field in the Now Cast data being used in the fire iginition pototential model (this will be the deficit field).

siteName = 'eastinlet_from_grid'  #Gridmet Station Name

#Define URL with the Gridmet data in Climate Analyzer
siteNameNoFromGrid = siteName.replace("_from_grid","")   #SiteName with the 'from_grid' removed
serviceURL = "http://www.climateanalyzer.science/python/wb2.py?csv_output=true&station=" + siteName + "&title=" + siteNameNoFromGrid + "&pet_type=hamon&max_soil_water=250&graph_table=table&table_type=daily&forgiving=very&year1=1980&year2=2022&station_type=GHCN&sim_snow=true&force=true?"

#Define the Historic/Current reference parameters:
refYearStartDate= '1/1/1984'   #Start Year/Date for which Fire Ignition Model was evaluated (Jan 1 of Start Year)
refYearEndDate = '12/31/2021'       #End Year/Date for which Fire Ignition Model was evaluated (Dec 31 of End Year

#Get Current Date
today = date.today()
strDate = today.strftime("%Y%m%d")

#Fire Year Parameters by Vegtation Type
nonforest_start_DOY = 79  #Non-Forested Start of Fire Year Day Number
nonforest_end_DOY = 303   #Non-Forested End of Fire Year Day Number
forest_start_DOY = 7      #Forest Start of Fire year Day Number
forest_end_DOY = 301      #Forest End of Fire year Day Number
movingWindowsDay = 14     #Number of days in the moving window average (default use 14)

#Output Directory/LogFile Information
outputFolder = r"C:\ROMN\Climate\ClimateAnalyzer\Dashboards\ROMO\GridMetStations\\" + siteName + "\\" + strDate #Folder for the output Data Package Products
#outputFolder = "./"  #Folder for the output Data Package Products
workspace = outputFolder + "\\workspace"   #workspace
#workspace = "./"  #workspace
outName = siteName + "_FireIgnitionRating_1991_2020_WinDays" + str(movingWindowsDay)   #Output .csv filename
logFileName = workspace + "\\" + outName + ".LogFile.txt"
#logFileName = outName + ".LogFile.txt"

#######################################
## Below are paths which are hard coded
#######################################

#################################
# Checking for directories and Log File
##################################
if os.path.exists(outputFolder):
    pass
else:
    os.makedirs(outputFolder)

if os.path.exists(workspace):
    pass
else:
    os.makedirs(workspace)

# Check if logFile exists
if os.path.exists(logFileName):
    pass
else:
    logFile = open(logFileName, "w")  # Creating index file if it doesn't exist
    logFile.close()

def main():

    try:

        ###########################################
        # Function to pull All GridMet Statation Data:
        outVal = processGridMetStation(serviceURL, siteName)
        if outVal[0] != "Success function":
            print("WARNING - Function processNowCast failed - Exiting Script")
            exit()
        else:

            dfAllGridMet = outVal[1]

            print("Success - Function processGridMetStation")

        #Create Reference List for Forested Vegetation Type
        outVal = define_ReferenceList(forest_start_DOY, forest_end_DOY, refYearStartDate, refYearEndDate, dfAllGridMet, inFieldFieldNowCast, "DATE")
        if outVal[0] != "Success function":
            print("WARNING - Function define_ReferenceList failed - Exiting Script")
            exit()
        else:
            print("Success - Function define_ReferenceList - Forested Vegetation")
            # Assign the reference Data Frame
            refDF_Forest = outVal[1]

        # Create Reference List for Non Forest Vegetation Type
        outVal = define_ReferenceList(nonforest_start_DOY, nonforest_end_DOY, refYearStartDate, refYearEndDate, dfAllGridMet, inFieldFieldNowCast, "DATE")
        if outVal[0] != "Success function":
            print("WARNING - Function define_ReferenceList failed - Exiting Script")
            exit()
        else:
            print("Success - Function define_ReferenceList - Non Forest Vegetation")
            # Assign the reference Data Frame
            refDF_NonForest = outVal[1]

        # Define Moving Window Averages
        outVal = define_MovingWindowAverage(dfAllGridMet, movingWindowsDay, inFieldFieldNowCast)
        if outVal[0] != "Success function":
            print("WARNING - Function define_MovingWindowAverage failed - Exiting Script")
            exit()
        else:
            print("Success - Function define_MovingWindowAverages")
            # Assign output to DF
            dfHistCur_wAvg = outVal[1]

        # Define Percentiles - for Historic/Cur data, and using Forest Fire Season
        field14Average = inFieldFieldNowCast + "_" + str(movingWindowsDay)
        outVal = define_IgnitionProportion(dfHistCur_wAvg, field14Average, refDF_Forest, inFieldFieldNowCast, "PercForest", "PropFiresForest", "Forest")
        if outVal[0] != "Success function":
            print("WARNING - Function define_IgnitionProportion failed - for 'Historic/Current - Forest Fire Season' - Exiting Script")
            exit()
        else:
            print("Success - Function define_IgnitionProportion - for 'Historic/Current - Forest Fire Season'")

            #Data Frame with the Ignition Potential Calculation
            dfwIgnitionForest = outVal[1]
            dfwIgnitionForest.reset_index(drop=True, inplace=True)

        # Define Percentiles - for Historic/Cur data, and using Grassland Fire Season
        field14Average = inFieldFieldNowCast + "_" + str(movingWindowsDay)
        outVal = define_IgnitionProportion(dfHistCur_wAvg, field14Average, refDF_NonForest, inFieldFieldNowCast, "PercNonForest", "PropFiresNonForest", "NonForest")
        if outVal[0] != "Success function":
            print("WARNING - Function define_IgnitionProportion failed - for 'Historic/Current - NonForest Fire Season' - Exiting Script")
            exit()
        else:
            print("Success - Function define_IgnitionProportion - for 'Historic/Current - NonForest Fire Season'")

            #Data Frame with the Ignition Potential Calculations both Forest and Grassland
            dfwIgnitionGrass = outVal[1]
            dfwIgnitionGrass.reset_index(drop=True, inplace=True)

        ############################################
        # Process Forest and Non-Forest
        ############################################

        #List to hold the processed dataframes
        appendDfList = []
        #Place holder - need to remove
        year = 2022
        #NonForest
        outFun = defineFireDangerRating(dfwIgnitionGrass, 'SiteName', 'FLFO', 'PercNonForest', year, year, 'Non-Forest', 'Mean', 'Year', 'na', 'DATE', 'no')
        if outFun[0] != "Success Function":
            print("WARNING - Function summarizeFireDangerRating failed - Exiting Script")
            exit()

        else:
            print("Success - Function summarizeFireDangerRating - Forested Vegetation - " + str(year) + " - Veg Type - NonForest")
            # Define the output dataframe
            appendDfList.append(outFun[1])

        #Forest
        outFun = defineFireDangerRating(dfwIgnitionForest, 'SiteName', 'FLFO', 'PercForest', year, year, 'Forest', 'Mean', 'Year', 'na', 'DATE', 'no')
        if outFun[0] != "Success Function":
            print("WARNING - Function summarizeFireDangerRating failed - Exiting Script")
            exit()

        else:
            print("Success - Function summarizeFireDangerRating - Forested Vegetation - " + str(year) + " - Veg Type - NonForest")
            # Define the output dataframe
            appendDfList.append(outFun[1])

        ################################
        # Append all processed Time Frames
        ################################
        outFun = appendFiles(appendDfList)
        if outFun[0].lower() != "success function":
            print("WARNING - Function appendFiles failed - Exiting Script")

        else:
            # Push non-ensemble output to a data frame
            dfallFiles = outFun[1]

            messageTime = timeFun()
            scriptMsg = "Success - Processing appended last Four Year Summaries - " + messageTime
            print(scriptMsg)
            logFile = open(logFileName, "a")
            logFile.write(scriptMsg + "\n")

        # Export output final dataframe
        currentDate = datetime.date.today()
        strCurrentDate = str(currentDate)
        outFull = outputFolder + "\\" + outName + "_" + strCurrentDate + ".csv"

        # Export
        dfallFiles.to_csv(outFull, index=False)

        scriptMsg = "Successfully processed Normal 1991-2020 - Fire Igintion Potential: " + outFull + " - " + messageTime
        print(scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")

    except:
        messageTime = timeFun()
        scriptMsg = "Exiting Error - FLFO_FireIgnitionRaw - " + messageTime
        print (scriptMsg)
        logFile = open(logFileName, "a")
        logFile.write(scriptMsg + "\n")

        traceback.print_exc(file=sys.stdout)
        logFile.close()

# Function to define the season
def defineSeason(x):
    try:
        winterList = [12, 1, 2]
        springList = [3, 4, 5]
        summerList = [6, 7, 8]
        fallList = [9, 10, 11]

        if x in winterList:
            outVal = "1-Winter-DJF"
        elif x in springList:
            outVal = "2-Spring-MAM"
        elif x in summerList:
            outVal = "3-Summer-JJA"
        elif x in fallList:
            outVal = "4-Fall_SON"
        else:
            outVal = SeasonNotDefinded - Error
        return outVal

    except:
        scriptMsg = "Exiting Error - defineSeason"
        print(scriptMsg)
        traceback.print_exc(file=sys.stdout)
        return "Failed"

    #Get count of Now Cast with Data
def numberNowCastDays3(inDf):

    try:
        # Change Deficit to Numeric - coerce so no data is NaN
        inDf['D (MM)'] = pd.to_numeric(inDf['D (MM)'], errors='coerce')

        # Get Count of records with deficit value
        futureRecordCount = inDf['D (MM)'].count()

        return "Success Function", futureRecordCount
    except:
        messageTime = timeFun()
        print("Error on numberNowCastDays3 Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'numberNowCastDays3'"

# Routine to get the summary statistic Value for the defined time period for the Fire Iginition Potential Fire Danger Rating
# Input: inFile - table full path with the Fire Ignition Potential Data to be summarizes
# inFileFieldAOA - Field in 'inFile that defines the site/AOA to be summarized
# inAOAWildcard - Syntax used to filter the 'inFileFieldAOA' records to be processed
# inFieldPerc - 'Field in 'inFile' with the derived 'Fire Ignition Potential' percentile rating (i.e. the Fire Ignition Potential Field to be summarized)
# startYear - first year to be processed
# endYear - end year to be processed
# coverType - defined the fire ignition potential relationship type to be applied ('Forest|'Non-Forest') - defines the Percentile values to be used for High, Medium,
# Low fire Danger Ratings

def defineFireDangerRating(inDf, inFileFieldAOA, inAOAWildcard, inFieldPerc, startYear, endYear, coverType, statistic, timeStep, rcp, timeField, NowCast):
    try:

        df3= inDf

        today = date.today()
        #Create datetime stamp
        today2 = pd.to_datetime(today)
        #strYearNow = int(today.strftime("%Y"))

        # #Subset to <= Current Year
        # # Create 'timeDate' field as datetime filed
        # df2["timeDate"] = pd.to_datetime(df2[inFileTime])
        # # Create the Day of Year Field
        # df2["DOY"] = df2['timeDate'].dt.dayofyear

        #Subset to <= Current date
        df3CurDate = df3[(df3['DATE'] <= today2)]

        df3CurDate["timeDate"] = pd.to_datetime(df3CurDate['DATE'])

        #Add Month Field
        df3CurDate['Mon'] = pd.DatetimeIndex(df3CurDate['timeDate']).month

        #Define Season
        df3CurDate['Season'] = df3CurDate.apply(lambda x: defineSeason(x['Mon']), axis=1)

        #Add Year Field
        df3CurDate['Year'] = pd.DatetimeIndex(df3CurDate['timeDate']).year

        #2)Apply the Ratings creating three fields High, Medium, Low - Binary
        # Define the Fire Danger Classes
        outFun = fireDangerCategories(coverType)
        highFireVal = outFun[0]
        mediumFireVal = outFun[1]

        # Apply the Binary Fire Ratings High, Medium, Low as new fields evaluated against the 'inFieldPerc' percentile field
        df3CurDate['High'] = np.where((df3CurDate[inFieldPerc] > highFireVal), 1, 0)
        df3CurDate['Medium'] = np.where((df3CurDate[inFieldPerc] <= highFireVal) & (df3CurDate[inFieldPerc] > mediumFireVal), 1, 0)
        df3CurDate['Low'] = np.where((df3CurDate[inFieldPerc] <= mediumFireVal), 1, 0)

        # Historic 1991-2020
        # Create pandas data time value with a UTC for subsetting
        startYearDT = pd.to_datetime('19910101', format='%Y%m%d')
        endYearDT = pd.to_datetime('20201231', format='%Y%m%d')
        # Change Date back to Datetime field
        #df3CurDate.reset_index(inplace=True)
        #df3CurDate['Date'] = pd.to_datetime(df_HistoricInterval['Date'], utc='None', errors='coerce')
        df3_HistoricInterval_1991_2020 = df3CurDate.loc[(df3CurDate['timeDate'] >= startYearDT) & (df3CurDate['timeDate'] < endYearDT)]

        # Add Cover Type
        df3_HistoricInterval_1991_2020.insert(loc=0, column='CoverType', value=coverType)

        #If Forest Cover Type change the 'SiteName' to 'FLFOGrass_1_Forest' to facilate WildCard usage in the Summarize Script.
        if coverType == "Forest":
            df3_HistoricInterval_1991_2020['SiteName'] = 'Forest'

        return "Success Function", df3_HistoricInterval_1991_2020

    except:
        messageTime = timeFun()
        print("Error on summarizeFireDangerRating Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'summarizeFireDangerRating'"

# Append Files in list to .csv file
def appendFiles(appendList):
    try:

        for count, file in enumerate(appendList):

            dfLoop = appendList[count]
            if count == 0:  # Make new dfallFiles

                dfallFiles = dfLoop

            # Append dfLoop to dfallFiles
            else:

                #dfallFiles = dfallFiles.append(dfLoop, ignore_index=True, verify_integrity=True)
                dfallFiles = dfallFiles._append(dfLoop, ignore_index=True, verify_integrity=True)

            del dfLoop

        return "Success function", dfallFiles

    except:
        messageTime = timeFun()
        print("Error on appendFiles Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'appendFiles'"

# Function summarizes the input DataFrame, defined field, for the desired timestep (e.g. Yearly), and defined summary statistic
# output is a DataFrame with the Rasample Summary Statistic
def resampleSummarize(inDataFrame, processField, timeStep, sumStat):
    try:

        # Calculate the Resample and Summary Statistic
        if sumStat == 'Sum' and timeStep == 'Annual':

            outSumSeries = inDataFrame[processField].resample(rule='AS').sum()

        else:
            print("'Time Step' variable - " + timeStep + " - or Summary Statistic - " + sumStat + " combo is not defined' - existing script Failed")
            sys.exit("'Time Step' variable - " + timeStep + " - or Summary Statistic - " + sumStat + " combo is not defined' - existing script Failed")

        outResampleDf = outSumSeries.to_frame().reset_index()

        # Define the derived Output Summary field name
        outFieldName = processField + "_" + sumStat
        # Export the Output Summary Series to a DataFrame
        outResampleDf.rename(columns={processField: outFieldName}, inplace=True)

        return "Success Function", outResampleDf

    except:

        print("Error on resampleSummarize Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'resampleSummarize'"

# Function defines the fire danger percentile breaks for High, Medium and Low by Cover Type Forest or Non-Forest.  See Figure 6 Thoma et. al. 2020
def fireDangerCategories(coverType):
    try:

        if coverType == 'Forest':
            #Thoma et. al. 2020 Fire Ignition Model Version 1.0 Southern Rockies Equation.
            #highFire = 86
            #mediumFire = 65

            #Steve Huysman 2023 Fire Ignition Model Version 2.0 Fire Order Model for the Southern Rockies.
            highFire = 84
            mediumFire = 53

        elif coverType == 'Non-Forest':
            # Thoma et. al. 2020 Fire Ignition Model Version 1.0 Southern Rockies Equation.
            # highFire = 90
            # mediumFire = 73

            # Steve Huysman 2023 Fire Ignition Model Version 2.0 Fire Order Model for the Southern Rockies.
            highFire = 84
            mediumFire = 51

        else:
            print("'coverType' variable is not defined as 'Forest' or 'Non-Forest' - existing script Failed")
            sys.exit("'coverType' variable is not defined as 'Forest' or 'Non-Forest' - existing script Failed")

        return highFire, mediumFire

    except:
        messageTime = timeFun()
        print("Error on fireDangerCategories Function - " + messageTime)
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'fireDangerCategories'"

#Functions Below
# Function to Get the Date/Time
def timeFun():
    from datetime import datetime
    b = datetime.now()
    messageTime = b.isoformat()
    return messageTime

# Function Creates a new DataFrame from the defined Columns/Series in a pre-existing dataframe
def select_columns(data_frame, column_names):
    new_frame = data_frame.loc[:, column_names]
    return new_frame

def processGridMetStation(serviceURL,siteName):
    try:

        #Import the data from climate analyzer
        workCSVFile = workspace + "\\gridmetData_wWB.csv"
        # Export Rest pull to .csv
        urlretrieve(serviceURL, workCSVFile)
        # Import work CSV file to dataframe  - Skipping first 5 head rows, and skipping footer rows - this should be dynamic
        # stop import at the first 'Text Line?'
        nowCastDf = pd.read_csv(workCSVFile, skiprows=5, engine='python')

        # Clean-up Dataframe
        # Trim white space from Field Names:
        nowCastDf.columns = nowCastDf.columns.str.strip()
        # Delete no data nan column being created on strip
        nowCastDf.drop([""], axis=1, inplace=True)
        # Trim Leading White Space from DATE
        nowCastDf['DATE'] = nowCastDf['DATE'].str.strip()

        #Convert 'DATE' field to Date Time now that text values have been removed
        #nowCastDf['DATE'] = pd.to_datetime(nowCastDf['DATE'], infer_datetime_format=True)
        nowCastDf['DATE'] = pd.to_datetime(nowCastDf['DATE'], format="%m/%d/%Y", errors='coerce')

        # currentDate = pd.to_datetime(currentDate, infer_datetime_format=True)
        # currentDatePlus60 = pd.to_datetime(currentDatePlus60, infer_datetime_format=True)
        # #Subset to the Current Date Plus 60 records
        # nowCastDfSubset = dfPlus60[(dfPlus60['DATE'] >= currentDate) & (dfPlus60['DATE'] <= currentDatePlus60)]

        # Add SiteNameField
        nowCastDf.insert(1, "SiteName", siteName, True)

        return "Success function", nowCastDf

    except:

        messageTime = timeFun()
        print("Error on define_ReferenceList Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'processGridMetStation'"

#Function - Subsets the Climate Analyzer Rest data to two data Frames:
# Outputs:
# nowCastDfSubset - Datframe with only data current date plus 60 days to be used for now cast summarization
# dfPlus60 - Dataframe from 1981 through todays date plus 60 days for the now cast
def processNowCast(serviceURL,siteName):
    try:

        # Import the data from climate analyzer
        workCSVFile = workspace + "gridmetData_wWB.csv"
        # Export Rest pull to .csv
        urlretrieve(serviceURL, workCSVFile)
        # Import work CSV file to dataframe  - Skipping first 5 head rows, and skipping footer rows - this should be dynamic
        # stop import at the first 'Text Line?'
        nowCastDf = pd.read_csv(workCSVFile, skiprows=5, engine='python')

        # Clean-up Dataframe
        # Trim white space from Field Names:
        nowCastDf.columns = nowCastDf.columns.str.strip()
        # Delete no data nan column being created on strip
        nowCastDf.drop([""], axis=1, inplace=True)
        # Trim Leading White Space from DATE
        nowCastDf['DATE'] = nowCastDf['DATE'].str.strip()

        #Define Current Date plus 60
        currentDate = datetime.date.today()
        # Add 60 days to current datatime
        currentDatePlus60 = currentDate + datetime.timedelta(days=60)
        # Define as String
        stringFormat = currentDatePlus60.strftime('%m/%d/%Y')

        # Find Record with the plus 60 date
        indexDf = nowCastDf.loc[nowCastDf['DATE'] == stringFormat]
        indexSubSet = indexDf['INDEX'].astype(int)

        # Get the Index Value
        indexSubSet = indexSubSet.index
        # Extract Index Value and add 1 - will be used to subset
        indexValuePlus1 = indexSubSet[0] + 1

        # Subset Records to plus 60 only removing the monthly summaries Mike included in the rest pull - this will be retained for historical calculations
        dfPlus60 = nowCastDf[0:indexValuePlus1]

        #Convert 'DATE' field to Date Time now that text values have been removed
        dfPlus60['DATE'] = pd.to_datetime(dfPlus60['DATE'], infer_datetime_format=True)

        # currentDate = pd.to_datetime(currentDate, infer_datetime_format=True)
        # currentDatePlus60 = pd.to_datetime(currentDatePlus60, infer_datetime_format=True)
        # #Subset to the Current Date Plus 60 records
        # nowCastDfSubset = dfPlus60[(dfPlus60['DATE'] >= currentDate) & (dfPlus60['DATE'] <= currentDatePlus60)]

        del(nowCastDf)
        # Add SiteNameField
        dfPlus60.insert(1, "SiteName", siteName, True)

        return "Success function", dfPlus60
    except:

        messageTime = timeFun()
        print("Error on define_ReferenceList Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'define_ReferenceList'"

###############################################################################################################
#Function 1. Routine to Derive the Historical 1984-2015 Fire Period Water Deficit full list for the Fire Period
###############################################################################################################
# startDate - First date of fire period across the 1984-2015 fire period
# enddate - Last date of fire period across the 1984-2015 fire period
# stationData - station data being processed
# inFileField - Parameter being evaluated (e.g. Deficit)
# inFileTime - Parameter defining the time field

# Output - pandas data series (i.e. 1D) with the input variable (e.g. Deficit) trimmed to the defined fire year range
def define_ReferenceList(startDate, endDate, refYearStartDate, refYearEndDate, stationData, inFileField, inFileTime):
    try:

        'Make data frame from Input List'
        #df = pd.read_csv(stationData)
        df = stationData
        selected_columns = []
        selected_columns.append(inFileTime)
        selected_columns.append(inFileField)

        # Function to take the defined Columns and Export the Columns to a new Pandas Data Frame
        df2 = select_columns(df, selected_columns)  # Creating new data frome with only the date and time series field being processed

        # Create 'timeDate' field as datetime filed
        df2["timeDate"] = pd.to_datetime(df2[inFileTime])
        # Create the Day of Year Field
        df2["DOY"] = df2['timeDate'].dt.dayofyear

        #Select where Day of Year >=Start Date and <= End Date
        outDfFireYear1 = df2[(df2['DOY'] >= startDate) & (df2['DOY'] < endDate)]

        #Select Second subset fire years of interest Year >= refYearStart Date and <= refYearEnd
        #Convert Inputs to DateTime parameters
        refYearStartDT = pd.to_datetime(refYearStartDate, format='%m/%d/%Y')
        refYearEndDT = pd.to_datetime(refYearEndDate, format='%m/%d/%Y')

        outDfFireYear = outDfFireYear1[(outDfFireYear1['timeDate'] >= refYearStartDT) & (outDfFireYear1['timeDate'] <= refYearEndDT)]

        #Reset Index to 0
        outDfFireYear.reset_index(drop=True, inplace = True)

        del outDfFireYear1
        return "Success function", outDfFireYear
    except:

        messageTime = timeFun()
        print("Error on define_ReferenceList Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'define_ReferenceList'"

#####################################################################
#Function 2. Define the 14 Day Moving Average Values by input Dataset
#####################################################################
# Input:
# - inDataSet: Dataset which is being evaluated (e.g. Historical/Current/NowCast/Future Projects)
# - movingWindowDays: Moving window number of days prior to the date to be derived (Default will be 14 day prior)
# - fieldToAverage: Field in the 'inDateSet' being averaged
# Output - New Field -'MovingAverage_{movingWindowDays}' in output dataframe with the moving window Average

# Note in Rolling/Moving window average min_periods is set to 1 (i.e. only 1 day of data needed)
def define_MovingWindowAverage(inDataSet, movingWindowDays, fieldToAverage):
    try:

        # Make data frame from Input
        #df = pd.read_csv(inDataSet)

        outField = fieldToAverage + "_" + str(movingWindowDays)
        # Use Rolling in dataframe to calculate the Moving Window Average
        #df[outField] = df[fieldToAverage].rolling(movingWindowDays, min_periods=1, win_type=None).mean()
        inDataSet[outField] = inDataSet[fieldToAverage].rolling(movingWindowDays, min_periods=1, win_type=None).mean()

        #Reset Index to Zero
        inDataSet.reset_index(drop=True, inplace = True)

        return "Success function", inDataSet
    except:

        messageTime = timeFun()
        print("Error on define_MovingWindowAverage Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'define_MovingWindowAverage'"

#####################################################################
#Function 3. Routine to Derive the Fire Ignition Potential Proportion
#Referenced against the trimmed dataframe (i.e. Forest and Non Forest Fire Season 1984-2015
# Applying the Fire Ignition Potential Relationship in Thoma et. al. 2020 Global Ecology Conservation
#####################################################################
#Non-Forest Equation where percentile is the percentile and f is the percent of historic fires (1984-2015) middle rockies
#that has ignited under the percentile dryness conditions
def nonforest_equation(percentile):
    #f = 0.047 * np.e**(0.075*percentile)  #Thoma et. al. 2020 Fire Ignition Model Version 1.0 Southern Rockies Equation.
    f = 0.0119265 * np.e ** (4.192916 * percentile)  #Steve Huysman 2023 Fire Ignition Model Version 2.0 Fire Order Model for the Southern Rockies.
    return f

#Forest Equation where percentile is the percentile and f is the percent of historic fires (1984-2015) middle rockies
#that has ignited under the percentile dryness conditions
def forest_equation(percentile):
    #f = 0.368 * np.e**(0.055*percentile)  #Thoma et. al. 2020 Fire Ignition Model Version 1.0 Southern Rockies Equation.
    f = 0.0095308 * np.e ** (4.4556479 * percentile)  #Steve Huysman 2023 Fire Ignition Model Version 2.0 Fire Order Model for the Southern Rockies.

    return f

#Subset to only NowCast Data - passing data frames that already have fire ignition potential calculated
def subsetToNowCast(inDf):

    try:

        # Define Current Date plus 60
        currentDate = datetime.date.today()
        # Add 60 days to current datatime
        currentDatePlus60 = currentDate + datetime.timedelta(days=60)
        # Define as String
        stringFormat = currentDatePlus60.strftime('%Y-%m-%d')

        # Find Record with the plus 60 date
        indexDf = inDf.loc[inDf['DATE'] == stringFormat]
        indexSubSet = indexDf['INDEX'].astype(int)

        # Get the Index Value
        # Get the Index Value
        indexSubSet = indexSubSet.index
        # Extract Index Value and add 1 - will be used to subset - not being used
        indexValuePlus1 = indexSubSet[0] + 1

        # Convert 'DATE' field to Date Time now that text values have been removed
        inDf['DATE'] = pd.to_datetime(inDf['DATE'], infer_datetime_format=True)

        currentDate = pd.to_datetime(currentDate, infer_datetime_format=True)
        currentDatePlus60 = pd.to_datetime(currentDatePlus60, infer_datetime_format=True)

        # subset dataframe to >= currentDateTime and <= curerntDateTimePlus60
        nowCastDfSubset = inDf[(inDf['DATE'] >= currentDate) & (inDf['DATE'] <= currentDatePlus60)]

        # Force Deficit Field to Numeric makeing all non Numeric a Pandas 'NAN'
        nowCastDfSubset['D (MM)'] = pd.to_numeric(nowCastDfSubset['D (MM)'], errors='coerce')

        #Retain only records with Data
        nowCastDfSubset['IsNull'] = nowCastDfSubset['D (MM)'].isnull()

        nowCastDfSubset2 = nowCastDfSubset.loc[(nowCastDfSubset['IsNull'] == False)]

        #Delete 'IsNull' field
        nowCastDfSubset2.drop(['IsNull'], axis=1, inplace=True)

        del nowCastDfSubset
        return "Success Function", nowCastDfSubset2

    except:

        messageTime = timeFun()
        print("Error on subsetToNowCast Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'subsetToNowCast'"

# Input:
# 1. processDF - Data frame being processed with 14 Day Moving Averages
# 2. field14Average - Field in 'processDF' with the 14 Day Moving Averages
# 3. refDF - Reference DataFrame that has been trimmed to the fire year only values
# 4. refDFField - Field in 'refDF' being checked (e.g. 'inFileField' or 'inFileFieldProj')
# 5. percentileField - output Percentile field in the 'processDF' (e.g. 'PercentileForest','PercentileNonForest')
# 6. ignitionProportionField - output Fire Ignition Proportion field in the 'processDF' (i.e. the Proportion of Fires for either Forest or Non-Forest)
# 7. equationType - defines if Forest or Non-Forest Equations (i.e. function - 'forest_equation' or 'Non-Forest' will be used to
#               derive the Fire Ignition Proportion Value

# Fire Iginition Proportion Equation is derived from Thoma et. al. 2020 Global Ecology Conservation.
# Output - 'processDF' Data Frame with percentile fields and Fire Ignition Proportion field

def define_IgnitionProportion(processDF, field14Average, refDF, refDFField, percentileField, ignitionPropField, equationType):
    try:

        # Field with Averages
        #processField = field14Average + "_" + str(movingWindowsDay)
        processField = field14Average
        # Add Percentile Field
        lastColumn = processDF.shape[1]
        processDF.insert(lastColumn, percentileField, "")
        # Change 'PercentileField' Data Type to Float
        processDF[percentileField] = pd.to_numeric(processDF[percentileField], errors='coerce')

        # Add 'ignitionPropField' (i.e. Ignition Potential Field)
        lastColumn = processDF.shape[1]
        processDF.insert(lastColumn, ignitionPropField, "")
        # Change 'PercentileField' Data Type to Float
        processDF[ignitionPropField] = pd.to_numeric(processDF[ignitionPropField], errors='coerce')

        #Converting the reference dataframe deficit field to numeric
        refDF[refDFField] = pd.to_numeric(refDF[refDFField], errors='coerce')

        # Number of records in the reference dataset - (i.e denominator) - Only interesting if count for the Respective Fire Season
        # By Type Forest or Grassland(i.e. 1984-2015)
        totalRecordsFireSeason = refDF.shape[0]

        # Records to iterate through in processDF
        totalRecordsProcess = processDF.shape[0]

        rowRange = range(0, totalRecordsProcess)
        # Loop thru the processDF dataframe and assign a percentile per record
        for row in rowRange:

            rowSeries = processDF.iloc[row]
            checkValue = rowSeries.get(processField)

            # Get Count of records in the 'refDF' (i.e. Reference DF) and Field that are greater than the input value in the 'processDF'
            n = np.sum(refDF[refDFField] < checkValue)

            # Calculate Percentile
            percentile = (n / totalRecordsFireSeason) * 100

            # Assign the percentile value to the processDf and 'percentileField'
            processDF.at[row, percentileField] = percentile

            # Calculation the Proportion of Fire Ignition
            if equationType == "Forest":
                outIgnitionPerc = forest_equation(percentile)

            if equationType == "NonForest":
                outIgnitionPerc = nonforest_equation(percentile)

            # Assign IgnitionPercentage to the 'processDF' dataframe
            processDF.at[row, ignitionPropField] = outIgnitionPerc

        return "Success function", processDF
    except:

        messageTime = timeFun()
        print("Error on define_IgnitionProportion Function ")
        traceback.print_exc(file=sys.stdout)
        return "Failed function - 'define_IgnitionProportion'"

if __name__ == '__main__':

    # Analyses routine ---------------------------------------------------------
    main()


import datetime
import itertools
import json
import os
from functools import cache
from pathlib import Path

from connexion import FlaskApp
from connexion.resolver import RelativeResolver
from dotenv import load_dotenv
from flask import request
from flask_cors import CORS
from pony.orm import Database, set_sql_debug, db_session

from src.models import define_entities_game, define_entities_meta, define_entities_user, GameTypeEnum, GameLengthEnum, \
    GroupSizeEnum, GroupNeedEnum, define_entities_stats, define_entities_api
from src.services.enforcedefaults import validator_remap

# Load project root and constants from dotenv file
ROOT = Path(__file__).parent.parent
dotenv_path = ROOT.joinpath('.env')
load_dotenv(dotenv_path)

@cache
def get_db(users_db=False):
    """Bind Pony ORM to database
    This acts like a singleton because of caching.
    """
    if users_db:
        credentials = json.loads(os.environ.get("DB_USERS_CONNECT"))
    else:
        credentials = json.loads(os.environ.get("DB_CONNECT"))

    if not os.environ.get("FLASK_DEBUG"):  # pragma: no cover / production
        database = Database(provider='mysql', **credentials)
    elif os.environ.get('FLASK_TESTING'):
        database = Database(provider='sqlite', filename=f':memory:', create_db=True)
    else:  # pragma: no cover / development
        database = Database(provider='sqlite', filename=f'{credentials["db"]}.sqlite', create_db=True)

    if users_db:
        define_entities_user(database)
    else:
        define_entities_game(database)
        define_entities_meta(database)
        define_entities_stats(database)
        define_entities_api(database)

    database.generate_mapping(create_tables=True)
    set_sql_debug(False)
    return database

@cache
def get_app():
    """Connexion Wrapper App
    Connexion creates a wrapper around a Flask app, through which all requests are passed.
    The Connexion wrapper app is called by wsgi.py and pytest.
    You can access the Flask app with `app.app.properties`.
    """

    # Todo: Sentry integration
    # sentry_sdk.init(
    #     dsn=os.environ.get('SENTRY_DSN'),
    #     integrations=[FlaskIntegration()],
    #     # Reduce this for production, maybe 0.25
    #     traces_sample_rate=0.25,
    #     release=get_project_version(),
    #     environment=Config().SENTRY
    # )

    swagger_options = {
        'swagger_url': '/',
        'swagger_ui_config': {
            'operationsSorter': 'method',
            'tagsSorter': 'alpha',
        }
    }
    connexion_app = FlaskApp(__name__, specification_dir=ROOT, options=swagger_options)
    CORS(connexion_app.app)
    connexion_app.add_api(
        'openapi.yml',
        resolver=RelativeResolver('src.views'),
        validate_responses=True,
        validator_map=validator_remap,
    )
    connexion_app.app.config.from_prefixed_env()

    @connexion_app.app.after_request
    @db_session
    def activity_tracking(response):
        from src.services import tracking
        if response.status_code in [200, 201]:
            session = tracking.get_session(request)
            tracking.add_request(session, request, response)
        return response

    return connexion_app

@db_session
def run_startup_tasks(db):  # pragma: no cover
    if hasattr(db, 'User'):
        if db.User.get(login="admin"):
            return
        startup_users_db(db)
    else:
        if db.APIInfo.get(name="aleleio-api"):
            return
        startup_api_info(db)
        startup_games_db(db)

def startup_users_db(udb):  # pragma: no cover
    for user in [os.environ.get("USER_ADMIN"), os.environ.get("USER_WEB"), os.environ.get("USER_ANDROID"), os.environ.get("USER_IOS")]:
        udb.User(**json.loads(user))

def startup_games_db(db):
    """Seed database with category enums if not already present
    """
    for item in GameTypeEnum:
        db.GameType(slug=item.value, full=item.full)
    for item in GameLengthEnum:
        db.GameLength(slug=item.value, full=item.full)
    for item in GroupSizeEnum:
        db.GroupSize(slug=item.value, full=item.full)
    for item in GroupNeedEnum:
        db.GroupNeed(slug=item.value, full=item.full)

    for item in itertools.chain(GameTypeEnum, GameLengthEnum, GroupSizeEnum, GroupNeedEnum):
        db.QueryParam(slug=item.value)

def startup_api_info(db):
    from src.services.api_info import get_project_version

    for project in ["aleleio-api", "aleleio-web", "teambuilding-games"]:
        params = dict(
            name=project,
            version=get_project_version(project),
            last_commit=datetime.datetime.utcnow(),
            url=f"https://github.com/aleleio/{project}"
        )
        db.APIInfo(**params)


import re
import csv

class XML_Syntax_Error(Exception):
    pass

class XML_Definition_Error(Exception):
    pass

class xml_reader:
    def __init__(self,name = "Root Element"):
        self.root_element = name                            #Name of the root element
        self.element_stack = []                             #Stack which contains name of Elements
        self.main_dict = {}                                 #Nested dict with element->{Sub_element}

    def find_element(self,element):                         #Finds out the element and returns the dictionary associated with it
        for i in self.main_dict:
            if i == element:
                return self.main_dict[i]

    def element_stacker(self,name):                         #A list of all the elements present
        self.element_stack.append(name)

    def add_dict(self,element):                             #Nested Dictionary keys as Element -> {Sub_elements}
        self.main_dict[element] = {}

    def add_dict_value(self,element,key,value):            #Adding Values to the nested dictionary
        self.main_dict[element][key] = value

def main():
   print("""
Hello Developer!, This is py4xml
In order to use my functionality try "import py4xml". If that didnt worked then try using "pip install py4xml".

To know more visit:
Readme : https://github.com/MasterANK/py4xml/blob/main/README.md
Github : https://github.com/MasterANK/py4xml
PyPI : https://pypi.org/project/py4xml/
""")

def read_xml(f):
    data = f.readlines()
    xml_file = xml_reader()
    element_flag = True     #Will be false if element found
    attribute_flag = False  #Will be true if Attr found
    header_flag = True      #Will be False if header found

    for i in data:
        a = re.fullmatch(r"<(\w+)>",i.strip())
        b = re.fullmatch(r"<(\w+) (\w+)=[\"|\']([^<>]+)[\"|\']>", i.strip())
        c = re.fullmatch(r"<(\w+)>([^<>]+)</(\w+)>",i.strip())
        d = re.fullmatch(r"</(\w+)>",i.strip())
        if a:                                                       #Init Header Check
            if header_flag:
                header_flag = False
                xml_file.root_element = a.group(1)
            else:
                if header_flag != False:
                    raise XML_Definition_Error("Root Element not defined")

                if element_flag:                                    #To resolve the Bug where tag closing error never occurs
                    element_flag = False
                else:
                    raise XML_Syntax_Error(element+" is not closed properly")

                element = a.group(1)
                xml_file.element_stacker(element)
                xml_file.add_dict(element)

        elif b:                                                     #Element with attributes check
            if header_flag:
                header_flag = False
                xml_file.root_element = b.group(1)
            else:
                if header_flag != False:
                    raise XML_Definition_Error("Root Element not defined")

                if element_flag:                                    #To resolve the Bug where tag closing error never occurs
                    element_flag = False
                else:
                    raise XML_Syntax_Error(element+" is not closed properly")

                attribute_flag = True
                value =  b.group(3)
                element = str(b.group(1))+"_"+str(value) ; category = 'category'+'_'+str(b.group(2))
                xml_file.element_stacker(element)
                xml_file.add_dict(element)
                xml_file.add_dict_value(element,category,value)

        elif c:                                                      #Sub_elements Check + Closing
            if c.group(1) != c.group(3):
                raise XML_Syntax_Error(c.group(1)+" is not closed properly")
            if element_flag:
                raise XML_Syntax_Error("Element is missing or not defined")
            xml_file.add_dict_value(element,c.group(1),c.group(2))

        elif d:                                                      #Closing Root_Element,Element Check
            if attribute_flag:
                if str(d.group(1))+"_"+str(value) == element:
                    attribute_flag = False
                    element_flag = True
            elif  d.group(1) == element:
                element_flag = True

            if d.group(1) == xml_file.root_element:
                header_flag = True

    if not header_flag:
        raise XML_Syntax_Error(xml_file.root_element + " is not closed properly")
    if not element_flag:
        raise XML_Syntax_Error(element + " is not closed properly")

    return xml_file

def dict_write_xml(root_element ,data : dict,  write_f : object):       #Takes dictionary and write data to xml file
    write_f.write("<"+str(root_element)+">\n")
    for i in data:
        write_f.write("  <"+str(i)+">\n")
        element_flag = False
        for j in data[i]:
            exp = "    <"+str(j)+">"+str(data[i][j])+"</"+str(j)+">\n"
            write_f.write(exp)
        else:
            if element_flag == False:
                element_flag == True
                write_f.write("  </"+str(i)+">\n")
    else:
        write_f.write("</"+str(root_element)+">\n")

def extend_xml(data, write_f):      #Extend an extensible file
    file = read_xml(write_f)
    write_f.seek(0)
    file_data = file.main_dict
    file_data.update(data)
    dict_write_xml(file.root_element,file_data,write_f)

def csv_to_xml(root_element,csv_f,key,xml_f):       #Convert CSV -> Dict -> XML
    csv_data = csv.DictReader(csv_f)
    holder_data = {}
    data = {}
    for i in csv_data:
        for j in i:
            if j == str(key):
                if i[j].isidentifier():
                    data[i[j]] = i
                else:
                    holder_data = i.copy()
                    i[j] = re.sub(r"[^a-zA-Z0-9\._\-]","_",i[j])
                    data[i[j]] = holder_data
    dict_write_xml(root_element,data,xml_f)

def xml_to_csv(xml_f,csv_f,elementcol = None):      #Convert XML -> Dict -> CSV
    xml_obj = read_xml(xml_f)
    xml_data = xml_obj.main_dict
    elements = xml_data[xml_obj.element_stack[0]].keys()
    elements = list(elements)
    elementcol_flag = False

    if elementcol != None and elementcol not in elements:
        elementcol_flag = True
        elements.insert(0,elementcol)

    csvwriter = csv.DictWriter(csv_f,elements)
    csvwriter.writeheader()

    for i in xml_data:
        if elementcol_flag:
            xml_data[i][elementcol] = i
        csvwriter.writerow(xml_data[i])

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# coding: utf-8
# File: question_parser.py
# Author: lhy<lhy_in_blcu@126.com,https://huangyong.github.io>
# Date: 18-10-4

from answer_search_KG import *

class QuestionPaser:

    def __init__(self):
        self.searcher = AnswerSearcher()

    '''构建实体节点'''
    def build_entitydict(self, args):
        entity_dict = {}
        for arg, types in args.items():
            #print(args.items())
            for type in types:
                if type not in entity_dict:
                    entity_dict[type] = [arg]
                else:
                    entity_dict[type].append(arg)
        return entity_dict

    '''解析主函数'''
    def parser_main(self, res_classify):
        args = res_classify['args']
        #print("heliu@@@@@@@@@@@@@@@@@@@@@@")
        #print(args)
        entity_dict = self.build_entitydict(args)
        #print("heliu%%%%%%%%%%%%%%%%%%%%%%%%%%")
        #print(entity_dict)
        question_types = res_classify['question_types']
        sqls = []
        for question_type in question_types:
            sql_ = {}
            sql_['question_type'] = question_type
            sql = []
            if 'classify' in question_type:
                sql = [question_type]  # FIXME:useless, can store other information maybe
            # '''deal with property'''
            elif '###' in question_type:
                pair = question_type.split('###')
                if len(pair) == 3:
                    sql = ["n.%s %s %s" % (pair[0], pair[1], pair[2])]
                else:
                    sql = ["n.%s %s %s and n.%s %s %s" % (pair[0], pair[1], pair[2], pair[3], pair[4], pair[5])]
            # '''deal with relationship'''
            elif '-->' in question_type:
                pair = question_type.split('-->')
                if pair[0] == 'dom_search_components':
                    sql = ["(n)-[:rels_component_city]->(city:components_city {China: %s})" % pair[1]]
                elif pair[0] == 'type_search_components':
                    sql = ["(n)-[:rels_component_comCats]->(cat:components_cats {type:'%s'})" % pair[1]]

            elif question_type == 'cats_search_components':
                label = 'components_cats'
                sql = ["(n)-[:rels_component_comCats]->(cat:%s {name:'%s'})" % (label, entity_dict.get(label)[0])]
            elif question_type == 'city_search_components':
                label = 'components_city'
                sql = ["(n)-[:rels_component_city]->(city:%s {name: '%s'})" % (label, entity_dict.get(label)[0])]
            elif question_type == 'org_search_components':
                label = 'components_organizations'
                sql = ["(n)-[:rels_component_organization]->(org:%s {name: '%s'})" % (label, entity_dict.get(label)[0])]

            # if question_type == 'weapons_description':
            #     sql = self.sql_transfer(question_type, entity_dict.get('weapons'))
            #
            # elif question_type == 'weapon_times':
            #     sql = self.sql_transfer(question_type, entity_dict.get('weapons'))
            #
            # elif question_type == 'weapon_sites':
            #     sql = self.sql_transfer(question_type, entity_dict.get('weapons'))
            #
            # elif question_type == 'weapon_organizations':
            #     sql = self.sql_transfer(question_type, entity_dict.get('weapons'))
            #
            # elif question_type == 'weapon_types':
            #     sql = self.sql_transfer(question_type, entity_dict.get('weapons'))
            #
            # elif question_type == 'system_fault_deal':
            #     sql = self.sql_transfer(question_type, entity_dict.get('systems_entity'))
            #
            # elif question_type == 'fault_deal':#如果输入的问题是故障-处理措施类型，返回有此故障的设备
            #     sql = self.sql_transfer(question_type, entity_dict.get('malfunctions_entity'))
            #
            # elif question_type == 'unit_fault_deal':#如果输入的问题是部件实体-故障-处理措施类型，返回部件+故障实体共同指向的解决方案
            #     sql = self.sql_transfer(question_type, entity_dict.get('units_entity')+entity_dict.get('malfunctions_entity'))
            #
            # elif question_type == 'unit_comCats_search':#如果输入的问题是故障-处理措施类型，返回有此故障的设备
            #     sql = self.sql_transfer(question_type, entity_dict.get('units_entity'))
            #
            # elif question_type == 'comCats_malfunction_search':#如果输入的问题是故障-处理措施类型，返回有此故障的设备
            #     sql = self.sql_transfer(question_type, entity_dict.get('components_cats'))

            if sql:
                sql_['sql'] = sql

                sqls.append(sql_)

        return sqls

    '''针对不同的问题，分开进行处理'''
    def sql_transfer(self, question_type, entities):
        if not entities:
            return []

        # 查询语句
        sql = []
        # 查询武器信息
        if question_type == 'weapons_description':
            sql = ["MATCH (m:weapons) where m.name = '{0}' return m.name, m.desc, m.max_V, m.max_L".format(i) for i in entities]

        # 查询武器首发时间
        elif question_type == 'weapon_times':
            sql = ["MATCH (m:weapons)-[r:first_time_use]->(n:weapon_times) where m.name = '{0}' return m.name, r.name, n.name".format(i) for i in entities]

        # 查询武器产国
        elif question_type == 'weapon_sites':
            sql = ["MATCH (m:weapons)-[r:weapon_country]->(n:weapon_sites) where m.name = '{0}' return m.name, r.name, n.name".format(i) for i in entities]

        # 查询武器产商
        elif question_type == 'weapon_organizations':
            sql = ["MATCH (m:weapons)-[r:weapon_organization]->(n:weapon_organizations) where m.name = '{0}' return m.name, r.name, n.name".format(i) for i in entities]

        # 查询武器类型
        elif question_type == 'weapon_types':
            sql = ["MATCH (m:weapons)-[r:belongs_to]->(n:weapon_types) where m.name = '{0}' return m.name, r.name, n.name".format(i) for i in entities]

        elif question_type == 'system_fault_deal':
            sql = ["MATCH (m:systems_entity)-[r:rels_system_unit]->(n:units_entity) where m.name = '{0}' return n.name".format(i) for i in entities]

        elif question_type == 'fault_deal':
            sql = ["MATCH (m:units_entity)-[r:rels_unit_fault]->(n:malfunctions_entity) where n.name = '{0}' return m.name".format(i) for i in entities]

        elif question_type == 'unit_fault_deal':
            sql = ["MATCH (m:units_entity)-[r:rels_fault_deal]->(n:faultDeal_entity) where m.name = '{0}' and r.name = '{1}' return m.name".format(entities[0], entities[1])]

        elif question_type == 'unit_comCats_search':
            sql = ["MATCH (m:units_entity)-[r:rels_unit_comCats]->(n:components_cats) where m.name = '{0}' return n.name".format(i) for i in entities]

        elif question_type == 'comCats_malfunction_search':
            sql = ["MATCH (m:components_cats)-[r:rels_comCats_malfunction]->(n:malfunctions_entity) where m.name = '{0}' return n.name".format(i) for i in entities]

        #elif question_type == 'unit_comCats_search':
            #sql = ["MATCH (m:units_entity)-[r:rels_unit_comCats]->(n:components_cats) where m.name = '{0}' return n.name".format(i) for i in entities]

        return sql

if __name__ == '__main__':
    handler = QuestionPaser()


#!/usr/bin/env python
# coding: utf-8

# # Data Preprocessing

# ### Importing the libraries

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# ### Reading the dataset

# In[ ]:

dataset = pd.read_csv('startups.csv')
dataset

# In[ ]:

dataset.describe()

# In[ ]:

# Separate Independent variables (X) and dependent variable (y)

X = dataset.iloc[:, 0:4].values
y = dataset.iloc[:, 4].values

print(X,'\n\n', y)

# ### Missing values

# In[ ]:

# Replace missing values by the column/variable average (mean)
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.impute import SimpleImputer                            #import the SimpleImputer class
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')     #create the imputer object

imputer.fit(X[:, 0:3])                          #fit the object to the data
X[:, 0:3] = imputer.transform(X[:, 0:3])        #transform data

print(X)

# ### Encoding categorical variables

# In[ ]:

# Encoding independent variables
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.compose import ColumnTransformer       # import class
from sklearn.preprocessing import OneHotEncoder     # import class

ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(drop = 'first'), [3])], remainder='passthrough')
#create object

X = np.array(ct.fit_transform(X))                   # fit object to data and transform data

print(X)

# ### Feature scaling

# In[ ]:

# Adjusting the scales of independent variables
# Import Class --> Create Object --> Fit Object to Data --> Transform Data

from sklearn.preprocessing import StandardScaler   # import class
sc = StandardScaler()                              # create object
X = sc.fit_transform(X)                            # fit and transform

print(X)

# In[ ]:

df = pd.DataFrame(X)
df.describe()

# ### Splitting the dataset into the Training set and Test set

# In[ ]:

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# In[ ]:

print(X_train)

# # Artificial Neural Net (ANN) for Regression

# ### Training the Model

# In[ ]:

# We use ANNs.
# Can also use linear, polynomial, support vector, decision trees or random forest regression.

# Import Class --> Create Object --> Fit Object to Data --> Predict

import tensorflow as tf               # import class
from tensorflow import keras

regressor = keras.Sequential([
    tf.keras.layers.Dense(units = 20, input_dim=5, activation='relu'),
    tf.keras.layers.Dense(units = 20, activation='relu'),
    tf.keras.layers.Dense(units = 20, activation='relu'),
    tf.keras.layers.Dense(units = 10, activation='relu'),
    tf.keras.layers.Dense(units = 1, activation = 'linear')
])                                                         # create object with the required layers

regressor.compile(
              loss='mean_squared_error',
              optimizer=tf.keras.optimizers.RMSprop(0.01),
              batch_size=20,
              metrics=['mean_absolute_error']
)                                                          # compile object selecting options

regressor.fit(X_train, y_train, epochs= 500)               # fit object and decide the number of epochs

# ### Testing the Model

# In[ ]:

# make predictions for test data

y_pred = regressor.predict(X_test).flatten()    # predict (flatten() reshapes y_pred to a one-dimensional array)

err = abs(y_test - y_pred)
df = pd.DataFrame({'y_pred': y_pred, 'y_test': y_test, 'error': err})     # compare
df

# In[ ]:

# determine model accuracy on the test set

loss, mae = regressor.evaluate(X_test, y_test, verbose=0)
print('Mean Absolute Error =', mae)

# ### Graphing (Optional)

# In[ ]:

plt.bar(range(10), err)

# In[ ]:

plt.bar(range(10), y_test)

# ### Forecasting (Optional)

# In[ ]:

# What is the profit for
# a California startup spending 10,000 on R&D, 50,000 on Administration and 10,000 on Marketing
# What happens if they spend 100,000 on R&D?

Z = sc.transform([[0, 0, 10000, 50000, 10000]])     # scale data using .transform() method but NOT .fit_transform()

prediction = regressor.predict(Z)
print('Scaled features = ', Z, ', Forecast = ', prediction)

# In[ ]:


#!/usr/bin/env python3
"""Compose tweets about Firehouse films"""

import re
import json
import html
import random
from pathlib import Path
from typing import List, Dict, Any

from bs4 import BeautifulSoup

class Film:
    """Film metadata"""

    def __init__(self, data: Dict[str, Any]) -> None:
        self.title = html.unescape(data["title"])
        self.categories = data["categories"]
        self.firehouse_no = self.categories[0]["title"].split()[0][1:]
        self.url = data["url"].split("http://", maxsplit=1)[1]
        self.excerpt = html.unescape(no_html(data["excerpt"])).strip()
        self.content = data["content"]
        #self.firehouse_name = unescape(" ".join(data["categories"][0]["title"].split()[1:]))
        #self.img_url = data["thumbnail_images"]["large"]["url"]

    def __len__(self) -> int:
        return len(self.text()) - len(self.url) - len(self.video_url) + 2 * 23

    @property
    def awards(self) -> List[str]:
        """List of awards the film won"""
        winner: List[str] = []
        if len(self.categories) > 1:
            for i in range(1, len(self.categories)):
                winner.append(self.categories[i]["title"].lower())
        return winner

    @property
    def video_url(self) -> str:
        """URL to film host"""
        soup = BeautifulSoup(self.content, "html.parser")
        for link in soup.findAll("a"):
            url = link.get("href")
            if url.startswith("https://"):
                if "youtu" or "vimeo" in url:
                    return url.split("&", maxsplit=1)[0]
        return ""

    @property
    def humans(self) -> List[str]:
        """Twitter handles of filmmakers"""
        handles = []
        with (Path(__file__).parent.parent / "data/handles.json").open("r") as handles_fd:
            handle_map = json.load(handles_fd)
        for name, handle in handle_map.items():
            if name.lower() in self.excerpt.lower():
                handles.append(handle)
        return handles

    def text(self) -> str:
        """Compose a tweet about a film"""
        tweet = ""
        if self.awards:
            tweet = "Winner of " + ", ".join(self.awards) + " in "
        else:
            tweet = "From "
        tweet += f"Firehouse {self.firehouse_no}, “{self.title}”"
        if self.humans:
            tweet += " by " + ", ".join(["@" + human for human in self.humans])
        tweet += f": {self.url}. {self.video_url}"
        return tweet

class Films:
    """Factory to create films more efficiently"""

    def __init__(self) -> None:
        self._raw_films = self._film_list()
        self._position = 0
        self._end = len(self._raw_films) - 1

    def __iter__(self) -> "Films":
        return self

    def __next__(self) -> Film:
        if self._position >= self._end:
            raise StopIteration
        self._position += 1
        return Film(self._raw_films[self._position])

    @staticmethod
    def _film_list() -> list:
        """Data from http://firehousefilmcontest.com/?json=1"""
        with (Path(__file__).parent.parent / "data/films.json").open("r") as film_fd:
            return json.load(film_fd)["posts"]

    def choice(self, max_length: int = 200) -> Film:
        """Get a random film subject to length constraint"""
        film = Film(random.choice(self._raw_films))
        if len(film.text()) <= max_length:
            return film
        return self.choice()

def no_html(text: str) -> str:
    """Helper to strip HTML from strings"""
    return re.sub("<[^<]+?>", "", text)

def main() -> None:
    """Entry point"""
    films = Films()
    for film in films:
        print(film.text(), end="\n\n")

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'SeanRGB'
SITENAME = 'SeanRGB'
SITEURL = 'http://seanrgb.github.io'

PATH = 'content'

TIMEZONE = 'America/Los_Angeles'

DEFAULT_LANG = 'en'

OUTPUT_PATH = '../seanrgb.github.io/'
PATH = 'content'
LOAD_CONTENT_CACHE = False

ARTICLE_URL = '{date:%Y}{date:%m}{date:%d}{slug}.html'
ARTICLE_SAVE_AS = '{date:%Y}{date:%m}{date:%d}{slug}.html'

# THEME STUFF (a lot of this is specifically for the Flex theme)

THEME = './pelican-themes/Flex'
SITETITLE = 'Sean RG Barberie'
SITESUBTITLE = 'Drones | Disasters | Data Science'
SITELOGO = './gravatar.png'
DISQUS_SITENAME = 'SeanRGB'
MAIN_MENU = False
ADD_THIS_ID = 'ra-56a037c909478ef3'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
#LINKS = (('acuasi', 'http://www.acuasi.alaska.edu'),
#	('software carpentry', 'http://software-carpentry.org'),
 #        ('data carpentry', 'http://www.datacarpentry.org'),
  #       ('esip', 'http://www.esipfed.org'),)

# Social widget
SOCIAL = (('twitter', 'https://twitter.com/SeanBarberie'),
          ('github', 'http://github.com/SeanRGB'),
		('linkedin', 'https://br.linkedin.com/in/seanbarberie/en'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True


import math

def unarn(x):
    result = ""
    for i in range(x):
      result += "1"
    return result + "0"

def bin_bites(x, y):
    if x == 0: return ""
    i = 0
    zeros = ""
    while i < (x - len(y)):
      zeros += "0"
      i += 1
    result = zeros + str(y)
    return result

def to_golombo(n,m):
    result = ""
    if math.log2(m) % 1 == 0 and m != 1:
      result = to_rice(n,m)
    else:
      q = math.floor(n / m)
      first = unarn(q)
      r = n % m
      b = math.ceil(math.log2(m))
      if r < (2 ** b) - m:
        bi = bin_bites(b-1, bin(r)[2:])
        result = first + str(bi)
      else:
        bi = bin_bites(b, bin(r + 2 ** b - m)[2:])
        result = first + str(bi)
    return result

def to_rice(n,m):
    bi = ""
    c = math.log(m,2) # логарифм
    r = n % m #остаток от деления числа на m
    q = int(math.floor(n / m)) # частное, округленное вниз, от числа на м
    first = unarn(q)
    bi = bin_bites(math.log2(m), bin(r)[2:])
    final = first + str(bi)
    return final

'''
a = 0
b = 1
while a < 18:
  for b in range(1, 9):
    if b == 7:
      b+=1
    else:
      print ("Код Голомбо для",a,"с параметром m",b,"=",to_golombo(a, b))
      b+=1
  print("=============================================================")
  a+=1
'''

print(to_golombo(19,3))


scores = input()
ascore = 0
bscore = 0
cscore = 0
dscore = 0
escore = 0
for x in scores:
    if x == "a":
        ascore += 1
    if x == "b":
        bscore += 1
    if x == "c":
        cscore += 1
    if x == "d":
        dscore += 1
    if x == "e":
        escore += 1

    if x == "A":
        ascore -= 1
    if x == "B":
        bscore -= 1
    if x == "C":
        cscore -= 1
    if x == "D":
        dscore -= 1
    if x == "E":
        escore -= 1

print("a: %s, b: %s, c: %s, d: %s, e: %s" % (ascore, bscore, cscore, dscore, escore))


"""
:author Madmegsox1
:author FSNCryo
:author Basker 12
:author Jaime
"""

import util
import os
import csv
from util import backgroundGenerator as bg

# This is the base of the project

class run:

    def __init__(self):
        Devs = ("Madmegsox1, FSNCryo, HiddenMask, Basker12, PointlessQuack, DraconicDroid, Squidnugi")
        bg.GenBackground("Pokemon Pictures/abra.png", "formating/pokeBackShadow.png")

    def name_search(self):
        s = open("pokemon data.txt", "r")
        pokename_var = s.readline().replace(',', '')
        s.close()
        pokename_var = util.csv_loader("Pokemon.csv").get_pokemon_by_char(pokename_var)

        searches = open("searches.txt", "w")
        line_count = 0

        csvfile = open("Pokemon.csv", 'r')
        pokemon = csv.reader(csvfile)
        #  Grass, Poison, 1, False, Attack, True,

        searches.write(f"Name Search")
        stats = open("pokemon data.txt", "w")
        r_val = {}
        count = 0
        if pokename_var != 0:
            for i in pokename_var:
                i = list(i.values())
                r_val[i[1]] = i

                count += 1

            stats.write(str(r_val))
            stats.close()
            searches.close()

            return len(r_val)
        else:
            return 0

    def refract_search(self):
        s = open("searches.txt", "r")
        var = s.readline()
        s.close()
        var = (var[:-1]).split(",")

        type1_var = str(var[0])
        type2_var = str(var[1])
        stats_var = var[4]

        gen_var = str(var[2])
        if gen_var != 'All':
            gen_var = int(var[2])

        Legendary_var = str(var[3])
        if Legendary_var == "False":
            Legendary_var = False
        elif Legendary_var == "True":
            Legendary_var = True

        AorD_var = str(var[5])
        if AorD_var == "False":
            AorD_var = True
        elif AorD_var == "True":
            AorD_var = False

        a = util.search()
        a = a.getList(stats_var, AorD_var, [type1_var, type2_var, gen_var, Legendary_var])  # filtered search
        names = open("pokemon data.txt", "w")

        if a != 0:
            for i in a:
                names.write(i)
                names.write(",")
            names.close()
            return len(a)
        else:
            return 0

run()


def tf_baseline_loss(self, states, internals, reward, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the baseline loss of a batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor.
        """
        if self.baseline_mode == 'states':
            loss = self.baseline.loss(
                states=states,
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        elif self.baseline_mode == 'network':
            loss = self.baseline.loss(
                states=self.network.apply(x=states, internals=internals, update=update),
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        regularization_loss = self.baseline.regularization_loss()
        if regularization_loss is not None:
            loss += regularization_loss

        return loss


import json
import logging
import uuid
import sys
import errno
import os
import pandas as pd
import seaborn as sns
import traceback
import shutil
import scipy.cluster.vq as vq
from matplotlib import pyplot as plt
from plotly.offline import plot
import plotly.express as px
import plotly.graph_objs as go
from sklearn.decomposition import PCA

from installed_clients.DataFileUtilClient import DataFileUtil
from installed_clients.KBaseReportClient import KBaseReport

class KmeansClusteringUtil:

    def _mkdir_p(self, path):
        """
        _mkdir_p: make directory for given path
        """
        if not path:
            return
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def _validate_run_kmeans_cluster_params(self, params):
        """
        _validate_run_kmeans_cluster_params:
                validates params passed to run_kmeans_cluster method
        """

        logging.info('start validating run_kmeans_cluster params')

        # check for required parameters
        for p in ['matrix_ref', 'workspace_id', 'cluster_set_name', 'k_num']:
            if p not in params:
                raise ValueError('"{}" parameter is required, but missing'.format(p))

    def _gen_clusters(self, clusters, conditionset_mapping):
        clusters_list = list()

        for cluster in list(clusters.values()):
            labeled_cluster = {}
            labeled_cluster.update({'id_to_data_position': cluster})
            if conditionset_mapping:
                id_to_condition = {k: v for k, v in list(conditionset_mapping.items()) if k in list(cluster.keys())}
                labeled_cluster.update({'id_to_condition': id_to_condition})

            clusters_list.append(labeled_cluster)

        return clusters_list

    def _build_kmeans_cluster_set(self, clusters, cluster_set_name, matrix_ref,
                                  conditionset_mapping, conditionset_ref, workspace_id,
                                  clustering_parameters):
        """
        _build_kmeans_cluster_set: build KBaseExperiments.ClusterSet object
        """

        logging.info('start saving KBaseExperiments.ClusterSet object')

        clusters_list = self._gen_clusters(clusters, conditionset_mapping)

        cluster_set_data = {'clusters': clusters_list,
                            'clustering_parameters': clustering_parameters,
                            'original_data': matrix_ref,
                            'condition_set_ref': conditionset_ref}

        cluster_set_data = {k: v for k, v in list(cluster_set_data.items()) if v}

        object_type = 'KBaseExperiments.ClusterSet'
        save_object_params = {
            'id': workspace_id,
            'objects': [{'type': object_type,
                         'data': cluster_set_data,
                         'name': cluster_set_name}]}

        dfu_oi = self.dfu.save_objects(save_object_params)[0]
        cluster_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(dfu_oi[4])

        return cluster_set_ref

    def _build_kmeans_cluster(self, data_matrix_df, k_num):
        """
        _build_kmeans_cluster: Build Kmeans cluster
        """

        logging.info('start building clusters')

        data_matrix_df.fillna(0, inplace=True)
        values = data_matrix_df.values
        rows = data_matrix_df.index

        # normalize observations
        logging.info('start normalizing raw data')
        whiten_values = vq.whiten(values)

        # run kmeans algorithm
        logging.info('start performing Kmeans algorithm')
        centroid, idx = vq.kmeans2(whiten_values, k_num, minit='points')

        clusters = {}
        for list_index, value in enumerate(idx):
            cluster = clusters.get(value)
            if not cluster:
                clusters.update({value: {rows[list_index]: list_index}})
            else:
                cluster.update({rows[list_index]: list_index})

        return clusters, idx

    def _generate_pairplot(self, data_df, cluster_labels):

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)

        col = data_df.columns

        if len(col) > 20:
            logging.info('start building PCA plot')
            pacplot_path = os.path.join(output_directory, 'pcaplot.html')
            n_components = 2
            s_values = data_df.values
            pca = PCA(n_components=n_components, whiten=True)
            principalComponents = pca.fit_transform(s_values)

            col = list()
            for i in range(n_components):
                col.append('principal_component_{}'.format(i+1))

            rotation_matrix_df = pd.DataFrame(data=principalComponents,
                                              columns=col,
                                              index=data_df.index)
            rotation_matrix_df['cluster'] = [str(cluster_label) for cluster_label in cluster_labels]

            traces = list()
            for name in set(rotation_matrix_df.cluster):
                trace = go.Scatter(
                    x=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)]['principal_component_1']),
                    y=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)]['principal_component_2']),
                    mode='markers',
                    name=name,
                    text=list(rotation_matrix_df.loc[rotation_matrix_df['cluster'].eq(name)].index),
                    textposition='bottom center',
                    marker=go.Marker(size=10, opacity=0.8, line=go.Line(color='rgba(217, 217, 217, 0.14)',
                                                                        width=0.5)))
                traces.append(trace)

            data = go.Data(traces)
            layout = go.Layout(xaxis=go.XAxis(title='Principal Component 1', showline=False),
                               yaxis=go.YAxis(title='Principal Component 2', showline=False))
            fig = go.Figure(data=data, layout=layout)
            fig.update_layout(legend_title=' Cluster ')

            plot(fig, filename=pacplot_path)
            return pacplot_path

        data_df['cluster'] = [str(cluster_label) for cluster_label in cluster_labels]
        try:
            logging.info('start building pairplot')
            pairplot_path = os.path.join(output_directory, 'pairplot.html')
            fig = px.scatter_matrix(data_df, dimensions=list(col), color='cluster',
                                    symbol='cluster')
            plot(fig, filename=pairplot_path)
        except Exception:
            logging.warning('failed to run plotly pairplot')
            logging.warning(traceback.format_exc())
            logging.warning(sys.exc_info()[2])
            pairplot_path = None

        if not pairplot_path:
            try:
                pairplot_path = os.path.join(output_directory, 'pairplot.png')
                sns_plot = sns.pairplot(data_df, hue='cluster', height=2.5, vars=list(col))
                sns_plot.savefig(pairplot_path)
            except Exception:
                logging.warning('failed to run seaborn pairplot')
                logging.warning(traceback.format_exc())
                logging.warning(sys.exc_info()[2])
                pairplot_path = None

        return pairplot_path

    def _generate_pairplot_content(self, pairplot_path, output_directory, dimension):

        pairplot_content = ''''''

        if pairplot_path:
            pairplot_name = dimension + '_' + os.path.basename(pairplot_path)
            shutil.copy2(pairplot_path,
                         os.path.join(output_directory, pairplot_name))

            if pairplot_name.endswith('.html'):
                pairplot_content += '''\n'''.format(
                                                                                pairplot_name)
            elif pairplot_name.endswith('.png'):
                pairplot_content += '''\n\n'''
            else:
                raise ValueError('Unexpected cluster pairplot file format')
        else:
            pairplot_content += '''\n
'''
            pairplot_content += '''Pairplot is too large to be displayed.
\n'''

        return pairplot_content

    def _generate_cluster_info_content(self, cluster_labels):
        cluster_info = ''''''
        cluster_info += '''\n

                                    Cluster Index
                                    Size

\n'''

        unique_labels = list(set(cluster_labels))
        unique_labels.sort(key=float)
        for index in unique_labels:
            cluster_info += '''\n

                                    {}
                                    {}

\n'''.format(index, cluster_labels.tolist().count(index))

        cluster_info += '''\n
\n'''

        return cluster_info

    def _generate_kmeans_html_report(self, data_matrix_df, cluster_labels, dimension):

        logging.info('start generating html report')
        html_report = list()

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)
        result_file_path = os.path.join(output_directory, 'hier_report.html')

        pairplot = self._generate_pairplot(data_matrix_df, cluster_labels)

        cluster_info = self._generate_cluster_info_content(cluster_labels)
        pairplot_content = self._generate_pairplot_content(pairplot, output_directory, dimension)

        with open(result_file_path, 'w') as result_file:
            with open(os.path.join(os.path.dirname(__file__), 'kmeans_report_template.html'),
                      'r') as report_template_file:
                report_template = report_template_file.read()
                report_template = report_template.replace('
Cluster_Info
',
                                                          cluster_info)
                report_template = report_template.replace('
Pairplot
',
                                                          pairplot_content)
                result_file.write(report_template)

        report_shock_id = self.dfu.file_to_shock({'file_path': output_directory,
                                                  'pack': 'zip'})['shock_id']

        html_report.append({'shock_id': report_shock_id,
                            'name': os.path.basename(result_file_path),
                            'label': os.path.basename(result_file_path),
                            'description': 'HTML summary report for Cluster App'
                            })
        return html_report

    def _generate_kmeans_cluster_report(self, cluster_set_ref, workspace_id,
                                        data_matrix_df, cluster_labels, dimension):
        """
        _generate_kmeans_cluster_report: generate summary report
        """

        objects_created = []
        objects_created.append({'ref': cluster_set_ref,
                                'description': 'Kmeans ClusterSet'})

        output_html_files = self._generate_kmeans_html_report(data_matrix_df,
                                                              cluster_labels,
                                                              dimension)

        report_params = {'message': '',
                         'workspace_id': workspace_id,
                         'objects_created': objects_created,
                         'html_links': output_html_files,
                         'direct_html_link_index': 0,
                         'html_window_height': 333,
                         'report_object_name': 'run_kmeans_cluster_' + str(uuid.uuid4())}

        kbase_report_client = KBaseReport(self.callback_url, token=self.token)
        output = kbase_report_client.create_extended_report(report_params)

        report_output = {'report_name': output['name'], 'report_ref': output['ref']}

        return report_output

    def __init__(self, config):
        self.callback_url = config['SDK_CALLBACK_URL']
        self.token = config['KB_AUTH_TOKEN']
        self.scratch = config['scratch']

        # helper kbase module
        self.dfu = DataFileUtil(self.callback_url)

        logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
                            level=logging.INFO)

        plt.switch_backend('agg')
        sys.setrecursionlimit(150000)

    def run_kmeans_cluster(self, params):
        """
        run_kmeans_cluster: generates Kmeans clusters for Matrix data object

        matrix_ref: Matrix object reference
        workspace_id: the id of the workspace
        cluster_set_name: KBaseExperiments.ClusterSet object name
        dimension: run cluster algorithm on dimension, col or row
        k_num: number of clusters to form

        Optional arguments:
        dist_metric: The distance metric to use. Default set to 'euclidean'.
                   The distance function can be
                   ["braycurtis", "canberra", "chebyshev", "cityblock", "correlation", "cosine",
                    "dice", "euclidean", "hamming", "jaccard", "kulsinski", "matching",
                    "rogerstanimoto", "russellrao", "sokalmichener", "sokalsneath", "sqeuclidean",
                    "yule"]
                   Details refer to:
                   https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.pdist.html

        return:
        cluster_set_ref: KBaseExperiments.ClusterSet object reference
        report_name: report name generated by KBaseReport
        report_ref: report reference generated by KBaseReport
        """

        logging.info('--->\nrunning run_kmeans_cluster\n' +
                     'params:\n{}'.format(json.dumps(params, indent=1)))

        self._validate_run_kmeans_cluster_params(params)

        matrix_ref = params.get('matrix_ref')
        workspace_id = params.get('workspace_id')
        cluster_set_name = params.get('cluster_set_name')
        dimension = params.get('dimension', 'col')
        k_num = params.get('k_num')

        if dimension not in ['col', 'row']:
            raise ValueError('Please use "col" or "row" for input dimension')

        matrix_data = self.dfu.get_objects({'object_refs': [matrix_ref]})['data'][0]['data']

        matrix_data_values = matrix_data['data']
        data_matrix_df = pd.DataFrame(matrix_data_values['values'],
                                      index=matrix_data_values['row_ids'],
                                      columns=matrix_data_values['col_ids'])
        if dimension == 'col':
            data_matrix_df = data_matrix_df.T

        (kmeans_clusters,
         cluster_labels) = self._build_kmeans_cluster(data_matrix_df, k_num)

        clustering_parameters = {'dimension': dimension,
                                 'k_num': str(k_num)}

        cluster_set_ref = self._build_kmeans_cluster_set(
                                        kmeans_clusters,
                                        cluster_set_name,
                                        matrix_ref,
                                        matrix_data.get('{}_mapping'.format(dimension)),
                                        matrix_data.get('{}_conditionset_ref'.format(dimension)),
                                        workspace_id,
                                        clustering_parameters)

        returnVal = {'cluster_set_ref': cluster_set_ref}

        report_output = self._generate_kmeans_cluster_report(cluster_set_ref, workspace_id,
                                                             data_matrix_df, cluster_labels,
                                                             dimension)

        returnVal.update(report_output)

        return returnVal


from collections import UserList

from metadata.models import DataSource
from metadata.models.constants import IGNORED_CONSUL_SYNC_DATA_IDS
from metadata.models.influxdb_cluster import InfluxDBClusterInfo

class MockHashConsul(object):
    result_list = []

    def put(self, key, value):
        self.result_list.append({"key": key, "value": value})

class MockClusterList(UserList):
    def count(self):
        return len(self.data)

class TestConsulCluster(object):
    def test_refresh_consul_cluster_config(self, mocker, patch_redis_tools):
        cluster_host_list = MockClusterList()
        cluster_host_list.append(InfluxDBClusterInfo(host_name="host1", cluster_name="cluster1", host_readable=True))
        cluster_host_list.append(InfluxDBClusterInfo(host_name="host2", cluster_name="cluster1", host_readable=False))
        mocker.patch("metadata.models.influxdb_cluster.InfluxDBClusterInfo.objects.all", return_value=cluster_host_list)
        mock_hash_consul = MockHashConsul()
        mocker.patch("metadata.utils.consul_tools.HashConsul", return_value=mock_hash_consul)
        InfluxDBClusterInfo.refresh_consul_cluster_config()
        assert mock_hash_consul.result_list[0]["key"] == "/".join([InfluxDBClusterInfo.CONSUL_PREFIX_PATH, "cluster1"])
        assert mock_hash_consul.result_list[0]["value"] == {
            "host_list": ["host1", "host2"],
            "unreadable_host_list": ["host2"],
        }

    def test_refresh_consul_data_source_config(self, mocker, patch_redis_tools):

        data_source_list = [DataSource(bk_data_id=data_id) for data_id in IGNORED_CONSUL_SYNC_DATA_IDS]
        data_source_list.append(DataSource(bk_data_id=12345))
        mocker.patch("metadata.models.DataSource.objects.all", return_value=data_source_list)
        mocker.patch("metadata.models.DataSource.to_json", return_value={})
        mock_hash_consul = MockHashConsul()
        mocker.patch("metadata.utils.consul_tools.HashConsul", return_value=mock_hash_consul)
        for ds in data_source_list:
            ds.refresh_consul_config()

        assert len(mock_hash_consul.result_list) == 1
        assert mock_hash_consul.result_list[0]["key"] == DataSource(bk_data_id=12345).consul_config_path


#!/usr/bin/env python

# Puzzle
# https://adventofcode.com/2021/day/4

class BingoBoard:

    def __init__(self, grid):
        self.grid = grid
        self.has_won = False

    def __repr__(self):
        return str(self.grid)

    def mark(self, num):
        for r_i, row in enumerate(self.grid):
            for c_i, col in enumerate(row):
                if col == num:
                    self.grid[r_i][c_i] = 'x'

    def check_row_bingo(self):
        for row in self.grid:
            if all(num == row[0] for num in row):
                return True

    def check_col_bingo(self):
        for i in range(5):
            col_list = [row[i] for row in self.grid]
            if all(num == col_list[0] for num in col_list):
                return True

    def check_bingo(self):
        return self.check_row_bingo() or self.check_col_bingo()

    def get_score(self, last_num):
        num_list = [col for row in self.grid for col in row if type(col) == int]
        return sum(num_list) * last_num

def ingest_bingo(filename):
    input_list = []
    with open(filename) as f:
        num_list = [int(n) for n in f.readline().split(",")]
        print(num_list)
        all_boards = []
        for line in f:
            grid = []
            for row in range(5):
                bingo_row = [int(n) for n in f.readline().split()]
                grid.append(bingo_row)
            all_boards.append(grid)
    return num_list, all_boards

if __name__ == "__main__":
    num_list, all_boards = ingest_bingo('input.txt')
    print(num_list)
    boards = [BingoBoard(grid) for grid in all_boards]
    print(boards)
    num_list_clone = num_list
    found_bingo = False
    while True not in [board.check_bingo() for board in boards]:
        print(f"num_list: {num_list}")
        num = num_list_clone.pop(0)
        for board in boards:
            board.mark(num)
            print(board)
            if board.check_bingo():
                print("found bingo")
                found_bingo = True
                print(board.get_score(num))
                break

    for num in num_list:
        for board in boards:
            if not board.has_won:
                board.mark(num)
                print(board)
                if board.check_bingo():
                    print("found bingo")
                    found_bingo = True
                    print(board.get_score(num))
                    board.has_won = True

    #print(f"Part 1: {product_p1}")
    #product_p2 = get_life_support_rating(input_list)
    #print(f"Part 2: {product_p2}")


import csv
import os

# Set the output directory name
output_dir = 'output'

# Create the output directory if it doesn't exist
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

# Get the path to the content.csv file in the same directory as the script
csv_file_path = os.path.join(os.path.dirname(__file__), 'content.csv')

# Open the CSV file
with open(csv_file_path, newline='') as csvfile:
    # Create a CSV reader object
    reader = csv.reader(csvfile)
    # Skip the first row (headings)
    next(reader, None)
    # Iterate over each row in the CSV file
    for row in reader:
        # Get the filename and contents from the row
        filename = row[0]
        contents = row[1]
        # Create the full path to the output file
        output_path = os.path.join(output_dir, filename)
        # Write the contents to the output file
        with open(output_path, 'w') as outfile:
            outfile.write(contents)


import os
import datetime
from tradetools.FoundationHelper import get_jingzhi

from flask_sqlalchemy import SQLAlchemy

db = SQLAlchemy()

WORKER_STOP = 0
WORKER_RUN = 1

CRASH_UNCHECKED = 0
CRASH_NEGATIVE = 1
CRASH_POSITIVE = 2

class Worker(db.Model):
    id = db.Column(db.String(36), primary_key=True)
    ip = db.Column(db.String(15))
    bind_port = db.Column(db.Integer)
    proc_count = db.Column(db.Integer)
    case_count = db.Column(db.Integer, default=0)
    log_path = db.Column(db.String(4096))
    status = db.Column(db.Integer)
    started_at = db.Column(db.DateTime)
    resource = db.relationship('Resource', backref='host', uselist=False)
    crashes = db.relationship('Crash', backref='host', lazy='dynamic')

    def __init__(self, id=None, ip=None, bind_port=None, proc_count=None, log_path=None, status=WORKER_RUN):
        self.id = id
        self.ip = ip
        self.bind_port = bind_port
        self.proc_count = proc_count
        self.log_path = log_path
        self.status = status
        self.started_at = datetime.datetime.now()

    def __repr__(self):
        return '<Worker %r>' % self.ip

class Resource(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    host_id = db.Column(db.String(36), db.ForeignKey('worker.id'))
    cpu_count = db.Column(db.Integer)
    cpu_usage = db.Column(db.Float)
    mem_usage = db.Column(db.Float)
    disk_usage = db.Column(db.Float)
    updated_at = db.Column(db.DateTime, onupdate=datetime.datetime.now)

    def __init__(self, worker=None, cpu_count=None, cpu_usage=None, mem_usage=None, disk_usage=None):
        self.host = worker
        self.cpu_count = cpu_count
        self.cpu_usage = cpu_usage
        self.mem_usage = mem_usage
        self.disk_usage = disk_usage
        self.updated_at = datetime.datetime.now()

    def __repr__(self):
        return '<Resource %r>' % self.ip

class Crash(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    host_id = db.Column(db.String(36), db.ForeignKey('worker.id'))
    crash_path = db.Column(db.String(4096))
    crash_name = db.Column(db.String(255))
    case_path = db.Column(db.String(4096))
    case_name = db.Column(db.String(255))
    status = db.Column(db.Integer)
    created_at = db.Column(db.DateTime)

    def __init__(self, worker=None, crash_path=None, case_path=None, status=CRASH_UNCHECKED):
        self.host = worker
        self.crash_path = crash_path
        self.crash_name = os.path.basename(self.crash_path)
        self.case_path = case_path
        self.case_name = os.path.basename(self.case_path)
        self.status = status
        self.created_at = datetime.datetime.now()

    def __repr__(self):
        return '<Crash %r>' % self.ip

class IokitOpen(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    service_name = db.Column(db.String(255))
    type = db.Column(db.Integer)
    class_name = db.Column(db.String(255))
    xnu_version_start = db.Column(db.Integer)
    submit_time = db.Column(db.DateTime)
    verify_open = db.Column(db.String(255))

    def __init__(self, service_name=None, type=None, class_name=None, xnu_version_start=None):
        self.service_name = service_name
        self.type = type
        self.class_name = class_name
        self.xnu_version_start = xnu_version_start
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<IokitOpen %r>' % self.id

class IokitConnect(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    class_name = db.Column(db.String(255))
    selector = db.Column(db.Integer)

    scalar_input_count = db.Column(db.Integer)
    scalar_output_count = db.Column(db.Integer)
    struct_input_size = db.Column(db.Integer)
    struct_output_size = db.Column(db.Integer)

    ool_input_size = db.Column(db.Integer)
    ool_output_size = db.Column(db.Integer)
    md5_no_content = db.Column(db.String(255))
    xnu_version_start = db.Column(db.Integer)
    xnu_version_end = db.Column(db.Integer)

    submit_time = db.Column(db.DateTime)
    most_success = db.Column(db.String(255))
    fuzzing_round = db.Column(db.Integer)
    fuzzing_success_rate = db.Column(db.Float)

    def __init__(self, class_name=None, selector=None, scalar_input_count=None, scalar_output_count=None,
                 struct_input_size=None, struct_output_size=None,
                 ool_input_size=None, ool_output_size=None, md5_no_content=None, xnu_version_start=None,
                 xun_version_end=None, most_success=None, fuzzing_round=None, fuzzing_success_rate=None):
        self.class_name = class_name
        self.selector = selector
        self.scalar_input_count = scalar_input_count
        self.scalar_output_count = scalar_output_count
        self.struct_input_size = struct_input_size
        self.struct_output_size = struct_output_size
        self.ool_input_size = ool_input_size
        self.ool_output_size = ool_output_size
        self.md5_no_content = md5_no_content
        self.xnu_version_start = xnu_version_start
        self.xun_version_end = xun_version_end
        self.most_success = most_success
        self.fuzzing_round = fuzzing_round
        self.fuzzing_success_rate = fuzzing_success_rate
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<IokitConnect %r>' % self.id

######################
class Record(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    base = db.Column(db.Float)
    beishu = db.Column(db.Float)
    code = db.Column(db.String(255))
    buy_date = db.Column(db.DateTime)
    follow = db.Column(db.Float)
    jingzhi = db.Column(db.Float)
    gushu = db.Column(db.Float)

    submit_time = db.Column(db.DateTime)

    def __init__(self, id, date, base, beishu, follow, code):
        self.id = id
        self.base = float(base)
        self.beishu = float(beishu)
        self.code = code
        self.buy_date = date if date else (datetime.date.today()).strftime('%Y-%m-%d')
        self.follow = float(follow) if follow else self.base * float(beishu)
        self.jingzhi = float(get_jingzhi(code, self.buy_date))
        self.gushu = round(float(self.follow) / float(self.jingzhi) if self.jingzhi != -1 else -1, 4)
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<record %r>' % self.id