import xlwt
import numpy as np
import os
import os.path

import colour

from ..configuration.base_configuration import Filter
from ..configuration.base_configuration import TimeOfDayFilter

from ..core.status import Status

from plots import MatplotlibPlotter
from power_deviation_matrix import PowerDeviationMatrixSheet

import version as ver

def get_valid_excel_sheet_name(sheet_name, if_too_long_replace={}):
    max_chars = 31
    invalid_chars = ':|\\/*?[]'
    for c in invalid_chars:
        sheet_name = sheet_name.replace(c, '')
    if len(sheet_name) > max_chars:
        for sub_str in if_too_long_replace:
            sheet_name = sheet_name.replace(sub_str, if_too_long_replace[sub_str])
    return sheet_name[:max_chars]

def chckMake(path):
    """Make a folder if it doesn't exist"""
    if not os.path.exists(path):
        os.makedirs(path)

class PNGPlotter:

    def plot(self, analysis, path):

        chckMake(path)

        plotter = MatplotlibPlotter(path, analysis)

        if analysis.hasActualPower:

            plotter.plotPowerCurve(analysis.baseline.wind_speed_column, analysis.actualPower, analysis.allMeasuredPowerCurve, specified_title = 'Warranted', mean_title = 'Measured Mean', gridLines = True)
            plotter.plotPowerCurve(analysis.baseline.wind_speed_column, analysis.actualPower, analysis.allMeasuredPowerCurve, show_scatter = False, fname = "PowerCurve - Warranted vs Measured Mean", specified_title = 'Warranted', mean_title = 'Measured Mean', mean_pc_color = 'blue', gridLines = True)

            if analysis.turbRenormActive:
                plotter.plotTurbCorrectedPowerCurve(analysis.baseline.wind_speed_column, analysis.measuredTurbulencePower, analysis.allMeasuredTurbCorrectedPowerCurve)

            if analysis.hasAllPowers:
                plotter.plotPowerLimits(specified_title = 'Warranted', gridLines = True)

        plotter.plotBy(analysis.windDirection, analysis.hubWindSpeed, analysis.dataFrame, gridLines = True)
        plotter.plotBy(analysis.windDirection, analysis.shearExponent, analysis.dataFrame, gridLines = True)
        plotter.plotBy(analysis.windDirection, analysis.hubTurbulence, analysis.dataFrame, gridLines = True)
        plotter.plotBy(analysis.hubWindSpeed, analysis.hubTurbulence, analysis.dataFrame, gridLines = True)

        if analysis.hasActualPower:
            plotter.plotBy(analysis.hubWindSpeed, analysis.powerCoeff, analysis.dataFrame, gridLines = True)
            plotter.plotBy('Input Hub Wind Speed', analysis.powerCoeff, analysis.allMeasuredPowerCurve, gridLines = True)

        if analysis.inflowAngle in analysis.dataFrame.columns:
            analysis.dataFrame.loc[analysis.dataFrame[analysis.inflowAngle]>180,analysis.inflowAngle] -= 360
            plotter.plotBy(analysis.windDirection,analysis.inflowAngle,analysis.dataFrame, gridLines = True)

        plotter.plotCalibrationSectors()

        if analysis.hasActualPower:

            if analysis.multiple_datasets:
                plotter.plot_multiple(analysis.baseline.wind_speed_column, analysis.actualPower, analysis.allMeasuredPowerCurve)

class TimeSeriesExporter:

    def export(self, analysis, time_series_path, clean=True,  full=True, calibration=True,
               full_df_output_dir="TimeSeriesData"):

        data_frame = analysis.dataFrame
        dataset_configs = analysis.datasetConfigs

        if clean:
            data_frame.to_csv(time_series_path, sep='\t')

        if full:

            root_path = os.path.join(os.path.dirname(time_series_path), full_df_output_dir)
            chckMake(root_path)

            for ds in dataset_configs:

                ds.data.fullDataFrame.to_csv(root_path + os.sep + "FilteredDataSet_AllColumns_{0}.dat".format(ds.name),
                                             sep='\t')

                if calibration and hasattr(ds.data,"filteredCalibrationDataframe"):
                    ds.data.filteredCalibrationDataframe.to_csv(
                        root_path + os.sep + "CalibrationDataSet_{0}.dat".format(ds.name), sep=',')

class Report:

    bold_style = xlwt.easyxf('font: bold 1')
    no_dp_style = xlwt.easyxf(num_format_str='0')
    one_dp_style = xlwt.easyxf(num_format_str='0.0')
    two_dp_style = xlwt.easyxf(num_format_str='0.00')
    three_dp_style = xlwt.easyxf(num_format_str='0.000')
    four_dp_style = xlwt.easyxf(num_format_str='0.0000')
    percent_style = xlwt.easyxf(num_format_str='0.00%')
    percent_no_dp_style = xlwt.easyxf(num_format_str='0%')

    def __init__(self, windSpeedBins, calculated_power_deviation_matrix_dimensions):

        self.version = ver.version

        self.windSpeedBins = windSpeedBins
        self.calculated_power_deviation_matrix_dimensions = calculated_power_deviation_matrix_dimensions

    def report(self, path, analysis):

        report_power_curve = analysis.hasActualPower

        book = xlwt.Workbook()

        plotsDir = os.path.dirname(path)

        plotter = PNGPlotter()
        plotter.plot(analysis, plotsDir)

        gradient = colour.ColourGradient(-0.1, 0.1, 0.01, book)

        if report_power_curve:
            sh = book.add_sheet("PowerCurves", cell_overwrite_ok=True)

        settingsSheet = book.add_sheet("Settings", cell_overwrite_ok=True)

        self.reportSettings(settingsSheet, analysis)

        if report_power_curve:

            rowsAfterCurves = []

            #rowsAfterCurves.append(self.reportPowerCurve(sh, 0, 0, 'uniqueAnalysisId', analysis.specified_power_curve, analysis)) #needs fixing + move to settings sheet

            if analysis.specified_power_curve is not None:
                if len(analysis.specified_power_curve.data_frame) != 0:
                    rowsAfterCurves.append(  self.reportPowerCurve(sh, 1, 0, 'Specified', analysis.specified_power_curve, analysis))

            if analysis.hasActualPower:

                #for name in analysis.residualWindSpeedMatrices:
                #    residualMatrix = analysis.residualWindSpeedMatrices[name]
                #
                #    if residualMatrix != None:
                #        self.reportPowerDeviations(book, "ResidualWindSpeed-%s" % name, residualMatrix, gradient)

                if analysis.hasShear and analysis.innerMeasuredPowerCurve != None:
                    rowsAfterCurves.append(self.reportPowerCurve(sh, 1, 5, 'Inner', analysis.innerMeasuredPowerCurve, analysis) )

                if analysis.hasShear and analysis.outerMeasuredPowerCurve != None:
                    rowsAfterCurves.append(self.reportPowerCurve(sh, 1, 10, 'Outer', analysis.outerMeasuredPowerCurve, analysis) )

                rowsAfterCurves.append( self.reportPowerCurve(sh, 1, 15, 'All', analysis.allMeasuredPowerCurve, analysis) )

                if analysis.turbRenormActive:
                    rowsAfterCurves.append(self.reportPowerCurve(sh, 1, 20, 'TurbulenceRenormalisedPower', analysis.allMeasuredTurbCorrectedPowerCurve, analysis) )

                if analysis.specified_power_curve is not None:

                    rowAfterCurves = max(rowsAfterCurves) + 5
                    sh.write(rowAfterCurves-2, 0, "Power Curves Interpolated to Specified Bins:", self.bold_style)
                    specifiedLevels = analysis.specified_power_curve.data_frame.index

                    if analysis.hasShear and analysis.innerMeasuredPowerCurve != None:
                        self.reportInterpolatedPowerCurve(sh, rowAfterCurves, 5, 'Inner', analysis.innerMeasuredPowerCurve, specifiedLevels)

                    if analysis.hasShear and analysis.outerMeasuredPowerCurve != None:
                        self.reportInterpolatedPowerCurve(sh, rowAfterCurves, 10, 'Outer', analysis.outerMeasuredPowerCurve, specifiedLevels)

                    self.reportInterpolatedPowerCurve(sh, rowAfterCurves, 15, 'All', analysis.allMeasuredPowerCurve, specifiedLevels)

                    if analysis.turbRenormActive:
                        self.reportInterpolatedPowerCurve(sh, rowAfterCurves, 20, 'TurbulenceRenormalisedPower', analysis.allMeasuredTurbCorrectedPowerCurve, specifiedLevels)

                    self.reportInterpolatedPowerCurve(sh, rowAfterCurves, (25 if analysis.turbRenormActive else 20), 'DayTime', analysis.dayTimePowerCurve, specifiedLevels)
                    self.reportInterpolatedPowerCurve(sh, rowAfterCurves, (30 if analysis.turbRenormActive else 25), 'NightTime', analysis.nightTimePowerCurve, specifiedLevels)

                self.reportPowerDeviations(book, "Baseline Power Deviations", analysis.baseline_power_deviations, gradient)

                #if analysis.rewsActive:
                #    self.reportPowerDeviations(book, "REWS Deviation", analysis.rewsMatrix, gradient)

                for correction_name in analysis.corrections:
                    correction = analysis.corrections[correction_name]
                    deviations = analysis.corrected_deviations[correction.correction_name]
                    sheet_name = get_valid_excel_sheet_name("{0} Power Deviations".format(
                        correction.short_correction_name), if_too_long_replace={'Power Deviations': 'PowDevs'})
                    self.reportPowerDeviations(book, sheet_name, deviations, gradient)

                if analysis.nominal_wind_speed_distribution.absolute_path is not None:
                    sh = book.add_sheet("EnergyAnalysis", cell_overwrite_ok=True)
                    self.report_aep(sh, analysis)

        if len(analysis.calibrations) == 1:
            calSheet = book.add_sheet("Calibration", cell_overwrite_ok=True)
            self.reportCalibration(calSheet,analysis.calibrations[0],timeStepInSeconds = analysis.timeStepInSeconds)
        elif len(analysis.calibrations) > 1:
            i = 0
            for cal in analysis.calibrations:
                i += 1
                calSheet = book.add_sheet("Calibration_%03d" % i, cell_overwrite_ok=True)
                self.reportCalibration(calSheet,cal,timeStepInSeconds = analysis.timeStepInSeconds)

        book.save(path)

    def reportCalibration(self,sh,calibration,timeStepInSeconds = 600.):
        conf, calib = calibration
        sh.write(0, 0, "Dataset Name", self.bold_style)
        sh.write(1, 0, conf.name)
        startRow = 3
        col = -14

        if 'belowAbove' in calib.calibrationSectorDataframe.columns :
            belowAbove = True
        else:
            belowAbove = False
        col+=16
        row=startRow
        sh.write(row,col,conf.name, self.bold_style)
        sh.write(row,col+1,"Method:"+conf.calibrationMethod, self.bold_style)
        row += 1
        sh.write(row,col,"Bin", self.bold_style)
        sh.write(row,col+1,"Slope", self.bold_style)
        sh.write(row,col+2,"Offset", self.bold_style)
        if conf.calibrationMethod != 'Specified':
            sh.write(row,col+3,"Count", self.bold_style)
            sh.write(row,col+4,"Hours", self.bold_style)
            if belowAbove:
                sh.write(row,col+5,"Count <= 8m/s", self.bold_style)
                sh.write(row,col+6,"Hours <= 8m/s", self.bold_style)
                sh.write(row,col+7,"Count >  8m/s", self.bold_style)
                sh.write(row,col+8,"Hours >  8m/s", self.bold_style)
            sh.write(row,col+9,"Speedup at 10m/s", self.bold_style)
            sh.write(row,col+10,"% Speedup at 10m/s", self.bold_style)
            sh.write(row,col+11,"Filter (Total Hours > 24)", self.bold_style)
            sh.write(row,col+12,"Filter (Hours Below/Above 8m/s > 6)", self.bold_style)
            sh.write(row,col+13,"Filter (Speedup Change < 2%)", self.bold_style)
            sh.write(row,col+14,"Valid Sector", self.bold_style)

        row+=1
        for key in sorted(calib.calibrationSectorDataframe.index):
            sh.write(row,col,float(key), self.bold_style)
            sh.write(row,col+1,calib.calibrationSectorDataframe['Slope'][key], self.four_dp_style)
            sh.write(row,col+2,calib.calibrationSectorDataframe['Offset'][key], self.four_dp_style)
            if conf.calibrationMethod != 'Specified':
                if 'Count' in calib.calibrationSectorDataframe.columns:
                    sh.write(row,col+3,calib.calibrationSectorDataframe['Count'][key], self.no_dp_style)
                    sh.write(row,col+4,calib.calibrationSectorDataframe['Count'][key]*(timeStepInSeconds/3600.0), self.one_dp_style)
                if belowAbove:
                    ba = calib.calibrationSectorDataframe.loc[key,'belowAbove']
                    sh.write(row,col+5,ba[0], self.no_dp_style)
                    sh.write(row,col+6,ba[0]*(timeStepInSeconds/3600.0), self.one_dp_style)
                    sh.write(row,col+7,ba[1], self.no_dp_style)
                    sh.write(row,col+8,ba[1]*(timeStepInSeconds/3600.0), self.one_dp_style)
                sh.write(row,col+9,calib.calibrationSectorDataframe['SpeedUpAt10'][key], self.four_dp_style)
                sh.write(row,col+10,(calib.calibrationSectorDataframe['SpeedUpAt10'][key]-1.0), self.percent_style)

                totalHoursValid = calib.getTotalHoursValidity(key, timeStepInSeconds)
                sh.write(row,col+11, "TRUE" if totalHoursValid else "FALSE")
                if belowAbove:
                    belowAboveValid = calib.getBelowAboveValidity(key, timeStepInSeconds)
                    sh.write(row,col+12, "TRUE" if belowAboveValid else "FALSE")
                speedUpChangeValid = calib.getSpeedUpChangeValidity(key)
                sh.write(row,col+13, "TRUE" if speedUpChangeValid else "FALSE")
                sectorValid = calib.getSectorValidity(key, timeStepInSeconds)
                sh.write(row,col+14, "TRUE" if sectorValid else "FALSE", self.bold_style)
            row += 1

        if len(conf.calibrationFilters) > 0:
            row += 2
            sh.write(row, col, "Calibration Filters", self.bold_style)
            row += 1
            sh.write(row, col, "Data Column", self.bold_style)
            sh.write(row, col+1, "Filter Type", self.bold_style)
            sh.write(row, col+2, "Inclusive", self.bold_style)
            sh.write(row, col+3, "Filter Value", self.bold_style)
            sh.write(row, col+4, "Active", self.bold_style)
            row += 1

            for filt in conf.calibrationFilters:
                if isinstance(Filter,TimeOfDayFilter):
                    sh.write(row, col, "Time Of Day Filter")
                    sh.write(row, col + 1, str(filt.startTime))
                    sh.write(row, col + 2, str(filt.endTime))
                    sh.write(row, col + 3, str(filt.daysOfTheWeek))
                    sh.write(row, col + 4, str(filt.months))
                else:
                    sh.write(row, col, filt.column)
                    sh.write(row, col+1, filt.filterType)
                    sh.write(row, col+2, filt.inclusive)
                    sh.write(row, col+3, str(filt))
                    sh.write(row, col+4, filt.active) # always true if in list...
                row += 1

    def reportSettings(self, sh, analysis):

        config = analysis
        sh.write(0, 1, "PCWG Tool Version Number:")
        sh.write(0, 2, self.version)
        sh.write(0, 3, xlwt.Formula('HYPERLINK("http://www.pcwg.org";"PCWG Website")'))

        row = 3

        labelColumn = 1
        dataColumn = 2

        sh.col(labelColumn).width = 256 * 30
        sh.col(dataColumn).width = 256 * 50
        sh.col(dataColumn+1).width = 256 * 50

        #Corretions
        sh.write(row, labelColumn, "Density Correction Active", self.bold_style)
        sh.write(row, dataColumn, config.densityCorrectionActive)
        row += 1

        sh.write(row, labelColumn, "REWS Correction Active", self.bold_style)
        sh.write(row, dataColumn, config.rewsActive)
        row += 1

        sh.write(row, labelColumn, "Turbulence Correction Active", self.bold_style)
        sh.write(row, dataColumn, config.turbRenormActive)
        row += 1

        #General Settings
        row += 1

        sh.write(row, labelColumn, "Time Step In Seconds", self.bold_style)
        sh.write(row, dataColumn, analysis.timeStepInSeconds)
        row += 1

        sh.write(row, labelColumn, "Power Curve Minimum Count", self.bold_style)
        sh.write(row, dataColumn, config.powerCurveMinimumCount)
        row += 1

        sh.write(row, labelColumn, "Power Curve Mode", self.bold_style)
        sh.write(row, dataColumn, config.powerCurveMode)
        row += 1

        #Inner Range
        row += 1
        sh.write(row, labelColumn, "Inner Range", self.bold_style)
        row += 1

        for dimension in config.inner_range_dimensions:

            sh.write(row, labelColumn, "Lower {0}".format(dimension.parameter), self.bold_style)
            sh.write(row, dataColumn, dimension.lower_limit)
            row += 1

            sh.write(row, labelColumn, "Upper {0}".format(dimension.parameter), self.bold_style)
            sh.write(row, dataColumn, dimension.upper_limit)
            row += 1

        #Turbine
        #row += 1
        #sh.write(row, labelColumn, "Turbine", self.bold_style)
        #row += 1

        #sh.write(row, labelColumn, "Specified Power Curve", self.bold_style)
        #sh.write(row, dataColumn, config.specified_power_curve.absolute_path)
        #row += 1

        #datasets
        row += 1
        sh.write(row, labelColumn, "Datasets", self.bold_style)
        row += 2

        for datasetConfig in analysis.datasetConfigs:

            sh.write(row, labelColumn, "Name", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.name)
            row += 1

            sh.write(row, labelColumn, "Path", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.path)
            row += 1

            sh.write(row, labelColumn, "Rated Power", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.ratedPower)
            row += 1

            sh.write(row, labelColumn, "HubHeight", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.hubHeight)
            row += 1

            sh.write(row, labelColumn, "Diameter", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.diameter)
            row += 1

            sh.write(row, labelColumn, "Cut In Wind Speed", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.cutInWindSpeed)
            row += 1

            sh.write(row, labelColumn, "Cut Out Wind Speed", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.cutOutWindSpeed)
            row += 1

            sh.write(row, labelColumn, "Start Date", self.bold_style)
            sh.write(row, dataColumn, str(datasetConfig.startDate))
            row += 1

            sh.write(row, labelColumn, "End Date", self.bold_style)
            sh.write(row, dataColumn, str(datasetConfig.endDate))
            row += 1

            sh.write(row, labelColumn, "Hub Wind Speed Mode", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.hubWindSpeedMode)
            row += 1

            sh.write(row, labelColumn, "Density Mode", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.densityMode)
            row += 2

            sh.write(row, labelColumn, "REWS Defined", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.rewsDefined)
            row += 1

            sh.write(row, labelColumn, "Rotor Mode", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.rotorMode)
            row += 1

            sh.write(row, labelColumn, "Hub Mode", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.hubMode)
            row += 1

            sh.write(row, labelColumn, "Number of Rotor Levels", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.numberOfRotorLevels)
            row += 2

            sh.write(row, labelColumn, "Measurements", self.bold_style)
            row += 1

            sh.write(row, labelColumn, "Input Time Series Path", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.input_time_series.absolute_path)
            row += 1

            sh.write(row, labelColumn, "Date Format", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.dateFormat)
            row += 1

            sh.write(row, labelColumn, "Time Step In Seconds", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.timeStepInSeconds)
            row += 1

            sh.write(row, labelColumn, "Time Stamp", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.timeStamp)
            row += 1

            sh.write(row, labelColumn, "Bad Data Value", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.badData)
            row += 1

            sh.write(row, labelColumn, "Header Rows", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.headerRows)
            row += 1

            sh.write(row, labelColumn, "Turbine Location Wind Speed", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.turbineLocationWindSpeed)
            row += 1

            sh.write(row, labelColumn, "Hub Wind Speed", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.hubWindSpeed)
            row += 1

            sh.write(row, labelColumn, "Hub Turbulence", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.hubTurbulence)
            row += 1

            sh.write(row, labelColumn, "Reference Wind Speed", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.referenceWindSpeed)
            row += 1

            sh.write(row, labelColumn, "Reference Wind Speed Std Dev", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.referenceWindSpeedStdDev)
            row += 1

            sh.write(row, labelColumn, "Reference Wind Direction", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.referenceWindDirection)
            row += 1

            sh.write(row, labelColumn, "Reference Wind Direction Offset", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.referenceWindDirectionOffset)
            row += 1

            sh.write(row, labelColumn, "Density", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.density)
            row += 1

            sh.write(row, labelColumn, "Temperature", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.temperature)
            row += 1

            sh.write(row, labelColumn, "Pressure", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.pressure)
            row += 1

            if len(datasetConfig.turbineShearMeasurements) > 0:
                row = self.writeShear(sh,labelColumn,dataColumn,row,datasetConfig.referenceShearMeasurements,'Reference Location ')
                row = self.writeShear(sh,labelColumn,dataColumn,row,datasetConfig.turbineShearMeasurements,'Turbine Location ')
            else:
                row = self.writeShear(sh,labelColumn,dataColumn,row,datasetConfig.referenceShearMeasurements)

            sh.write(row, labelColumn, "Power", self.bold_style)
            sh.write(row, dataColumn, datasetConfig.power)
            row += 2

            if datasetConfig.rewsDefined:
                sh.write(row, labelColumn, "Profile Levels", self.bold_style)
                row += 1

                sh.write(row, labelColumn, "Height", self.bold_style)
                sh.write(row, dataColumn, "Speed", self.bold_style)
                sh.write(row, dataColumn + 1, "Direction", self.bold_style)
                row += 1

                for height in sorted(datasetConfig.data.windSpeedLevels):

                    sh.write(row, labelColumn, height)
                    sh.write(row, dataColumn, datasetConfig.data.windSpeedLevels[height])

                    if hasattr(datasetConfig.data, 'windDirectionLevels'): # we are not using this in REWS yet
                        if height in datasetConfig.data.windDirectionLevels:
                            sh.write(row, dataColumn + 1, datasetConfig.data.windDirectionLevels[height])

                    row += 1

            sh.write(row, labelColumn, "Filters", self.bold_style)
            row += 1

            sh.write(row, labelColumn, "Data Column", self.bold_style)
            sh.write(row, dataColumn, "Filter Type", self.bold_style)
            sh.write(row, dataColumn + 1, "Inclusive", self.bold_style)
            sh.write(row, dataColumn + 2, "Filter Value", self.bold_style)
            sh.write(row, dataColumn + 3, "Active", self.bold_style)
            row += 1

            for filter in datasetConfig.filters:
                if isinstance(Filter,TimeOfDayFilter):
                    sh.write(row, labelColumn, "Time Of Day Filter")
                    sh.write(row, dataColumn,     str(filter.startTime))
                    sh.write(row, dataColumn + 1, str(filter.endTime))
                    sh.write(row, dataColumn + 2, str(filter.daysOfTheWeek))
                    sh.write(row, dataColumn + 3, str(filter.months))
                else:
                    sh.write(row, labelColumn, filter.column)
                    sh.write(row, dataColumn, filter.filterType)
                    sh.write(row, dataColumn + 1, filter.inclusive)
                    sh.write(row, dataColumn + 2, str(filter))
                    sh.write(row, dataColumn + 3, "True") # always true if in list...

                row += 1

    def writeShear(self,sh,labelColumn,dataColumn,row,shearList,prefix=""):
        i = 0
        for sh_meas in shearList:
            sh.write(row, labelColumn, prefix+"Shear Measurement " + str(i+1), self.bold_style)
            sh.write(row, dataColumn, sh_meas.wind_speed_column)
            row += 1
            sh.write(row, labelColumn, prefix+"Shear Measurement {0} Height ".format(i+1), self.bold_style)
            sh.write(row, dataColumn, sh_meas.height)
            row += 1
            i += 1
        return row

    def reportPowerCurve(self, sh, rowOffset, columnOffset, name, powerCurve, analysis):

        powerCurveLevels = powerCurve.data_frame.copy()

        if powerCurve.wind_speed_column is None:
            powerCurveLevels['Specified Wind Speed'] = powerCurveLevels.index
            wind_speed_col = 'Specified Wind Speed'
        else:
            wind_speed_col = powerCurve.wind_speed_column

        powerCurveLevels = powerCurveLevels.sort(wind_speed_col)

        sh.write(rowOffset, columnOffset + 2, name, self.bold_style)

        sh.col(columnOffset + 1).width = 256 * 15
        sh.col(columnOffset + 2).width = 256 * 15
        sh.col(columnOffset + 3).width = 256 * 15

        if powerCurve.wind_speed_column is None:
            sh.col(columnOffset + 5).width = 256 * 5
        else:
            sh.col(columnOffset + 4).width = 256 * 15

        sh.col(columnOffset + 5).width = 256 * 5

        rowOrders = {'Data Count': 4, analysis.actualPower: 2,   analysis.hubTurbulence: 3,
                     analysis.baseline.wind_speed_column: 1, 'Specified Power': 2, 'Specified Turbulence': 3,
                     'Specified Wind Speed': 1, analysis.measuredTurbulencePower:2, wind_speed_col: 1}

        styles = {'Data Count': self.no_dp_style, analysis.baseline.wind_speed_column: self.two_dp_style,
                  analysis.actualPower: self.no_dp_style,  analysis.hubTurbulence: self.percent_no_dp_style,
                  'Specified Power': self.no_dp_style, 'Specified Turbulence': self.percent_no_dp_style,
                  'Specified Wind Speed': self.two_dp_style, analysis.measuredTurbulencePower: self.no_dp_style,
                  wind_speed_col: self.two_dp_style}

        for colname in powerCurveLevels.columns:
            if colname in styles.keys():
                sh.write(rowOffset + 1, columnOffset + rowOrders[colname], colname, self.bold_style)

        countRow = 1
        for windSpeed in powerCurveLevels.index:
            for colname in powerCurveLevels.columns:
                if colname in styles.keys():
                    val = powerCurveLevels[colname][windSpeed]
                    if type(val) is np.int64:
                        #xlwt needs numbers to be recognisable as integers or floats; isinstance(np.int64(1), int) returns False.
                        #Other numpy types (int32, float64, etc) are recognised as int and float appropriately.
                        val = int(val)
                    sh.write(rowOffset + countRow + 1, columnOffset + rowOrders[colname], val, styles[colname])
            countRow += 1

        if hasattr(powerCurve, 'zeroTurbulencePowerCurve'):
            countRow += 3
            try:
                pc = powerCurve.zeroTurbulencePowerCurve.dfPowerLevels
                sh.write(rowOffset + countRow, columnOffset + 2, name + ' Zero TI Power Curve', self.bold_style)
                countRow += 1
                sh.write(rowOffset + countRow, columnOffset + 1, 'Wind Speed', self.bold_style)
                sh.write(rowOffset + countRow, columnOffset + 2, 'Power', self.bold_style)
                for ws in pc.index:
                    sh.write(rowOffset + countRow + 1, columnOffset + 1, ws, styles['Specified Wind Speed'])
                    sh.write(rowOffset + countRow + 1, columnOffset + 2, pc.loc[ws, 'Power'], styles['Specified Wind Speed'])
                    countRow += 1
            except:
                sh.write(rowOffset + countRow, columnOffset + 2,'Zero TI Power Curve not calculated successfully for %s power curve.' % name)
                countRow+=1
        else:
            countRow += 3
            Status.add("Not reporting zero TI power curve for %s as it is not defined." % (name), verbosity=2)
            sh.write(rowOffset + countRow, columnOffset + 2,"Not reporting zero TI power curve for %s as it is not defined." % (name))
            countRow+=1

        return countRow

    def reportInterpolatedPowerCurve(self, sh, rowOffset, columnOffset, name, powerCurve, levels):

        sh.write(rowOffset, columnOffset + 2, name, self.bold_style)
        sh.write(rowOffset + 1, columnOffset + 1, "Wind Speed", self.bold_style)
        sh.write(rowOffset + 1, columnOffset + 2, "Power", self.bold_style)
        sh.write(rowOffset + 1, columnOffset + 3, "Turbulence", self.bold_style)

        count = 1
        for windSpeed in sorted(levels):
            sh.write(rowOffset + count + 1, columnOffset + 1, windSpeed, self.two_dp_style)
            sh.write(rowOffset + count + 1, columnOffset + 2, float(powerCurve.power_function(windSpeed)), self.no_dp_style)
            sh.write(rowOffset + count + 1, columnOffset + 3, float(powerCurve.turbulence_function(windSpeed)), self.percent_no_dp_style)
            count += 1

    def reportPowerDeviations(self, book, sheetName, powerDeviations, gradient):

        sheet = PowerDeviationMatrixSheet(self.calculated_power_deviation_matrix_dimensions)
        sheet.report(book, sheetName, powerDeviations, gradient)

    def report_aep(self,sh,analysis):
        sh # get tables in PP report form
        # Summary of EY acceptance test results:
        hrsMultiplier = (analysis.timeStepInSeconds/3600.0)
        row = 2
        tall_style = xlwt.easyxf('font:height 360;') # 18pt
        first_row = sh.row(row)
        first_row.set_style(tall_style)
        sh.write(row,2, "Reference Turbine", self.bold_style)
        sh.write(row,3, "Measured (LCB) Pct of Warranted Annual Energy Yield (%)", self.bold_style)
        sh.write(row,4, "Extrapolated Pct of Warranted Annual Energy Yield (%)", self.bold_style)
        sh.write(row,5, "Last Complete Bin (LCB)", self.bold_style)
        sh.write(row,6, "Direction Sectors Analysed (degrees)", self.bold_style)
        sh.write(row,7, "Measured Hours", self.bold_style)
        #sh.write(row,8, "Annual Energy Yield Uncertainty as a percentage of the Warranted Annual Yield (%)", self.bold_style)
        row += 1
        sh.write(row,2, analysis.Name)
        sh.write(row,3, analysis.aepCalcLCB.AEP*100, self.two_dp_style)
        sh.write(row,4, analysis.aepCalc.AEP*100, self.two_dp_style)
        sh.write(row,5, analysis.aepCalcLCB.lcb, self.two_dp_style)
        sh.write(row,6, "{mi} - {ma}".format(mi=analysis.dataFrame[analysis.windDirection].min(),ma=analysis.dataFrame[analysis.windDirection].max()))
        timeCovered = analysis.allMeasuredPowerCurve.data_frame[analysis.dataCount].sum() * hrsMultiplier
        sh.write(row,7, timeCovered, self.two_dp_style)
        #sh.write(row,8, "NOT YET CALCULATED")

        row += 3
        if hasattr(analysis.specified_power_curve,"referenceDensity"):
            sh.write_merge(row,row,2,6, "Measured Power Curve\n Reference Air Density = {ref} kg/m^3".format(ref=analysis.specified_power_curve.referenceDensity), self.bold_style)
        #sh.write(row,7, "Category A Uncertainty", self.bold_style)
        #sh.write(row,8, "Category B Uncertainty", self.bold_style)
        #sh.write(row,9, "Category C Uncertainty", self.bold_style)
        row += 1

        sh.write(row,2, "Bin No", self.bold_style)

        sh.write(row,3, "Bin Centre Wind Speed", self.bold_style)

        sh.write(row,4, "Hub Height Wind Speed", self.bold_style)
        sh.write(row,5, "Power Output", self.bold_style)
        sh.write(row,6, "Cp", self.bold_style)
        sh.write(row,7, "Qty 10-Min Data", self.bold_style)
        sh.write(row,8, "Standard Deviation", self.bold_style)

        #sh.write(row,7, "Standard Uncertainty", self.bold_style)
        #sh.write(row,8, "Standard Uncertainty", self.bold_style)
        #sh.write(row,9, "Standard Uncertainty", self.bold_style)

        row += 1
        sh.write(row,2, "I", self.bold_style)

        sh.write(row,3, "Vi_centre", self.bold_style)

        sh.write(row,4, "Vi", self.bold_style)
        sh.write(row,5, "Pi", self.bold_style)

        sh.write(row,7, "Ni", self.bold_style)

        sh.write(row,8, "StDev i", self.bold_style)

        #sh.write(row,7, "si", self.bold_style)
        #sh.write(row,8, "ui", self.bold_style)
        #sh.write(row,9, "uc,I", self.bold_style)
        row += 1
        sh.write(row,3, "[m/s]", self.bold_style)
        sh.write(row,4, "[kW]", self.bold_style)

        sh.write(row,8, "[kW]", self.bold_style)

        #sh.write(row,7, "[kW]", self.bold_style)
        #sh.write(row,8, "[kW]", self.bold_style)
        #sh.write(row,9, "[kW]", self.bold_style)

        for binNo,ws in enumerate(analysis.allMeasuredPowerCurve.data_frame.index):
            if ws <= analysis.aepCalcLCB.lcb and analysis.allMeasuredPowerCurve.data_frame[analysis.dataCount][ws] > 0:
                row+=1
                sh.write(row,2, binNo+1, self.no_dp_style)

                sh.write(row,3, ws, self.one_dp_style)

                sh.write(row,4, analysis.allMeasuredPowerCurve.data_frame[analysis.baseline.wind_speed_column][ws], self.two_dp_style)
                sh.write(row,5, analysis.allMeasuredPowerCurve.data_frame[analysis.actualPower][ws], self.two_dp_style)
                if analysis.powerCoeff in analysis.allMeasuredPowerCurve.data_frame.columns:
                    sh.write(row,6, analysis.allMeasuredPowerCurve.data_frame[analysis.powerCoeff][ws], self.two_dp_style)
                else:
                    sh.write(row,6, "-", self.no_dp_style)
                datCount = analysis.allMeasuredPowerCurve.data_frame[analysis.dataCount][ws]
                sh.write(row,7, datCount, self.no_dp_style)

                if analysis.powerStandDev in analysis.allMeasuredPowerCurve.data_frame.columns:
                    sh.write(row,8, analysis.allMeasuredPowerCurve.data_frame[analysis.powerStandDev][ws])
                else:
                    sh.write(row,8, "-", self.no_dp_style)

                #sh.write(row,7, "-", self.no_dp_style)
                #sh.write(row,8, "~", self.no_dp_style)
                #sh.write(row,9, "-", self.no_dp_style)

        row+=2
        sh.write_merge(row,row,2,5, "More than 180 hours of data:", self.bold_style)
        sh.write(row,6, "TRUE" if timeCovered  > 180 else "FALSE")
        sh.write(row,7, "({0} Hours)".format(round(timeCovered,2)) , self.two_dp_style)
        row+=1
        if hasattr(analysis,"windSpeedAt85pctX1pnt5"):
            sh.write_merge(row,row,2,5, "Largest WindSpeed > {0}:".format(round(analysis.windSpeedAt85pctX1pnt5,2)), self.bold_style)
            sh.write(row,6, "TRUE" if analysis.aepCalcLCB.lcb > analysis.windSpeedAt85pctX1pnt5 else "FALSE")
            sh.write(row,7, "Threshold is 1.5*(WindSpeed@0.85*RatedPower)")
            row+=1
        sh.write_merge(row,row,2,5, "AEP Extrap. within 1% of AEP LCB:",self.bold_style)
        ans = abs(1-(analysis.aepCalc.AEP/analysis.aepCalcLCB.AEP)) < 0.01
        sh.write(row,6, "TRUE" if ans else "FALSE")
        if not ans:
             sh.write(row,8, analysis.aepCalc.AEP)
             sh.write(row,9, analysis.aepCalcLCB.AEP)
        if analysis.turbRenormActive:
            row += 2
            sh.write(row,3, "Turbulence Corrected Measured (LCB) Pct of Warranted Annual Energy Yield (%)", self.bold_style)
            sh.write(row,4, "Turbulence Corrected Extrapolated Pct of Warranted Annual Energy Yield (%)", self.bold_style)
            sh.write(row+1,3, analysis.turbCorrectedAepCalcLCB.AEP*100, self.two_dp_style)
            sh.write(row+1,4, analysis.turbCorrectedAepCalc.AEP*100, self.two_dp_style)
        row+=2

        sh.write_merge(row,row,3,10,"AEP Distribution",self.bold_style)
        row+=1
        sh.write_merge(row,row,3,6, "Reference", self.bold_style)
        sh.write_merge(row,row,7,10, "Measured", self.bold_style)
        row+=1
        sh.write(row,2,"Wind Speed",self.bold_style)
        sh.write(row,3,'Reference Freq',self.bold_style)
        sh.write(row,4,'Reference Power',self.bold_style)
        sh.write(row,5,'Reference Power (Resampled)',self.bold_style)
        sh.write(row,6,"Reference Energy",self.bold_style)
        sh.write(row,7,'Measured Freq',self.bold_style)
        sh.write(row,8,'Measured Power',self.bold_style)
        sh.write(row,9,'Measured Power (Resampled)',self.bold_style)
        sh.write(row,10,"Measured Energy",self.bold_style)
        for binNum in analysis.aepCalc.energy_distribution.index:
            row+=1
            sh.write(row,2,binNum,self.two_dp_style)
            sh.write(row,3,analysis.aepCalc.energy_distribution.loc[binNum,"Reference_Freq"] ,self.four_dp_style)
            sh.write(row,4,analysis.aepCalc.energy_distribution.loc[binNum,"Reference_Upper"] ,self.four_dp_style)
            sh.write(row,5,analysis.aepCalc.energy_distribution.loc[binNum,"Reference_Power"] ,self.four_dp_style)
            sh.write(row,6,analysis.aepCalc.energy_distribution.loc[binNum,"Reference_Energy"] ,self.four_dp_style)
            sh.write(row,7,analysis.aepCalc.energy_distribution.loc[binNum,"Measured_Freq"] ,self.four_dp_style)
            sh.write(row,8,analysis.aepCalc.energy_distribution.loc[binNum,"Measured_Upper"] ,self.four_dp_style)
            sh.write(row,9,analysis.aepCalc.energy_distribution.loc[binNum,"Measured_Power"] ,self.four_dp_style)
            sh.write(row,10,analysis.aepCalc.energy_distribution.loc[binNum,"Measured_Energy"] ,self.four_dp_style)
        row+=3

    def write_power_curves(self):

        Status.add("Wind Speed\tSpecified\tInner\tOuter\tAll", verbosity=2)

        for i in range(self.windSpeedBins.numberOfBins):

            windSpeed = self.windSpeedBins.binCenterByIndex(i)

            text = "%0.4f\t" % windSpeed

            if windSpeed in self.specified_power_curve.data_frame:
                text += "%0.4f\t" % self.specified_power_curve.data_frame[windSpeed]
            else:
                text += "\t"

            if windSpeed in self.innerMeasuredPowerCurve.data_frame:
                text += "%0.4f\t" % self.innerMeasuredPowerCurve.data_frame[windSpeed]
            else:
                text += "\t"

            if windSpeed in self.outerMeasuredPowerCurve.data_frame:
                text += "%0.4f\t" % self.outerMeasuredPowerCurve.data_frame[windSpeed]
            else:
                text += "\t"

            if windSpeed in self.allMeasuredPowerCurve.data_frame:
                text += "%0.4f\t" % self.allMeasuredPowerCurve.data_frame[windSpeed]
            else:
                text += "\t"

            Status.add(text, verbosity=2)

    def write_power_deviation_matrix(self):

        for j in reversed(range(self.turbulenceBins.numberOfBins)):

            turbulence = self.turbulenceBins.binCenterByIndex(j)

            text = "%f\t" % turbulence

            for i in range(self.windSpeedBins.numberOfBins):

                windSpeed = self.windSpeedBins.binCenterByIndex(i)

                if windSpeed in self.powerDeviations:
                    if turbulence in self.powerDeviations[windSpeed]:
                        text += "%f\t" % self.powerDeviations[windSpeed][turbulence]
                    else:
                        text += "\t"
                else:
                    text += "\t"

            Status.add(text, verbosity=2)

        text = "\t"

        for i in range(self.windSpeedBins.numberOfBins):
            text += "%f\t" % self.windSpeedBins.binCenterByIndex(i)

        Status.add(text, verbosity=2)

    def report_scatter_metric(self,sh,analysis,row, turbRenormActive):
        row += 5
        sh.write(row,   1, "Scatter Metric Before TI Renormalisation:", self.bold_style)
        sh.write(row+1, 1, analysis.powerCurveScatterMetric, self.percent_style)
        if turbRenormActive:
            sh.write(row,   2, "Scatter Metric After TI Renormalisation:", self.bold_style)
            sh.write(row+1, 2, analysis.powerCurveScatterMetricAfterTiRenorm , self.percent_style)
        return row + 3

class AnonReport(Report):

    def __init__(self,targetPowerCurve,wind_bins, turbulence_bins, version="unknown"):

        self.version = version
        self.targetPowerCurve = targetPowerCurve
        self.turbulenceBins = turbulence_bins
        self.normalisedWindSpeedBins = wind_bins

    def report(self, path, analysis, powerDeviationMatrix = True, scatterMetric=True):

        self.analysis = analysis
        book = xlwt.Workbook()

        sh = book.add_sheet("Anonymous Report", cell_overwrite_ok=True)
        sh.write(0, 0, "PCWG Tool Version Number:")
        sh.write(0, 1, self.version)
        sh.write(0, 2, xlwt.Formula('HYPERLINK("http://www.pcwg.org";"PCWG Website")'))
        row = 1

        if powerDeviationMatrix:
            row = self.report_power_deviation_matrix(sh,analysis,book)

        if scatterMetric:
            row = self.report_scatter_metric(sh,analysis,row, analysis.turbRenormActive)

        book.save(path)

    def report_power_deviation_matrix(self,sh,analysis,book):

        gradient = colour.ColourGradient(-0.1, 0.1, 0.01, book)
        pcStart = 2
        pcEnd   = pcStart + self.normalisedWindSpeedBins.numberOfBins + 5

        deviationMatrixStart = pcEnd + 5
        row= []

        row.append( self.reportPowerCurve(sh, pcStart, 0, self.targetPowerCurve.name + ' Power Curve', self.targetPowerCurve) )

        row.append( self.reportPowerDeviations(sh,deviationMatrixStart, analysis.normalisedHubPowerDeviations, gradient, "Hub Power"))

        if analysis.normalisedTurbPowerDeviations != None:
            deviationMatrixStart += (self.turbulenceBins.numberOfBins + 5) * 2
            row.append(self.reportPowerDeviations(sh,deviationMatrixStart, analysis.normalisedTurbPowerDeviations, gradient, "Turb Corrected Power") )

        return max(row)

    def reportPowerDeviations(self,sh, startRow, powerDeviations, gradient, name):

        countShift = self.turbulenceBins.numberOfBins + 5

        sh.write(startRow, 1, "Deviations Matrix (%s)" % name, self.bold_style)
        sh.write(startRow + countShift, 1, "Data Count Matrix (%s)" % name, self.bold_style)

        for j in range(self.turbulenceBins.numberOfBins):

            turbulence = self.turbulenceBins.binCenterByIndex(j)

            row = startRow + self.turbulenceBins.numberOfBins - j
            countRow = row + countShift

            sh.write(row, 0, turbulence, self.percent_no_dp_style)
            sh.write(countRow, 0, turbulence, self.percent_no_dp_style)

            for i in range(self.normalisedWindSpeedBins.numberOfBins):

                windSpeed = self.normalisedWindSpeedBins.binCenterByIndex(i)
                col = i + 1

                if j == 0:
                    sh.write(row + 1, col, windSpeed, self.two_dp_style)
                    sh.write(countRow + 1, col, windSpeed, self.two_dp_style)

                if windSpeed in powerDeviations.matrix:
                    if turbulence in powerDeviations.matrix[windSpeed]:

                        deviation = powerDeviations.matrix[windSpeed][turbulence]
                        count = int(powerDeviations.count[windSpeed][turbulence])

                        if not np.isnan(deviation):
                            sh.write(row, col, deviation, gradient.getStyle(deviation))
                            sh.write(countRow, col, count, self.no_dp_style)

        return startRow + self.turbulenceBins.numberOfBins + countShift

    def reportPowerCurve(self, sh, rowOffset, columnOffset, name, powerCurve):

        sh.write(rowOffset, columnOffset + 2, name, self.bold_style)
        rowOrders = { 'Data Count':4, 'Normalised Wind Speed':1,'Normalised Power':2, 'Turbulence':3}

        for colname in rowOrders.keys():
            sh.write(rowOffset + 1, columnOffset + rowOrders[colname], colname, self.bold_style)
            countRow = 1

        for i in range(self.normalisedWindSpeedBins.numberOfBins):

            windSpeed = self.normalisedWindSpeedBins.binCenterByIndex(i)
            mask = self.analysis.dataFrame['Normalised WS Bin'] == windSpeed
            dataCount = self.analysis.dataFrame[mask]['Normalised WS Bin'].count()
            absoluteWindSpeed = windSpeed * self.analysis.observedRatedWindSpeed

            sh.write(rowOffset + countRow + 1, columnOffset + 1, windSpeed, self.two_dp_style)
            sh.write(rowOffset + countRow + 1, columnOffset + 4,
                         dataCount, self.no_dp_style)

            if dataCount > 0:

                sh.write(rowOffset + countRow + 1, columnOffset + 2,
                         float(powerCurve.powerFunction(absoluteWindSpeed))/self.analysis.observedRatedPower, self.two_dp_style)

                sh.write(rowOffset + countRow + 1, columnOffset + 3,
                         float(powerCurve.turbulenceFunction(absoluteWindSpeed)), self.percent_no_dp_style)

            countRow += 1

        return countRow


#!/usr/bin/env python
# coding: utf-8

import requests
import pymongo
import pandas as pd

from splinter import Browser
from bs4 import BeautifulSoup
import time

# #### Open chrome driver

# open chrome driver browser
def init_browser():
    executable_path = {'executable_path': 'chromedriver'}
    return Browser('chrome', **executable_path, headless=False)

# ## NASA Mars News - Collect Latest News Title and Paragraph Text

def scrape():
    browser = init_browser()

    # define url

    mars_news_url = "https://mars.nasa.gov/news/"
    time.sleep(3)
    browser.visit(mars_news_url)
    #putting a sleep function here seems to make the flask application run
    time.sleep(3)
    # create beautiful soup object
    html = browser.html
    mars_news_soup = BeautifulSoup(html, 'html.parser')

    # I added a few time.sleep(3) functions to allow the browser time to scrape the data. Hopefully that works.

    # find the first news title
    news_title = mars_news_soup.body.find("div", class_="content_title").text

    # find the paragraph associated with the first title

    news_p = mars_news_soup.body.find("div", class_="article_teaser_body").text

    time.sleep(3)

    mars_image_url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"

    browser.visit(mars_image_url)

    # create the soup item

    html_image = browser.html
    mars_imaging = BeautifulSoup(html_image, 'html.parser')

    # the large image is within the figue element with class = lede
    image = mars_imaging.body.find("figure", class_="lede")

    #obtaining the url for the photo
    feat_img_url = image.find('figure', class_='lede').a['href']
    featured_image_url = f'https://www.jpl.nasa.gov{feat_img_url}'

    featured_image_url

    # ## Mars Weather

    # open url in browser
    #needs time to load
    time.sleep(3)

    # create a soup item

    # ## Mars Facts
    time.sleep(3)

    # define url
    mars_facts_url = "https://space-facts.com/mars/"

    # read html into pandas
    table = pd.read_html(mars_facts_url)

    # returns the value from an html table

    df = table[2]
    df.columns = ["Description", "Value"]

    # converting data to html table

    mars_facts_html=df.to_html()

    mars_facts_html

    # ## Mars Hemispheres

    # define url and open in browser

    time.sleep(3)

    mars_hemispheres_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"

    browser.visit(mars_hemispheres_url)

    # #### Cerberus hemisphere

    # click on the link for the Cerberus hemisphere

    browser.click_link_by_partial_text('Cerberus')

    # click on the open button to get to enhanced picture
    browser.click_link_by_partial_text('Open')

    # create a soup item
    hemispheres_html = browser.html
    cerberus_soup = BeautifulSoup(hemispheres_html, 'html.parser')

    cerberus = cerberus_soup.body.find('img', class_ = 'wide-image')
    cerberus_img = cerberus['src']

    hem_base_url = 'https://astrogeology.usgs.gov'
    #will store url later
    cerberus_url = hem_base_url + cerberus_img

    # #### Schiaperelli hemisphere

    # define url and open in browser

    time.sleep(3)

    mars_hemispheres_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"

    browser.visit(mars_hemispheres_url)

    # click on the link for the Cerberus hemisphere

    browser.click_link_by_partial_text('Schiaparelli')

    # click on the open button to get to enhanced picture
    browser.click_link_by_partial_text('Open')
    #schiap html page
    # create a soup item
    schiap_html = browser.html
    schiap_soup = BeautifulSoup(schiap_html, 'html.parser')
    #obtaining the image of the schiaparelli
    schiap = schiap_soup.body.find('img', class_ = 'wide-image')
    schiap_img = schiap['src']

    hem_base_url = 'https://astrogeology.usgs.gov'
    schiap_url = hem_base_url + schiap_img
    # print(schiap_url)

    # #### Syrtis hemisphere

    # define url and open in browser

    time.sleep(3)

    mars_hemispheres_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"

    browser.visit(mars_hemispheres_url)

    # click on the link for the Cerberus hemisphere

    browser.click_link_by_partial_text('Syrtis')

    # click on the link for the Cerberus hemisphere

    browser.click_link_by_partial_text('Open')

    # create a soup item
    syrtis_html = browser.html
    syrtis_soup = BeautifulSoup(syrtis_html, 'html.parser')

    syrtis = syrtis_soup.body.find('img', class_ = 'wide-image')
    syrtis_img = syrtis['src']

    hem_base_url = 'https://astrogeology.usgs.gov'
    syrtis_url = hem_base_url + syrtis_img
    # print(syrtis_url)

    # #### Valles hemisphere

    # define url and open in browser

    time.sleep(3)

    mars_hemispheres_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"

    browser.visit(mars_hemispheres_url)

    # click on the link for the Valles hemisphere

    browser.click_link_by_partial_text('Valles')

    # click on the link for the Valles hemisphere

    browser.click_link_by_partial_text('Open')

    # create a soup item
    valles_html = browser.html
    valles_soup = BeautifulSoup(valles_html, 'html.parser')

    valles = valles_soup.body.find('img', class_ = 'wide-image')
    valles_img = valles['src']

    hem_base_url = 'https://astrogeology.usgs.gov'
    valles_url = hem_base_url + valles_img
    # print(valles_url)

    # #### Define list of dictionaries that include each hemisphere

    hemispheres_image_urls = [
        {"title": "Valles Marineris Hemisphere", "img_url": valles_url},
        {"title": "Cerberus Hemisphere", "img_url": cerberus_url},
        {"title": "Schiaparelli Marineris Hemisphere", "img_url": schiap_url},
        {"title": "Syrtis Major Hemisphere", "img_url": syrtis_url}
    ]

    # dictionary should be returned
    mars_dict = {
        'headline': news_title,
        'paragraph':  news_p,
        'featuredimage': featured_image_url,
    #    'currentweather': mars_weather,
        'factstable': mars_facts_html,
        "va_title": "Valles Marineris Hemisphere", "va_img_url": valles_url,
        "ce_title": "Cerberus Hemisphere", "ce_img_url": cerberus_url,
        "sc_title": "Schiaparelli Marineris Hemisphere", "sc_img_url": schiap_url,
        "sy_title": "Syrtis Major Hemisphere", "sy_img_url": syrtis_url}

    # print(mars_dictionary)
    browser.quit()
    return mars_dict


import csv
from collections import defaultdict, OrderedDict
import itertools
import json
class clicknode:
	node_count = itertools.count()
	group_count = itertools.count()
	group_map = {}
	def __init__(self, **nodedict):
		group = nodedict['REGION_VIEW_ID']

		if group not in clicknode.group_map:
			clicknode.group_map[group] = next(clicknode.group_count)

		# use dictionary to populate object's fields
		self.__dict__.update(nodedict)

		self.id = next(clicknode.node_count)

		# that each node is a single entity (used in merging group of nodes)
		self.count = 1

	def to_JSON(self):
		return json.dumps(self, default=lambda o: o.__dict__,
			sort_keys=True, indent=4)

	def __str__(self):
		return str(self.id)+" "+self.REGION_VIEW_ID+" "+self.CLIENT_ID

class clicklink:
	def __init__(self, nodea, nodeb, edge):
		self.source = nodea
		self.dest = nodeb
		self.linkwt = edge

	def __str__(self):
		return ";".join(map(lambda x: str(x), [self.source, self.dest, self.linkwt]))

class linkwt:
	name = None
	def __init__(self, src, dest):
		self.count = 1
		self.val = getattr(dest, self.name)
		self.length = 80

class countwt(linkwt):
	name = "count"

	def merge(self, linkwt):
		self.val+=linkwt.val
		self.count+=linkwt.count

class responsetimewt(linkwt):
	name = "RESPONSE_TIME"

	def merge(self, linkwt):
		self.val = (self.count*self.val + linkwt.val*linkwt.count)/(self.count+linkwt.count)
		self.count += linkwt.count

# make nodes from a click logs of one user
def make_nodes(click_session):
	click_session.sort(key=lambda x:[x[('DATE')], x[('STARTTIME')]])
	last = None
	nodes = []
	links = []
	link_map = {}

	for i in click_session:
		if i['REGION_VIEW_ID'] == '/AtkNotificationFlowTF/AtkNotificationPage':
			continue
		node = clicknode(**i)
		nodes.append(node)

	return {"nodes":nodes}

# make links from the sequence of clicks based on the node field and link type given. such as response time links between all component types or count(frequency) links between all client ids
# returns list of nodes (id, group), list of edges (src, dest, linkwt)
def make_links(nodes, field, link_type):
	#Ordered so that index of a key is constant with updates to the dict
	node_map = OrderedDict()
	if len(nodes) <= 1: return None
	last = nodes[0]
	link_map = {}
	# get the field of an object dynamically
	node_map[(getattr(last,field), last.REGION_VIEW_ID)] = nodes[0]
	links = []
	for node in nodes[1:]:
		# None nodes are breaks representing change of sessions
		if node is None:
			continue

		if node not in node_map:
			# node not in node_map
			node_map[(getattr(node,field), node.REGION_VIEW_ID)] = node

		dest = node_map.keys().index((getattr(node,field), node.REGION_VIEW_ID))
		src = node_map.keys().index((getattr(last,field), last.REGION_VIEW_ID))

		edge = link_type(last, node)
		if (src,dest) not in link_map:
			link = clicklink(src, dest, edge)
			link_map[(src, dest)] = link
		else:
			link = link_map[(src, dest)]
			(link.linkwt).merge(edge)

		last = node

	return (node_map,link_map)

# to put all elements of the same RVID together, create extra links of 0 weight between all nodes of the same RVID
def converge_rvid_nodes(response):
	nodes = response["nodes"]

	# better algorithm: get_pairs(nodes) --> sorts the nodes based on group field so that each group ends at a known index
	# and then for each group, return all pairs of indices
	for i in range(len(nodes)):
		for j in range(i+1, len(nodes)):
			if nodes[i]["group"] == nodes[j]["group"]:
				data = {}
				data["source"] = i
				data["target"] = j
				data["value"] = 0
				data["len"] = 40
				response["links"].append(data)

# input from make_links()
# outputs json string format to be send as response
def jsonify_data(node_data, link_data):
	response = {"nodes":[], "links":[]}
	for field, group in node_data:
		data = {}
		data["name"] = field
		data["group"] = group
		data["prop"] = node_data[(field, group)].__dict__
		response["nodes"].append(data)

	for link in link_data:
		l = link_data[link]
		data = {}
		data["source"] = l.source
		data["target"] = l.dest
		data["value"] = 1
		data["len"] = l.linkwt.length
		response["links"].append(data)

	return response

def parse(lines):
	users = defaultdict(list)
	rvidDict = defaultdict(list)
	ctypeDict = defaultdict(list)
	for line in lines:
		users[line['DSID']].append(line)
		rvidDict[line['REGION_VIEW_ID']].append(line)
		ctypeDict[line['COMPONENT_TYPE']].append(line)

	return users

# pick the session numbered "num" from click history data
def session_fetch(user_data, num, envfilters=[]):
	users = filter(lambda x: user_data[x][0]['ENVIRONMENT'] not in envfilters, user_data)
	num = num%len(users)
	session = user_data[users[num]]
	return make_nodes(session)

def longest_session(users):
	b = map(lambda x: [len(users[x]), x], users)
	click_one = max(b)[1]
	#users[click_one]

	click_session = users[click_one]

	return make_nodes(click_session)

def all_sessions(users):
	nodes = []
	for i in users:
		s = users[i]
		nodes.extend(make_nodes(s)["nodes"])
		nodes.append(None)

	return {"nodes":nodes}


# -*- coding:utf-8 -*-
# ! usr/bin/env python3
"""
Created on 28/12/2020 9:16
@Author: XINZHI YAO
"""

import os
import argparse

def pubtator_split(pubtator_file: str, num_per_file: int,
                   save_path: str):

    if not os.path.exists(save_path):
        os.mkdir(save_path)

    split_file_idx = 0
    file_save_num = 0

    base_prefix = os.path.basename(pubtator_file).split('.')[0]

    save_file = f'{save_path}/{base_prefix}.{split_file_idx}.txt'
    wf = open(save_file, 'w')

    with open(pubtator_file) as f:
        for line in f:
            l = line.strip().split('|')
            if l == ['']:
                pass
                # wf.write('\n')

            if len(l) > 2:
                if l[1] == 't':

                    file_save_num += 1
                    if file_save_num % num_per_file == 0:
                        print(f'{base_prefix}.{split_file_idx}.txt save done.')
                        wf.close()
                        split_file_idx += 1
                        save_file = f'{save_path}/{base_prefix}.{split_file_idx}.txt'
                        wf = open(save_file, 'w')

                    wf.write(f'{line.strip()}\n')

                elif l[1] == 'a':
                    wf.write(f'{line.strip()}\n')
            else:
                wf.write(f'{line.strip()}\n')

    print(f'{base_prefix}.{split_file_idx}.txt save done.')
    wf.close()

if __name__  == '__main__':
    parser = argparse.ArgumentParser(description='PubTator Split.')
    parser.add_argument('-pf', dest='pubtator_file', type=str, required=True)
    parser.add_argument('-pn', dest='pubtator_num_per_file', type=int,
                        default=2000, help='default: 2000')
    parser.add_argument('-sp', dest='split_path', type=str, required=True)
    args = parser.parse_args()

    pubtator_split(args.pubtator_file, args.pubtator_num_per_file, args.split_path)


import numpy as np
import torch
from torchvision import models
import torch.nn as nn
# from resnet import resnet34
# import resnet
from torch.nn import functional as F

class ConvBnRelu(nn.Module):
    def __init__(self, in_planes, out_planes, ksize, stride, pad, dilation=1,
                 groups=1, has_bn=True, norm_layer=nn.BatchNorm2d,
                 has_relu=True, inplace=True, has_bias=False):
        super(ConvBnRelu, self).__init__()
        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=ksize,
                              stride=stride, padding=pad,
                              dilation=dilation, groups=groups, bias=has_bias)
        self.has_bn = has_bn
        if self.has_bn:
            self.bn = nn.BatchNorm2d(out_planes)
        self.has_relu = has_relu
        if self.has_relu:
            self.relu = nn.ReLU(inplace=inplace)

    def forward(self, x):
        x = self.conv(x)
        if self.has_bn:
            x = self.bn(x)
        if self.has_relu:
            x = self.relu(x)

        return x

class double_conv(nn.Module):
    '''(conv => BN => ReLU) * 2'''

    def __init__(self, in_ch, out_ch, reduction=16):
        super(double_conv, self).__init__()

        self.conv = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 3, padding=1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(),
            nn.Conv2d(out_ch, out_ch, 3, padding=1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU()
        )
        self.channel_conv = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, kernel_size=1, stride=1, bias=False),
            nn.BatchNorm2d(out_ch)
        )

    def forward(self, x):
        residual = x
        x = self.conv(x)
        # x = self.se(x)

        if residual.shape[1] != x.shape[1]:
            residual = self.channel_conv(residual)
        x += residual
        return x

class up_edge(nn.Module):
    def __init__(self, in_ch, out_ch, bilinear=True):
        super(up_edge, self).__init__()

        if bilinear:
            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
        else:
            self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2)

        self.conv = double_conv(in_ch, out_ch)
        self.sigmoid = nn.Sigmoid()
        self.change_ch = nn.Conv2d(int(in_ch), int(in_ch/2), kernel_size=1)
    def forward(self, x1, x2,edge):
        #x1:Decoder x2:Encoder,a_map edge
        # print("x1", x1.size())
        # print("x2", x2.size())
        # print("a_map", a_map.size())

        # print("a_map1", a_map.size())
        x1 = self.up(x1)

        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = F.pad(x1, (diffX // 2, diffX - diffX // 2,
                        diffY // 2, diffY - diffY // 2))

        x = torch.cat([edge,x2, x1], dim=1)
        x = self.conv(x)
        return x

class up(nn.Module):
    def __init__(self, in_ch, out_ch, bilinear=True):
        super(up, self).__init__()

        if bilinear:
            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
        else:
            self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2)

        self.conv = double_conv(in_ch, out_ch)
        self.sigmoid = nn.Sigmoid()
        self.change_ch = nn.Conv2d(int(in_ch), int(in_ch/2), kernel_size=1)
    def forward(self, x1, x2):
        # print("x1", x1.size())
        # print("x2", x2.size())
        # print("a_map", a_map.size())
        x1 = self.up(x1)

        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = F.pad(x1, (diffX // 2, diffX - diffX // 2,
                        diffY // 2, diffY - diffY // 2))

        if x2.shape[1]!=x1.shape[1]:
            x1=self.change_ch(x1)
        # print("x2", x2.shape)
        # print("x1", x1.shape)
        x = torch.cat([x2, x1], dim=1)
        x = self.conv(x)
        return x

class outconv(nn.Module):
    def __init__(self, in_ch, out_ch, dropout=False, rate=0.1):
        super(outconv, self).__init__()
        self.dropout = dropout
        if dropout:
            print('dropout', rate)
            self.dp = nn.Dropout2d(rate)
        self.conv = nn.Conv2d(in_ch, out_ch, 1)

    def forward(self, x):
        if self.dropout:
            x = self.dp(x)
        x = self.conv(x)
        return x

def autopad(k, p=None):
    if p is None:
        p = k // 2 if isinstance(k, int) else [x // 2 for x in k]
    return p

class dual_down(nn.Module):
    def __init__(self, in_ch,out_ch):
        super(dual_down, self).__init__()
        self.conv1 = nn.Sequential(nn.Conv2d(in_ch, in_ch, 3,2,autopad(3, 1),groups=1),nn.ReLU(),nn.Dropout2d())
        self.conv2 = nn.Sequential(nn.Conv2d(2*in_ch, out_ch, 1), nn.ReLU(), nn.Dropout2d())
    def forward(self, x1, x2):
        x1=self.conv1(x1)
        # print("x1",x1.shape,"x2",x2.shape)
        x=torch.cat([x1,x2],dim=1)
        x=self.conv2(x)
        return x

class atten_down(nn.Module):
    def __init__(self, in_ch):
        super(atten_down, self).__init__()
        self.edge_atten = nn.Sequential(nn.Conv2d(in_ch,in_ch,kernel_size=3, padding=1),
                                        nn.Sigmoid())
        self.conv = nn.Conv2d(in_ch, in_ch, kernel_size=3, bias=False)
        self.bn = nn.BatchNorm2d(in_ch, eps=0.001, momentum=0.03)
        self.act = nn.LeakyReLU(0.1, inplace=True)

    def forward(self, mask, edge):
        e_atten=self.edge_atten(edge)
        mask=self.act(self.bn(self.edge_atten(mask)))
        mask=mask*e_atten

        return mask


from typing import Tuple, Optional
import albumentations as A
import cv2
import numpy as np
import torch
import torchvision
from torch.utils.data import Dataset
import os
from PIL import Image
from tqdm import tqdm
import pandas as pd
import pywt
import logging

from utils.image_utils import random_crop_with_transforms, load_image, split_by_wavelets
from utils.tensor_utils import preprocess_image

class WaveletSuperSamplingDataset(Dataset):
    def __init__(self, folder_path, window_size: int = 224, dataset_size: int = 1000):
        images_names_list = os.listdir(folder_path)
        images_names_list.sort()

        self.images_paths = [
            os.path.join(folder_path, image_name)
            for image_name in images_names_list
        ]

        self.window_size = window_size
        self.dataset_size = dataset_size
        self.images_count = len(self.images_paths)

        self.interpolations = [
            cv2.INTER_AREA,
            cv2.INTER_LANCZOS4,
            cv2.INTER_NEAREST,
            cv2.INTER_LINEAR,
            cv2.INTER_CUBIC,
            None
        ]

    def __len__(self):
        return self.dataset_size

    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        image_idx = np.random.randint(0, self.images_count)
        image = load_image(self.images_paths[image_idx])

        if min(image.shape[:2]) < self.window_size:
            logging.info('Image {} so small, resizing!'.format(self.images_paths[image_idx]))
            image = cv2.resize(image, (self.window_size + 5, self.window_size + 5), interpolation=cv2.INTER_AREA)

        crop = random_crop_with_transforms(
            image1=image,
            window_size=self.window_size
        )

        selected_inter_method: Optional[int] = self.interpolations[np.random.randint(0, len(self.interpolations))]
        # TODO: Add transform which changed OpenCV image to LL wavelet representation
        selected_inter_method = None
        ycrcb_ll_crop: Optional[np.ndarray] = None

        if selected_inter_method is not None:
            lr_crop = cv2.resize(
                crop,
                (self.window_size // 2, self.window_size // 2),
                interpolation=selected_inter_method
            )
            ycrcb_ll_crop = cv2.cvtColor(lr_crop, cv2.COLOR_RGB2YCrCb)
            ycrcb_ll_crop = ycrcb_ll_crop.astype(np.float32) / 255.0 * self.window_size * 2

        ycrcb_crop = cv2.cvtColor(crop, cv2.COLOR_RGB2YCrCb)
        y, cr, cb = cv2.split(ycrcb_crop)

        # LL, LH, HL, HH <- C
        y_ll, y_lh, y_hl, y_hh = split_by_wavelets(y)
        cr_ll, cr_lh, cr_hl, cr_hh = split_by_wavelets(cr)
        cb_ll, cb_lh, cb_hl, cb_hh = split_by_wavelets(cb)

        if selected_inter_method is None:
            ycrcb_ll_crop = cv2.merge((y_ll, cr_ll, cb_ll))

        # 9 channels
        gt_wavelets = cv2.merge((y_lh, y_hl, y_hh, cr_lh, cr_hl, cr_hh, cb_lh, cb_hl, cb_hh))

        return preprocess_image(ycrcb_ll_crop), preprocess_image(gt_wavelets, 0, 1), preprocess_image(ycrcb_crop)

class SuperSamplingDataset(WaveletSuperSamplingDataset):
    def __init__(self, folder_path, window_size: int = 224, dataset_size: int = 1000):
        super().__init__(folder_path, window_size, dataset_size)

        self.interpolations = [
            cv2.INTER_AREA,
            cv2.INTER_LANCZOS4,
            cv2.INTER_NEAREST,
            cv2.INTER_LINEAR,
            cv2.INTER_CUBIC
        ]

    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
        image_idx = np.random.randint(0, self.images_count)
        image = load_image(self.images_paths[image_idx])

        if min(image.shape[:2]) < self.window_size:
            logging.info('Image {} so small, resizing!'.format(self.images_paths[image_idx]))
            image = cv2.resize(image, (self.window_size + 5, self.window_size + 5), interpolation=cv2.INTER_AREA)

        crop = random_crop_with_transforms(
            image1=image,
            window_size=self.window_size
        )

        selected_inter_method: int = self.interpolations[np.random.randint(0, len(self.interpolations))]

        low_res_crop = cv2.resize(
            crop,
            (self.window_size // 2, self.window_size // 2),
            interpolation=selected_inter_method
        )

        return preprocess_image(low_res_crop, 0, 1), preprocess_image(crop, 0, 1)


from django.shortcuts import render
from .forms import ProductCreationForm
from .models import Product
from django.contrib import messages
import random

# Create your views here.

def create(request):
    if request.method == 'POST':
        form = ProductCreationForm(request.POST, request.FILES)

        if form.is_valid():

            product = form.save(commit=False)

            while True:
                productNo = random.randint(100000, 999999)
                try:
                    Product.objects.get(orderNo=productNo)
                except:
                    break

            product.productNo = productNo

            try: product.save()
            except:
                messages.warning(request, 'Could not create product')
            else:
                messages.success(request, 'Product Created')

    else:
        form = ProductCreationForm()

    context = {
    "title": "Products",
    "form": form
    }
    return render(request, 'products/create.html.django', context)

def product(request, productId):

    product = Product.objects.get(id=productId)

    context = {
        "title": "Product - "+product.productName,
        "product": product
    }
    return render(request, 'products/product.html.django', context)


import tempfile
import os
import posixpath
import stat
import logging
import collections

from kubeflow.fairing import utils as fairing_utils
from kubeflow.fairing.preprocessors.base import BasePreProcessor
from kubeflow.fairing.builders.append.append import AppendBuilder
from kubeflow.fairing.deployers.job.job import Job
from kubeflow.fairing.deployers.tfjob.tfjob import TfJob
from kubeflow.fairing.constants import constants
from kubeflow.fairing.kubernetes import utils as k8s_utils
from kubeflow.fairing.cloud import storage
from kubeflow.fairing.cloud import gcp
from kubeflow.fairing.frameworks import lightgbm_dist_training_init
from kubeflow.fairing.frameworks import utils

logger = logging.getLogger(__name__)

TRAIN_DATA_FIELDS = ["data", "train", "train_data",
                     "train_data_file", "data_filename"]
TEST_DATA_FIELDS = ["valid", "test", "valid_data", "valid_data_file", "test_data",
                    "test_data_file", "valid_filenames"]
NUM_MACHINES_FILEDS = ["num_machines", "num_machine"]
PORT_FIELDS = ["local_listen_port", "local_port"]
MLIST_FIELDS = ["machine_list_filename",
                "machine_list_file", "machine_list", "mlist"]
OUTPUT_MODEL_FIELDS = ["output_model", "model_output", "model_out"]
INPUT_MODEL_FIELDS = ["input_model", "model_input", "model_in"]
OUTPUT_RESULT_FIELDS = ["output_result", "predict_result", "prediction_result",
                        "predict_name", "prediction_name", "pred_name", "name_pred"]
MACHINE_FIELDS = ["machines", "workers", "nodes"]
TREE_LEARNER_FIELDS = ["tree_learner",
                       "tree", "tree_type", "tree_learner_type"]
ENTRYPOINT = posixpath.join(constants.DEFAULT_DEST_PREFIX, "entrypoint.sh")
LIGHTGBM_EXECUTABLE = "lightgbm"
CONFIG_FILE_NAME = "config.conf"
MLIST_FILE_NAME = "mlist.txt"
BLACKLISTED_FIELDS = PORT_FIELDS + MLIST_FIELDS + MACHINE_FIELDS
WEIGHT_FILE_EXT = ".weight"
DATA_PARALLEL_MODES = ["data", "voting"]

def _modify_paths_in_config(config, field_names, dst_base_dir):
    """modify lightgbm config fields

    :param config: config entries
    :param field_names: list of fields
    :param dst_base_dir: path to destination directory

    """
    field_name, field_value = utils.get_config_value(config, field_names)
    if field_value is None:
        return [], []
    src_paths = field_value.split(",")
    dst_paths = []
    for src_path in src_paths:
        file_name = os.path.split(src_path)[-1]
        dst_paths.append(posixpath.join(dst_base_dir, file_name))
    config[field_name] = ",".join(dst_paths)
    return src_paths, dst_paths

def _update_maps(output_map, copy_files, src_paths, dst_paths):
    """update maps

    :param output_map: output map entries
    :param copy_files: files to be copied
    :param src_paths: source paths
    :param dst_paths: destination paths

    """
    for src_path, dst_path in zip(src_paths, dst_paths):
        if os.path.exists(src_path):
            output_map[src_path] = dst_path
        else:
            copy_files[src_path] = dst_path

def _get_commands_for_file_ransfer(files_map):
    """get commands for file transfer

    :param files_map: files to be mapped

    """
    cmds = []
    for k, v in files_map.items():
        storage_obj = storage.get_storage_class(k)()
        if storage_obj.exists(k):
            cmds.append(storage_obj.copy_cmd(k, v))
        else:
            raise RuntimeError("Remote file {} does't exist".format(k))
    return cmds

def _generate_entrypoint(copy_files_before, copy_files_after, config_file,
                         init_cmds=None, copy_patitioned_files=None):
    """ generate entry point

    :param copy_files_before: previous copied files
    :param copy_files_after: files to be copied
    :param config_file: path to config file
    :param init_cmds:  commands(Default value = None)
    :param copy_patitioned_files:  (Default value = None)

    """
    buf = ["#!/bin/sh",
           "set -e"]
    if init_cmds:
        buf.extend(init_cmds)
    # In data prallel mode, copying files based on RANK of the worker in the cluster.
    # The data is partitioned (#partitions=#workers) and each worker gets one partition of the data.
    if copy_patitioned_files and len(copy_patitioned_files) > 0: #pylint:disable=len-as-condition
        buf.append("case $RANK in")
        for rank, files in copy_patitioned_files.items():
            buf.append("\t{})".format(rank))
            buf.extend(
                ["\t\t" + cmd for cmd in _get_commands_for_file_ransfer(files)])
            buf.append("\t\t;;")
        buf.append("esac")

    # copying files that are common to all workers
    buf.extend(_get_commands_for_file_ransfer(copy_files_before))

    buf.append("echo 'All files are copied!'")
    buf.append("{} config={}".format(LIGHTGBM_EXECUTABLE, config_file))
    for k, v in copy_files_after.items():
        storage_obj = storage.get_storage_class(k)()
        buf.append(storage_obj.copy_cmd(v, k))
    _, file_name = tempfile.mkstemp()
    with open(file_name, 'w') as fh:
        content = "\n".join(buf)
        fh.write(content)
        fh.write("\n")
    st = os.stat(file_name)
    os.chmod(file_name, st.st_mode | stat.S_IEXEC)
    return file_name

def _add_train_weight_file(config, dst_base_dir):
    """add train weight file

    :param config: config entries
    :param dst_base_dir: destination directory

    """
    _, field_value = utils.get_config_value(config, TRAIN_DATA_FIELDS)
    if field_value is None:
        return [], []
    else:
        src_paths = field_value.split(",")
        weight_paths = [x+WEIGHT_FILE_EXT for x in src_paths]
        weight_paths_found = []
        weight_paths_dst = []
        for path in weight_paths:
            found = os.path.exists(path)
            if not found:
                # in case the path is local and doesn't exist
                storage_class = storage.lookup_storage_class(path)
                if storage_class:
                    found = storage_class().exists(path)
            if found:
                weight_paths_found.append(path)
                file_name = os.path.split(path)[-1]
                weight_paths_dst.append(
                    posixpath.join(dst_base_dir, file_name))
        return weight_paths_found, weight_paths_dst

def generate_context_files(config, config_file_name, num_machines):
    """generate context files

    :param config: config entries
    :param config_file_name: config file name
    :param num_machines: number of machines

    """
    # Using ordered dict to have consistent behaviour around order in which
    # files are copied in the worker nodes.
    output_map = collections.OrderedDict()
    copy_files_before = collections.OrderedDict()
    copy_files_after = collections.OrderedDict()
    copy_patitioned_files = collections.OrderedDict()

    # config will be modified inplace in this function so taking a copy
    config = config.copy()  # shallow copy is good enough

    _, tree_learner = utils.get_config_value(config, TREE_LEARNER_FIELDS)
    parition_data = tree_learner and tree_learner.lower() in DATA_PARALLEL_MODES
    remote_files = [(copy_files_before,
                     [TEST_DATA_FIELDS, INPUT_MODEL_FIELDS]),
                    (copy_files_after,
                     [OUTPUT_MODEL_FIELDS, OUTPUT_RESULT_FIELDS])]

    if parition_data:
        train_data_field, train_data_value = utils.get_config_value(
            config, TRAIN_DATA_FIELDS)
        train_files = train_data_value.split(",")
        if len(train_files) != num_machines:
            raise RuntimeError("#Training files listed in the {}={} field in the config should be "
                               "equal to the num_machines={} config value."\
                                .format(train_data_field, train_data_value, num_machines))
        weight_src_paths, weight_dst_paths = _add_train_weight_file(config,
                                                                    constants.DEFAULT_DEST_PREFIX)
        dst = posixpath.join(constants.DEFAULT_DEST_PREFIX, "train_data")
        config[train_data_field] = dst
        for i, f in enumerate(train_files):
            copy_patitioned_files[i] = collections.OrderedDict()
            copy_patitioned_files[i][f] = dst
            if f+WEIGHT_FILE_EXT in weight_src_paths:
                copy_patitioned_files[i][f +
                                         WEIGHT_FILE_EXT] = dst+WEIGHT_FILE_EXT
    else:
        train_data_field, train_data_value = utils.get_config_value(
            config, TRAIN_DATA_FIELDS)
        if len(train_data_value.split(",")) > 1:
            raise RuntimeError("{} has more than one file specified but tree-learner is set to {} "
                               "which can't handle multiple files. For distributing data across "
                               "multiple workers, please use one of {} as a tree-learner method. "
                               "For more information please refer the LightGBM parallel guide"
                               " https://github.com/microsoft/LightGBM/blob/master/docs/"
                               "Parallel-Learning-Guide.rst".format(
                                   train_data_field, tree_learner, DATA_PARALLEL_MODES))
        remote_files[0][1].insert(0, TRAIN_DATA_FIELDS)
        weight_src_paths, weight_dst_paths = _add_train_weight_file(config,
                                                                    constants.DEFAULT_DEST_PREFIX)
        _update_maps(output_map, copy_files_before, weight_src_paths, weight_dst_paths)

    for copy_files, field_names_list in remote_files:
        for field_names in field_names_list:
            src_paths, dst_paths = _modify_paths_in_config(
                config, field_names, constants.DEFAULT_DEST_PREFIX)
            _update_maps(output_map, copy_files, src_paths, dst_paths)

    if len(output_map) + len(copy_files_before) + len(copy_patitioned_files) == 0:
        raise RuntimeError("Both train and test data is missing in the config")
    modified_config_file_name = utils.save_properties_config_file(config)
    config_in_docker = posixpath.join(
        constants.DEFAULT_DEST_PREFIX, CONFIG_FILE_NAME)
    output_map[modified_config_file_name] = config_in_docker
    output_map[config_file_name] = config_in_docker + ".original"

    init_cmds = None
    if num_machines > 1:
        init_file = lightgbm_dist_training_init.__file__
        init_file_name = os.path.split(init_file)[1]
        output_map[init_file] = os.path.join(
            constants.DEFAULT_DEST_PREFIX, init_file_name)
        init_cmds = ["RANK=`python {} {} {}`".format(init_file_name,
                                                     CONFIG_FILE_NAME,
                                                     MLIST_FILE_NAME)]
    entrypoint_file_name = _generate_entrypoint(
        copy_files_before, copy_files_after, config_in_docker, init_cmds, copy_patitioned_files)
    output_map[entrypoint_file_name] = ENTRYPOINT
    output_map[utils.__file__] = os.path.join(
        constants.DEFAULT_DEST_PREFIX, "utils.py")

    return output_map

def execute(config,
            docker_registry,
            base_image="gcr.io/kubeflow-fairing/lightgbm:latest",
            namespace=None,
            stream_log=True,
            cores_per_worker=None,
            memory_per_worker=None,
            pod_spec_mutators=None):
    """Runs the LightGBM CLI in a single pod in user's Kubeflow cluster.
    Users can configure it to be a train, predict, and other supported tasks
    by using the right config.
    Please refere https://github.com/microsoft/LightGBM/blob/master/docs/Parameters.rst
    for more information on config options.

    :param config: config entries
    :param docker_registry: docker registry name
    :param base_image: base image (Default value = "gcr.io/kubeflow-fairing/lightgbm:latest")
    :param namespace: k8s namespace (Default value = None)
    :param stream_log: should that stream log? (Default value = True)
    :param cores_per_worker: number of cores per worker (Default value = None)
    :param memory_per_worker: memory value per worker (Default value = None)
    :param pod_spec_mutators: pod spec mutators (Default value = None)

    """
    if not namespace and not fairing_utils.is_running_in_k8s():
        namespace = "kubeflow"
    namespace = namespace or fairing_utils.get_default_target_namespace()
    config_file_name = None
    if isinstance(config, str):
        config_file_name = config
        config = utils.load_properties_config_file(config)
    elif isinstance(config, dict):
        config_file_name = utils.save_properties_config_file(config)
    else:
        raise RuntimeError("config should be of type dict or string(filepath) "
                           "but got {}".format(type(dict)))

    utils.scrub_fields(config, BLACKLISTED_FIELDS)

    _, num_machines = utils.get_config_value(config, NUM_MACHINES_FILEDS)
    num_machines = num_machines or 1
    if num_machines:
        try:
            num_machines = int(num_machines)
        except ValueError:
            raise ValueError("num_machines value in config should be an int >= 1 "
                             "but got {}".format(config.get('num_machines')))
        if num_machines < 1:
            raise ValueError(
                "num_machines value in config should >= 1 but got {}".format(num_machines))

    if num_machines > 1:
        config['machine_list_file'] = "mlist.txt"
    output_map = generate_context_files(
        config, config_file_name, num_machines)

    preprocessor = BasePreProcessor(
        command=[ENTRYPOINT], output_map=output_map)
    builder = AppendBuilder(registry=docker_registry,
                            base_image=base_image, preprocessor=preprocessor)
    builder.build()
    pod_spec = builder.generate_pod_spec()

    pod_spec_mutators = pod_spec_mutators or []
    pod_spec_mutators.append(gcp.add_gcp_credentials_if_exists)
    pod_spec_mutators.append(k8s_utils.get_resource_mutator(
        cores_per_worker, memory_per_worker))

    if num_machines == 1:
        # non-distributed mode
        deployer = Job(namespace=namespace,
                       pod_spec_mutators=pod_spec_mutators,
                       stream_log=stream_log)
    else:
        # distributed mode
        deployer = TfJob(namespace=namespace,
                         pod_spec_mutators=pod_spec_mutators,
                         chief_count=1,
                         worker_count=num_machines-1,
                         stream_log=stream_log)
    deployer.deploy(pod_spec)
    return deployer


import matplotlib.pyplot as plot
from pymongo import MongoClient
import numpy as np

from sys import argv
import random
from constants import CONNECTION_STRING, DATABASE_NAME, CLUSTER_COLLECTION_NAME, GENRE_K_DICT
from q2 import get_k_g, main as q2_main, client as client2
from q3 import main as q3_main, client as client3

client = MongoClient(CONNECTION_STRING)
db = client.get_database(DATABASE_NAME)

def get_clusters(g: str) -> list:
    return list(db.get_collection(CLUSTER_COLLECTION_NAME).aggregate([
    {
        '$match': {
            'genres': g
        }
    }, {
        '$group': {
            '_id': '$cluster',
            'points': {
                '$push': '$kmeansNorm'
            }
        }
    }]))

def get_random_color(palette=[]):
    colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
    shapes = ['o', 's', 'p', 'd', 'D', '*', '+']
    color_index = random.randint(0, len(colors) - 1)
    shape_index = random.randint(0, len(shapes) - 1)
    result = f'{colors[color_index]}{shapes[shape_index]}'
    while result in palette:
        color_index = random.randint(0, len(colors) - 1)
        shape_index = random.randint(0, len(shapes) - 1)
        result = f'{colors[color_index]}{shapes[shape_index]}'
    return result

def plot_points(clusters: list, g: str):
    plot.title(g)
    plot.xlabel('Normalized startYear')
    plot.ylabel('Normalized avgRating')
    plot.xticks(np.arange(0, 1.2, 0.1))
    plot.yticks(np.arange(0, 1.2, 0.1))
    for cluster in clusters:
        cluster_colors = []
        cluster_color = get_random_color(cluster_colors)
        cluster_colors.append(cluster_color)
        for point in cluster['points']:
            plot.plot(point[0], point[1], cluster_color, markersize=5)

    plot.savefig(f'./img/q5/{g}.jpg', format='jpg')
    plot.clf()

def main():
    if len(argv) == 1:
        for g in GENRE_K_DICT:
            q2_main(GENRE_K_DICT[g], g)
            q3_main(g)
            clusters = get_clusters(g)
            plot_points(clusters, g)
    else:
        k, g = get_k_g()
        q2_main(k, g)
        q3_main(g)
        clusters = get_clusters(g)
        plot_points(clusters, g)
    client2.close()
    client3.close()

if __name__ == "__main__":
    main()


from django.contrib.auth import get_user_model
from django.db import models

User = get_user_model()

class Group(models.Model):
    title = models.CharField('название группы', max_length=200)
    slug = models.SlugField('слаг', unique=True)
    description = models.TextField('описание')

    class Meta:
        verbose_name = 'группа'
        verbose_name_plural = 'группы'

    def __str__(self):
        return self.title

class Post(models.Model):
    text = models.TextField(
        'текст', help_text='Перед публикацией заполните поле.')
    pub_date = models.DateTimeField(
        'дата публикации', auto_now_add=True)
    author = models.ForeignKey(
        User, on_delete=models.CASCADE,
        related_name='posts', verbose_name='автор')
    group = models.ForeignKey(
        Group, models.SET_NULL, blank=True,
        null=True, related_name='posts', verbose_name='группа',
        help_text='Выберите группу для публикации поста.')
    image = models.ImageField(
        'картинка', upload_to='posts/', blank=True, null=True,
        help_text='Выберите картинку для публикации поста.')

    class Meta:
        verbose_name = 'пост'
        verbose_name_plural = 'посты'
        ordering = ['-pub_date']

    def __str__(self):
        return self.text[:15]

class Comment(models.Model):
    post = models.ForeignKey(
        Post, on_delete=models.CASCADE,
        related_name='comments', verbose_name='пост')
    author = models.ForeignKey(
        User, on_delete=models.CASCADE,
        related_name='comments', verbose_name='автор')
    text = models.TextField(
        'текст комментария', help_text='Перед публикацией заполните поле.')
    created = models.DateTimeField(
        'дата публикации', auto_now_add=True)

    class Meta:
        verbose_name = 'комментарий'
        verbose_name_plural = 'комментарии'
        ordering = ['-created']

    def __str__(self):
        return self.text[:15]

class Follow(models.Model):
    user = models.ForeignKey(
        User, on_delete=models.CASCADE,
        related_name='follower', verbose_name='подписчик')
    author = models.ForeignKey(
        User, on_delete=models.CASCADE,
        related_name='following', verbose_name='автор')

    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields=['user', 'author'],
                name='following_unique',
            ),
        ]


import sys
import cv2 as cv

__doc__ = """Wrapper to create new classifiers from OpenCV or other libraries.
"""

class NormalBayes(object):
    """Wraps a trained OpenCV Normal Bayes Classifier.
    More info: http://docs.opencv.org/modules/ml/doc/normal_bayes_classifier.html
    """

    def __init__(self):
        self.model = cv.NormalBayesClassifier()

    def train(self, dataset, responses):
        """Dataset and responses are assumed to be a 2D and 1D numpy matrix of type np.float32.
        """

        self.model.train(dataset, responses)

    def predict(self, samples):
        """Samples have to be a 2D numpy array of type np.float32.
        Returns a list of prediction values.
        """

        pred_results = self.model.predict(samples)
        return [int(x[0]) for x in pred_results[1]]

class KNN(object):
    """Wraps a trained OpenCV k_nn classifier.
    More info: http://docs.opencv.org/modules/ml/doc/k_nearest_neighbors.html
    """

    def __init__(self):
        self.model = cv.KNearest()
        self.max_K = 32

    def train(self, dataset, responses, params):
        """Dataset and responses are assumed to be a 2D and 1D numpy matrix of type np.float32.
        Additionally, optional max_neighbors argument can be provided.
        """

        if "nmax" in params:
            self.max_K = params["nmax"]
        else:
            self.max_K = 32

        self.model.train(dataset, responses, maxK=self.max_K)

    def predict(self, samples, params):
        """Accepts samples for classification and K - number of neighbors to use.
        Notice: K has to be <= maxK that was set while training.
        Refer here: http://docs.opencv.org/modules/ml/doc/k_nearest_neighbors.html
        for more info. Samples are 2D numpy array of type np.float32.
        Returns a list of prediction values.
        """

        if "nclass" in params:
            K = params["nclass"]
        else:
            K = 7

        if K > self.max_K:
            print ("Bad argument: K")
            return []

        out = self.model.find_nearest(samples, K)
        return [int(x[0]) for x in out[1]]

class RandomTrees(object):
    """Wraps a trained OpenCV RTrees classifier.
    More info: http://docs.opencv.org/modules/ml/doc/random_trees.html
    """

    def __init__(self):
        self.model = cv.RTrees()

    def train(self, dataset, responses, params):
        """Dataset and responses are assumed to be a 2D and 1D numpy matrix of type np.float32.
        max_d corresponds to the max tree depth. Parameter criteria can be:
        --CV_TERMCRIT_ITER Terminate learning by the max_num_of_trees_in_the_forest;
        --CV_TERMCRIT_EPS Terminate learning by the forest_accuracy;
        --CV_TERMCRIT_ITER + CV_TERMCRIT_EPS Use both termination criteria.
        Refer here: http://docs.opencv.org/modules/ml/doc/random_trees.html
        """

        if "maxdepth" in params:
            max_d = params["maxdepth"]
        else:
            max_d = 4
        if "criteria" in params:
            criteria = params["criteria"]
        else:
            criteria=cv.TERM_CRITERIA_MAX_ITER+cv.TERM_CRITERIA_EPS
        if "maxerror" in params:
            max_error = params["maxerror"]
        else:
            max_error = 0.1
        if "maxtrees" in params:
            max_num_trees = params["maxtrees"]
        else:
            max_num_trees = 10

        parameters = dict(max_depth=max_d, min_sample_count=1, use_surrogates=False,
            calc_var_importance=True, max_categories=10, nactive_vars=0,
            term_crit=(criteria, max_num_trees, max_error)) # not sure if max_error belongs here :D
        self.model.train(dataset, cv.CV_ROW_SAMPLE, responses, params=parameters)
        # print ("Num of trees: " + str(self.model.getVarImportance()))

    def predict(self, samples):
        """Returns a list of prediction values for all samples.
        Assuming samples are 2D numpy array of type np.float32.
        """

        return [int(self.model.predict(s)) for s in samples]


#
# @lc app=leetcode.cn id=337 lang=python3
#
# [337] 打家劫舍 III
#
from util import TreeNode
# @lc code=start
from functools import lru_cache
class Solution:
    def rob(self, root: TreeNode) -> int:
        nums = []
        @lru_cache(None)
        def dfs(node: TreeNode, can: bool) -> int:
            if node is None:
                return 0
            node_sum = 0
            if can:
                t_sum = node.val
                t_sum += dfs(node.left, False)
                t_sum += dfs(node.right, False)
                node_sum = t_sum
            t_sum = 0
            t_sum += dfs(node.left, True)
            t_sum += dfs(node.right, True)
            node_sum = max(node_sum, t_sum)
            return node_sum
        return dfs(root, True)
# @lc code=end


from django.conf.urls import include, url
from . import views
from rest_framework.urlpatterns import format_suffix_patterns

urlpatterns = [

	url(r'^$', views.index, name='index'),
	 url(r'^yolog/$', views.yolo_index, name='yolo_index'),
	url(r'^result/$', views.result, name='result'),
	url(r'^list/$', views.RestaurantListView.as_view(), name="rlistview"),
	url(r'^restaurants/$', views.RestaurantAllListView.as_view(), name="rallview"),
	url(r'^restaurant/(?P<venue_id>[\w-]+)/$', views.restaurantwithid,name='rwithid'),
	url(r'^restaurants/map/$', views.RestaurantAllMapListView.as_view(), name="rlistmapview"),
	url(r'^api/v1/$',views.RestaurantList.as_view()),
	url(r'^api/v1/pizzalist/$',views.PizzaList.as_view()),

]

urlpatterns = format_suffix_patterns(urlpatterns)


from utils.parse_json import parse_json
from utils.save_json import save_json
import logging

def put_command(sala: str, nivel: int, chave: str):
    data = parse_json('src/json/comandos.json')
    data[sala][0]['outputs'][nivel]['status'] = chave
    save_json('src/json/comandos.json', data)

def get_command(sala: str, nivel: int):
    data = parse_json('src/json/comandos.json')
    return data[sala][0]['outputs'][nivel]['status']

def swap_command(escolha_input: int, sala: str):
    if (escolha_input == 1):
        if (get_command(sala, 0) == "ON"):
            put_command(sala, 0, 'OFF')
            logging.info('Lamapada 01 Desligada')
        else:
            put_command(sala, 0, 'ON')
            logging.info('Lamapada 01 Ligada')

    if (escolha_input == 2):
        if (get_command(sala, 1) == 'ON'):
            put_command(sala, 1, 'OFF')
            logging.info('Lamapada 02 Desligada')
        else:
            put_command(sala, 1, 'ON')
            logging.info('Lamapada 02 Ligada')

    if (escolha_input == 3):
        if (get_command(sala, 2) == 'ON'):
            put_command(sala, 2, 'OFF')
            logging.info('Projetor Desligado')
        else:
            put_command(sala, 2, 'ON')
            logging.info('Projetor Ligada')

    if (escolha_input == 4):
        if (get_command(sala, 3) == 'ON'):
            put_command(sala, 3, 'OFF')
            logging.info('Ar-condicionado Desligado')
        else:
            put_command(sala, 3, 'ON')
            logging.info('Ar-condicionado Ligado')

    if (escolha_input == 5):
        put_command(sala, 0, 'OFF')
        logging.info('Lamapada 01 Desligada')
        put_command(sala, 1, 'OFF')
        logging.info('Lamapada 02 Desligada')
        put_command(sala, 2, 'OFF')
        logging.info('Projetor Desligado')
        put_command(sala, 3, 'OFF')
        logging.info('Ar-condicionado Desligado')


from flask import Blueprint, request, abort

from epmanage.lib.auth import AuthController, AuthException

auth_component = Blueprint('auth_component', __name__)

@auth_component.route('/', methods=['POST'])
def auth_do():
    """Perform authentication"""
    try:
        return AuthController.get_token_agent(request.json)
    except AuthException:
        abort(503)
    except:
        abort(503)

@auth_component.route('/enroll', methods=['POST'])
def enroll_do():
    """Perform enrollment"""
    try:
        return AuthController.enroll_agent(request.json)
    except AuthException:
        abort(503)
    except:
        abort(503)