#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import sys
import os
import unittest

try:
    from iniparser import *
except:
    _src_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    print('src_path', _src_path)
    sys.path.append(_src_path)

"""
指定运行单个测试用例
python iniparser_test.py -v TestIniParser.test3
"""

class TestIniParser(unittest.TestCase):

    def test1(self):
        p = IniParser()
        p.LoadFromFile("app.conf")
        self.assertEqual(p.getd("appname"), "annotations_tool")
        self.assertEqual(p.getd("httpaddr"), "127.0.0.1")
        self.assertEqual(p.getd("HostURL"), "http://127.0.0.1:8666")

    def test2(self):
        p = IniParser()
        p.LoadFromFile("app.conf")
        p.SaveToFile("newapp.ini")

    def test3(self):
        p = IniParser()
        p.LoadFromFile("test.ini")
        self.assertListEqual(p.sections(),
                             ['Section_a', 'Section_c', 'Section_b'])
        self.assertListEqual(p.items("Section_a"),
                             [('option_a1', 'apple_a1'),
                              ('option_a2', 'banana_a2')])

    def testReadAndWrite(self):
        p = IniParser()
        p.read("test.ini")
        p.write("test2.ini")
        p2 = IniParser()
        p2.read("test2.ini")
        self.assertListEqual(p.sections(), p2.sections())

        for sectionname in p.sections():
            self.assertListEqual(p.items(sectionname), p2.items(sectionname))

    def testRemove(self):
        p = IniParser()
        p.read("test.ini")

        self.assertTrue(p.hassection("Section_a"))
        p.remove("Section_a")
        self.assertFalse(p.hassection("Section_a"))

        self.assertTrue(p.hasKey("Section_b", "option_b2"))
        p.remove("Section_b", "option_b2")
        self.assertFalse(p.hasKey("Section_b", "option_b2"))

    def testAddSection(self):
        p = IniParser()
        p.addsection("section1")
        self.assertTrue(p.hassection("section1"))

    def testKeys(self):
        p = IniParser()
        p.read("test.ini")
        self.assertListEqual(p.keys("Section_a"), ["option_a1", "option_a2"])

    def testGetSet(self):
        p = IniParser()
        p.read("test.ini")
        p.set("Section_a", "name", "huangjian")
        self.assertTrue(p.hasKey("Section_a", "name"))
        self.assertEqual(p.get("Section_a", "name"), "huangjian")

        p.setd("globalname", "ghuangjian")
        self.assertTrue(p.hasKey("default", "globalname"))
        self.assertEqual(p.get("default", "globalname"), "ghuangjian")

    def testGetNotExistKey(self):
        p = IniParser()
        p.read("test.ini")

        self.assertEqual(p.get("not_exist_section", "aa"), "")

        self.assertEqual(p.get("Section_a", "not_exist_key"), "")

def main():
    pass

if __name__ == "__main__":
    main()
    unittest.main()


# -*- coding: utf-8 -*-
"""
Created on Mon May 20 14:57:18 2019

@author: Ethan
"""

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
import matplotlib.cm as cmx
import seaborn as sns
import pandas as pd
import statsmodels.api as sm
import el_load_global_plants
import pickle, gzip
import sys, os

import warnings
warnings.filterwarnings('ignore')

#matplotlib.rcParams['font.family'] = 'Helvetica'
#matplotlib.rcParams['font.weight'] = 'normal'

#dataDir = '/dartfs-hpc/rc/lab/C/CMIG'
#dataDir = 'e:/data/'
dataDirDiscovery = '/dartfs-hpc/rc/lab/C/CMIG/ecoffel/data/projects/electricity'

plotFigs = True

rebuild = True

models = ['bcc-csm1-1-m', 'canesm2', \
              'ccsm4', 'cesm1-bgc', 'cesm1-cam5', 'cnrm-cm5', 'csiro-mk3-6-0', \
              'gfdl-esm2g', 'gfdl-esm2m', \
              'inmcm4', 'miroc5', 'miroc-esm', \
              'mpi-esm-mr', 'mri-cgcm3', 'noresm1-m']

# models = [sys.argv[1]]

#grdc or gldas
runoffData = 'grdc'

# world, useu, or entsoe-nuke
plantData = 'useu'

modelPower = 'pow2-noInteraction'

if plantData == 'useu':
    globalPlants = el_load_global_plants.loadGlobalPlants(world=False)
elif plantData == 'world':
    globalPlants = el_load_global_plants.loadGlobalPlants(world=True)

with open('%s/script-data/active-pp-inds-40-%s.dat'%(dataDirDiscovery, plantData), 'rb') as f:
    livingPlantsInds40 = pickle.load(f)

# load temp-runoff outage models
pcModel10 = []
pcModel50 = []
pcModel90 = []
plantIds = []
plantYears = []
with gzip.open('%s/script-data/pPolyData-%s-%s.dat'%(dataDirDiscovery, runoffData, modelPower), 'rb') as f:
    pPolyData = pickle.load(f)
    # these are mislabeled in dict for now (90 is 10, 10 is 90)
    pcModel10 = pPolyData['pcModel10'][0]
    pcModel50 = pPolyData['pcModel50'][0]
    pcModel90 = pPolyData['pcModel90'][0]
    plantIds = pPolyData['plantIds']
    plantYears = pPolyData['plantYears']
    plantCooling = pPolyData['plantCooling']
    plantFuel = pPolyData['plantFuel']
    plantAge = pPolyData['plantAge']

yearRange = [1981, 2018]

monthLen = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

baseTx = 27
baseQs = 0

dfpred = pd.DataFrame({'T1':[baseTx]*len(plantIds), 'T2':[baseTx**2]*len(plantIds), \
                         'QS1':[baseQs]*len(plantIds), 'QS2':[baseQs**2]*len(plantIds), \
                         'QST':[baseTx*baseQs]*len(plantIds), 'QS2T2':[(baseTx**2)*(baseQs**2)]*len(plantIds), \
                         'PlantIds':plantIds, 'PlantYears':plantYears, 'PlantAge':plantAge, 'PlantCooling':plantCooling, 'PlantFuel':plantFuel})

basePred10 = np.nanmean(pcModel10.predict(dfpred))
basePred50 = np.nanmean(pcModel50.predict(dfpred))
basePred90 = np.nanmean(pcModel90.predict(dfpred))

if not os.path.isfile('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower)) and not rebuild:

    extra=''
    if not 'globalPCHist50' in locals():

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-10%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist10 = globalPCHist['globalPCHist10']

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-50%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist50 = globalPCHist['globalPCHist50']

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-90%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist90 = globalPCHist['globalPCHist90']

    yearlyOutagesHist10 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
    yearlyOutagesHist50 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
    yearlyOutagesHist90 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)

    print('calculating total capacity outage for historical')

    numPlants10 = 0
    numPlants50 = 0
    numPlants90 = 0

    # over all plants that are active in 2018
    for pInd, p in enumerate(livingPlantsInds40[2018]):#range(globalPCHist50.shape[0]):

        if pInd % 500 == 0:
            print('plant %d...'%pInd)

        numYears = 0

        yearlyOutagesHistCurPlant10 = np.full([globalPCHist50.shape[1], 12], np.nan)
        yearlyOutagesHistCurPlant50 = np.full([globalPCHist50.shape[1], 12], np.nan)
        yearlyOutagesHistCurPlant90 = np.full([globalPCHist50.shape[1], 12], np.nan)

        plantHasData10 = False
        plantHasData50 = False
        plantHasData90 = False

        # calculate the total outage (MW) on each day of each year
        for year in range(globalPCHist50.shape[1]):
            yearlyOutagesHistCurYear10 = np.full([12], np.nan)
            yearlyOutagesHistCurYear50 = np.full([12], np.nan)
            yearlyOutagesHistCurYear90 = np.full([12], np.nan)

            numDays10 = 0
            numDays50 = 0
            numDays90 = 0

            for month in range(12):
                curMonthPc = np.reshape(globalPCHist10[p,year,month,:,:], [globalPCHist10[p,year,month,:,:].size])
                monthlyOutages10 = (basePred10-curMonthPc) / 100.0
                monthlyOutages10[monthlyOutages10 < 0] = 0
                monthlyOutages10[monthlyOutages10 > 1] = np.nan
                numHours10 = len(np.where(~np.isnan(monthlyOutages10))[0])
                # this is in MWh
                monthlyTotal10 = np.nansum(globalPlants['caps'][p] * monthlyOutages10)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours10 > 0:
                    monthlyTotal10 /= numHours10
                    # now this too is MWh
                    monthlyTotal10 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear10[month] = monthlyTotal10

                curMonthPc = np.reshape(globalPCHist50[p,year,month,:,:], [globalPCHist50[p,year,month,:,:].size])
                monthlyOutages50 = (basePred50-curMonthPc) / 100.0
                monthlyOutages50[monthlyOutages50 < 0] = 0
                monthlyOutages50[monthlyOutages50 > 1] = np.nan
                numHours50 = len(np.where(~np.isnan(monthlyOutages50))[0])
                # this is in MWh
                monthlyTotal50 = np.nansum(globalPlants['caps'][p] * monthlyOutages50)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours50 > 0:
                    monthlyTotal50 /= numHours50
                    # now this too is MWh
                    monthlyTotal50 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear50[month] = monthlyTotal50

                curMonthPc = np.reshape(globalPCHist90[p,year,month,:,:], [globalPCHist90[p,year,month,:,:].size])
                monthlyOutages90 = (basePred90-curMonthPc) / 100.0
                monthlyOutages90[monthlyOutages90 < 0] = 0
                monthlyOutages90[monthlyOutages90 > 1] = np.nan
                numHours90 = len(np.where(~np.isnan(monthlyOutages90))[0])
                # this is in MWh
                monthlyTotal90 = np.nansum(globalPlants['caps'][p] * monthlyOutages90)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours90 > 0:
                    monthlyTotal90 /= numHours90
                    # now this too is MWh
                    monthlyTotal90 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear90[month] = monthlyTotal90

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear10))[0]) == 12:
                yearlyOutagesHistCurPlant10[year, :] = yearlyOutagesHistCurYear10
                plantHasData10 = True

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear50))[0]) == 12:
                yearlyOutagesHistCurPlant50[year, :] = yearlyOutagesHistCurYear50
                plantHasData50 = True

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear90))[0]) == 12:
                yearlyOutagesHistCurPlant90[year, :] = yearlyOutagesHistCurYear90
                plantHasData90 = True

        # divide by # of years to get total outage per month/year, if there is data
#         if len(yearlyOutagesHistCurPlant10) > 0:
        yearlyOutagesHistCurPlant10 = np.nanmean(yearlyOutagesHistCurPlant10, axis=0)
        if plantHasData10: numPlants10 += 1
        yearlyOutagesHist10[pInd, :] = yearlyOutagesHistCurPlant10

        yearlyOutagesHistCurPlant50 = np.nanmean(yearlyOutagesHistCurPlant50, axis=0)
        if plantHasData50: numPlants50 += 1
        yearlyOutagesHist50[pInd, :] = yearlyOutagesHistCurPlant50

        yearlyOutagesHistCurPlant90 = np.nanmean(yearlyOutagesHistCurPlant90, axis=0)
        if plantHasData90: numPlants90 += 1
        yearlyOutagesHist90[pInd, :] = yearlyOutagesHistCurPlant90

#         if len(yearlyOutagesHistCurPlant50) > 0:
#             yearlyOutagesHistCurPlant50 = np.array(yearlyOutagesHistCurPlant50)
#             yearlyOutagesHistCurPlant50 = np.nanmean(yearlyOutagesHistCurPlant50, axis=0)

#             if plantHasData50: numPlants50 += 1

#             yearlyOutagesHist50.append(yearlyOutagesHistCurPlant50)

#         if len(yearlyOutagesHistCurPlant90) > 0:
#             yearlyOutagesHistCurPlant90 = np.array(yearlyOutagesHistCurPlant90)
#             yearlyOutagesHistCurPlant90 = np.nanmean(yearlyOutagesHistCurPlant90, axis=0)

#             if plantHasData90: numPlants90 += 1

#             yearlyOutagesHist90.append(yearlyOutagesHistCurPlant90)

    # sum over all plants, divide by # plants with data, multiply by total number of plants regardless
    # of whether they have data
    yearlyOutagesHist10 = (np.nansum(yearlyOutagesHist10, axis=0)/numPlants10)*yearlyOutagesHist10.shape[0]
    yearlyOutagesHist50 = (np.nansum(yearlyOutagesHist50, axis=0)/numPlants50)*yearlyOutagesHist50.shape[0]
    yearlyOutagesHist90 = (np.nansum(yearlyOutagesHist90, axis=0)/numPlants90)*yearlyOutagesHist90.shape[0]

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist10, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-50.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist50, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-90.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist90, f)

for model in range(len(models)):

    yearlyOutagesCurModel10 = np.full([4, 12], np.nan)
    yearlyOutagesCurModel50 = np.full([4, 12], np.nan)
    yearlyOutagesCurModel90 = np.full([4, 12], np.nan)

    if not rebuild and os.path.isfile('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])):
        print('skipping %s...'%models[model])
        continue

    for w in range(1,4+1):

        fileName10 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-10-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])
        fileName50 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-50-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])
        fileName90 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-90-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])

        if os.path.isfile(fileName10):
            with open(fileName10, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut10 = globalPC['globalPCFut10']
            with open(fileName50, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut50 = globalPC['globalPCFut50']
            with open(fileName90, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut90 = globalPC['globalPCFut90']
        else:
            print('%s not found!'%fileName10)
            continue

        yearlyOutagesGMT10 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
        yearlyOutagesGMT50 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
        yearlyOutagesGMT90 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)

        print('calculating total capacity outage for %s/+%dC'%(models[model],w))

        # num plants for current model with data
        numPlants10 = 0
        numPlants50 = 0
        numPlants90 = 0

        # over all plants living in 2018
        for pInd, p in enumerate(livingPlantsInds40[2018]):#range(globalPCFut10.shape[0]):
            yearlyOutagesCurPlant10 = np.full([globalPCFut10.shape[1], 12], np.nan)
            yearlyOutagesCurPlant50 = np.full([globalPCFut10.shape[1], 12], np.nan)
            yearlyOutagesCurPlant90 = np.full([globalPCFut10.shape[1], 12], np.nan)

            if pInd%500 == 0:
                print('plant %d...'%pInd)

            plantHasData10 = False
            plantHasData50 = False
            plantHasData90 = False

            # calculate the total outage (MW) on each day of each year
            for year in range(globalPCFut10.shape[1]):
                yearlyOutagesCurYear10 = np.full([12], np.nan)
                yearlyOutagesCurYear50 = np.full([12], np.nan)
                yearlyOutagesCurYear90 = np.full([12], np.nan)

                for month in range(12):

                    curMonthPc = np.reshape(globalPCFut10[p,year,month,:,:], [globalPCFut10[p,year,month,:,:].size])
                    monthlyOutages10 = (basePred10-curMonthPc) / 100.0
                    monthlyOutages10[monthlyOutages10<0] = 0
                    monthlyOutages10[monthlyOutages10>1] = np.nan

                    curMonthPc = np.reshape(globalPCFut50[p,year,month,:,:], [globalPCFut50[p,year,month,:,:].size])
                    monthlyOutages50 = (basePred50-curMonthPc) / 100.0
                    monthlyOutages50[monthlyOutages50<0] = 0
                    monthlyOutages50[monthlyOutages50>1] = np.nan

                    curMonthPc = np.reshape(globalPCFut90[p,year,month,:,:], [globalPCFut90[p,year,month,:,:].size])
                    monthlyOutages90 = (basePred90-curMonthPc) / 100.0
                    monthlyOutages90[monthlyOutages90<0] = 0
                    monthlyOutages90[monthlyOutages90>1] = np.nan
#
#                    indBadData10 = np.where((monthlyOutages10 < 0) | (monthlyOutages10 > 1))[0]
#                    monthlyOutages10[indBadData10] = np.nan
#
#                    indBadData50 = np.where((monthlyOutages50 < 0) | (monthlyOutages50 > 1))[0]
#                    monthlyOutages50[indBadData50] = np.nan
#
#                    indBadData90 = np.where((monthlyOutages90 < 0) | (monthlyOutages90 > 1))[0]
#                    monthlyOutages90[indBadData90] = np.nan

                    numHours10 = len(np.where(~np.isnan(monthlyOutages10))[0])
                    monthlyTotal10 = np.nansum(globalPlants['caps'][p] * monthlyOutages10)

                    # divide by actual # of days in this month, then multiply by full summer (62 days)
                    # this accounts for model/years where there are nans
                    if numHours10 > 0:
                        monthlyTotal10 /= numHours10
                        monthlyTotal10 *= monthLen[month] * 24
                        yearlyOutagesCurYear10[month] = monthlyTotal10

                    numHours50 = len(np.where(~np.isnan(monthlyOutages50))[0])
                    monthlyTotal50 = np.nansum(globalPlants['caps'][p] * monthlyOutages50)

                    if numHours50 > 0:
                        monthlyTotal50 /= numHours50
                        monthlyTotal50 *= monthLen[month] * 24
                        yearlyOutagesCurYear50[month] = monthlyTotal50

                    numHours90 = len(np.where(~np.isnan(monthlyOutages90))[0])
                    monthlyTotal90 = np.nansum(globalPlants['caps'][p] * monthlyOutages90)

                    if numHours90 > 0:
                        monthlyTotal90 /= numHours90
                        monthlyTotal90 *= monthLen[month] * 24
                        yearlyOutagesCurYear90[month] = monthlyTotal90

                if len(np.where(~np.isnan(yearlyOutagesCurYear10))[0]) == 12:
                    yearlyOutagesCurPlant10[year, :] = yearlyOutagesCurYear10
                    plantHasData10 = True

                if len(np.where(~np.isnan(yearlyOutagesCurYear50))[0]) == 12:
                    yearlyOutagesCurPlant50[year, :] = yearlyOutagesCurYear50
                    plantHasData50 = True

                if len(np.where(~np.isnan(yearlyOutagesCurYear90))[0]) == 12:
                    yearlyOutagesCurPlant90[year, :] = yearlyOutagesCurYear90
                    plantHasData90 = True

            yearlyOutagesCurPlant10 = np.nanmean(yearlyOutagesCurPlant10, axis=0)
            if plantHasData10: numPlants10 += 1
            yearlyOutagesGMT10[pInd, :] = yearlyOutagesCurPlant10

            yearlyOutagesCurPlant50 = np.nanmean(yearlyOutagesCurPlant50, axis=0)
            if plantHasData50: numPlants50 += 1
            yearlyOutagesGMT50[pInd, :] = yearlyOutagesCurPlant50

            yearlyOutagesCurPlant90 = np.nanmean(yearlyOutagesCurPlant90, axis=0)
            if plantHasData90: numPlants90 += 1
            yearlyOutagesGMT90[pInd, :] = yearlyOutagesCurPlant90

#             # divide by # of years to get total outage per year, if there is data
#             if len(yearlyOutagesCurPlant10) > 0:
#                 yearlyOutagesCurPlant10 = np.array(yearlyOutagesCurPlant10)
#                 yearlyOutagesCurPlant10 = np.nanmean(yearlyOutagesCurPlant10, axis=0)
#                 yearlyOutagesCurGMT10.append(yearlyOutagesCurPlant10)
#                 if plantHasData10: numPlants10 += 1

#             if len(yearlyOutagesCurPlant50) > 0:
#                 yearlyOutagesCurPlant50 = np.array(yearlyOutagesCurPlant50)
#                 yearlyOutagesCurPlant50 = np.nanmean(yearlyOutagesCurPlant50, axis=0)
#                 yearlyOutagesCurGMT50.append(yearlyOutagesCurPlant50)
#                 if plantHasData50: numPlants50 += 1

#             if len(yearlyOutagesCurPlant90) > 0:
#                 yearlyOutagesCurPlant90 = np.array(yearlyOutagesCurPlant90)
#                 yearlyOutagesCurPlant90 = np.nanmean(yearlyOutagesCurPlant90, axis=0)
#                 yearlyOutagesCurGMT90.append(yearlyOutagesCurPlant90)
#                 if plantHasData90: numPlants90 += 1

        # divide by number of plants
        yearlyOutagesGMT10 = (np.nansum(yearlyOutagesGMT10, axis=0)/numPlants10)*yearlyOutagesGMT10.shape[0]
        yearlyOutagesCurModel10[w-1, :] = yearlyOutagesGMT10

        yearlyOutagesGMT50 = (np.nansum(yearlyOutagesGMT50, axis=0)/numPlants50)*yearlyOutagesGMT50.shape[0]
        yearlyOutagesCurModel50[w-1, :] = yearlyOutagesGMT50

        yearlyOutagesGMT90 = (np.nansum(yearlyOutagesGMT90, axis=0)/numPlants90)*yearlyOutagesGMT90.shape[0]
        yearlyOutagesCurModel90[w-1, :] = yearlyOutagesGMT90

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel10, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-50-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel50, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-90-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel90, f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist10 = pickle.load(f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-50.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist50 = pickle.load(f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-90.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist90 = pickle.load(f)

yearlyOutagesFut10 = []
yearlyOutagesFut50 = []
yearlyOutagesFut90 = []

for model in range(len(models[0])):
    yearlyOutagesCurModel10 = []
    yearlyOutagesCurModel50 = []
    yearlyOutagesCurModel90 = []

    fileName10 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])
    fileName50 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-50-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])
    fileName90 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-90-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])

    if os.path.isfile(fileName10):

        with gzip.open(fileName10) as f:
            yearlyOutagesCurModel10 = pickle.load(f)
        with gzip.open(fileName50) as f:
            yearlyOutagesCurModel50 = pickle.load(f)
        with gzip.open(fileName90) as f:
            yearlyOutagesCurModel90 = pickle.load(f)

        yearlyOutagesFut10.append(yearlyOutagesCurModel10)
        yearlyOutagesFut50.append(yearlyOutagesCurModel50)
        yearlyOutagesFut90.append(yearlyOutagesCurModel90)

# convert all to twh
yearlyOutagesHist10 /= 1e6
yearlyOutagesHist50 /= 1e6
yearlyOutagesHist90 /= 1e6
yearlyOutagesFut10 = np.array(yearlyOutagesFut10)/1e6
yearlyOutagesFut50 = np.array(yearlyOutagesFut50)/1e6
yearlyOutagesFut90 = np.array(yearlyOutagesFut90)/1e6

snsColors = sns.color_palette(["#3498db", "#e74c3c"])

#xd = np.array(list(range(1981, 2018+1)))-1981+1

#z = np.polyfit(xd, mean10[0,:], 1)
#histPolyTx10 = np.poly1d(z)
#z = np.polyfit(xd, mean50[0,:], 1)
#histPolyTx50 = np.poly1d(z)
#z = np.polyfit(xd, mean90[0,:], 1)
#histPolyTx90 = np.poly1d(z)

# in PJ
# yearlyOutagesHist10 = yearlyOutagesHist10/1e18*1e3
# yearlyOutagesHist50 = yearlyOutagesHist50/1e18*1e3
# yearlyOutagesHist90 = yearlyOutagesHist90/1e18*1e3

# yearlyOutagesFut10 = yearlyOutagesFut10/1e18*1e3
# yearlyOutagesFut50 = yearlyOutagesFut50/1e18*1e3
# yearlyOutagesFut90 = yearlyOutagesFut90/1e18*1e3

yearlyOutagesFut10 = np.moveaxis(yearlyOutagesFut10, 1, 0)
yearlyOutagesFut50 = np.moveaxis(yearlyOutagesFut50, 1, 0)
yearlyOutagesFut90 = np.moveaxis(yearlyOutagesFut90, 1, 0)

if plantData == 'world':
    yLimSet = 850
    yTickStep = 200
    yticks = np.arange(0,yLimSet,yTickStep)
else:
    yLimSet = 45
    yTickStep = 10
    yTicks = [0, 10, 20, 30, 40]
    yticks = np.arange(0,45,yTickStep)

# Twh
totalMonthlyEnergy = np.nansum(globalPlants['caps'][livingPlantsInds40[2018]])*30*24/1e6
totalAnnualEnergy = np.nansum(globalPlants['caps'][livingPlantsInds40[2018]])*24*365/1e6
xpos = [1,2,3,4,5,6,7,8,9,10,11,12]

pctEnergyGrid = np.round(yticks/totalAnnualEnergy*100,decimals=1)

outageSumHist = []
outageSum1 = []
outageSum2 = []
outageSum3 = []
outageSum4 = []

for m in range(0,12):
    outageSumHist.append(np.nansum(yearlyOutagesHist50[0:m+1]))
    outageSum1.append(np.nansum(yearlyOutagesFut50[0,:,0:m+1], axis=1))
    outageSum2.append(np.nansum(yearlyOutagesFut50[1,:,0:m+1], axis=1))
    outageSum3.append(np.nansum(yearlyOutagesFut50[2,:,0:m+1], axis=1))
    outageSum4.append(np.nansum(yearlyOutagesFut50[3,:,0:m+1], axis=1))

outageSum1 = np.sort(np.array(outageSum1), axis=1)
outageSum2 = np.sort(np.array(outageSum2), axis=1)
outageSum3 = np.sort(np.array(outageSum3), axis=1)
outageSum4 = np.sort(np.array(outageSum4), axis=1)

#plt.rc('font', family='helvetica', weight='normal')
#plt.rc('axes', labelweight='normal')

plt.figure(figsize=(5,2))
#plt.ylim([0, 320])
plt.ylim([0,yLimSet])
plt.xlim([0,13])
plt.grid(True, alpha = 0.25)
plt.gca().set_axisbelow(True)

plt.plot(xpos, outageSumHist, '-', lw=2, color='black', label='Historical')
plt.plot(xpos, np.nanmean(outageSum2, axis=1), '-', lw=2, color='#ffb835', label='+ 2$\degree$C')
plt.plot(xpos, outageSum2[:,-1], '--', lw=2, color='#ffb835')
plt.plot(xpos, np.nanmean(outageSum4, axis=1), '-', lw=2, color=snsColors[1], label='+ 4$\degree$C')
plt.plot(xpos, outageSum4[:,-1], '--', lw=2, color=snsColors[1])

plt.fill_between(xpos, outageSumHist, [0]*12, facecolor='black', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(outageSum2, axis=1), outageSumHist, facecolor='#ffb835', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(outageSum4, axis=1), np.nanmean(outageSum2, axis=1), facecolor=snsColors[1], alpha=.5, interpolate=True)

plt.xticks(range(1,13))
plt.gca().set_xticklabels(range(1,13))
plt.yticks(yticks)
plt.xlabel('Month', fontname = 'Helvetica', fontsize=16)
#plt.ylabel('Cumulative US-EU outage (EJ)', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

#plt.tick_params(
#    axis='x',          # changes apply to the x-axis
#    which='both',      # both major and minor ticks are affected
#    bottom=False,      # ticks along the bottom edge are off
#    top=False,         # ticks along the top edge are off
#    labelbottom=False) # labels along the bottom edge are off

ax2 = plt.gca().twinx()
plt.xlim([0,13])
plt.ylim([0,yLimSet])
plt.yticks(yticks)
plt.gca().set_yticklabels(pctEnergyGrid)

for tick in plt.gca().yaxis.get_major_ticks():
    tick.label2.set_fontname('Helvetica')
    tick.label2.set_fontsize(14)

if plotFigs:
    plt.savefig('accumulated-annual-outage-cdf-%s-%s-%s.png'%(plantData,runoffData, modelPower), format='png', dpi=500, bbox_inches = 'tight', pad_inches = 0, transparent=True)

if plantData == 'world':
    yLimSet = 160
    yTickStep = 50
else:
    yLimSet = 11.5
    yTickStep = 2

yticks = np.arange(0,yLimSet,yTickStep)
pctEnergyGrid = np.round(yticks/totalMonthlyEnergy*100,decimals=1)

yearlyOutagesFut50GMT2Sorted = np.sort(yearlyOutagesFut50[1,:,:],axis=0)
yearlyOutagesFut50GMT4Sorted = np.sort(yearlyOutagesFut50[3,:,:],axis=0)

plt.figure(figsize=(5,2))
#plt.xlim([0, 7])
plt.xlim([0,13])
plt.ylim([0, yLimSet])
plt.grid(True, alpha = 0.25)
plt.gca().set_axisbelow(True)

plt.plot(xpos, yearlyOutagesHist50, '-', lw=2, color='black', label='Historical')
plt.plot(xpos, np.nanmean(yearlyOutagesFut50[1],axis=0), '-', lw=2, color='#ffb835', label='+ 2$\degree$C')
plt.plot(xpos, yearlyOutagesFut50GMT2Sorted[-2,:], '--', lw=2, color='#ffb835')
plt.plot(xpos, np.nanmean(yearlyOutagesFut50[3],axis=0), '-', lw=2, color=snsColors[1], label='+ 4$\degree$C')
plt.plot(xpos, yearlyOutagesFut50GMT4Sorted[-3,:], '--', lw=2, color=snsColors[1])

plt.fill_between(xpos, yearlyOutagesHist50, [0]*12, facecolor='black', alpha=.5, interpolate=True)
plt.fill_between(xpos, yearlyOutagesHist50, np.nanmean(yearlyOutagesFut50[1],axis=0), facecolor='#ffb835', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(yearlyOutagesFut50[1],axis=0), np.nanmean(yearlyOutagesFut50[3],axis=0), facecolor=snsColors[1], alpha=.5, interpolate=True)

plt.xticks(xpos)
plt.yticks(yticks)
#plt.xlabel('Month', fontname = 'Helvetica', fontsize=16)
#plt.ylabel('Monthly US-EU outage (EJ)', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.tick_params(
    axis='x',          # changes apply to the x-axis
    which='both',      # both major and minor ticks are affected
    bottom=False,      # ticks along the bottom edge are off
    top=False,         # ticks along the top edge are off
    labelbottom=False) # labels along the bottom edge are off

leg = plt.legend(prop = {'size':12, 'family':'Helvetica', 'weight':'normal'}, loc = 'upper left')
leg.get_frame().set_linewidth(0.0)

#leg = plt.legend(prop = {'size':11, 'family':'Helvetica'})
#leg.get_frame().set_linewidth(0.0)

ax2 = plt.gca().twinx()
plt.xlim([0,13])
plt.ylim([0, yLimSet])
plt.yticks(yticks)
plt.gca().set_yticklabels(pctEnergyGrid)
#plt.ylabel('% of US-EU electricity capacity', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().yaxis.get_major_ticks():
    tick.label2.set_fontname('Helvetica')
    tick.label2.set_fontsize(14)

#x0,x1 = plt.gca().get_xlim()
#y0,y1 = plt.gca().get_ylim()
#plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))

if plotFigs:
    plt.savefig('accumulated-monthly-outage-%s-%s-%s.png'%(plantData, runoffData, modelPower), format='png', dpi=500, bbox_inches = 'tight', pad_inches = 0, transparent=True)

plt.show()
sys.exit()


import csv
import numpy as np
from stable_baselines3.common.callbacks import BaseCallback
from stable_baselines3.common.vec_env import VecEnv

class CustomCallback(BaseCallback):
    def __init__(self, csv_file):
        super(CustomCallback, self).__init__()
        self.csv_file = csv_file
        self.writer = None

    def _on_training_start(self) -> None:

        header = ['Reward']
        for i in range(len(self.getValues())):
            header.append(f"Feature{i}")
        with open(self.csv_file, 'w') as f:
            writer = csv.writer(f)
            writer.writerow(header)

    def _on_step(self) -> bool:
        reward = self.locals['rewards'][0]
        flat_obs = self.getValues()
        with open(self.csv_file, 'a') as f:
            writer = csv.writer(f)
            new = [reward]
            new.extend(map(str, flat_obs))
            writer.writerow(new)
        return True

    def getValues(self):
        values = []
        if isinstance(self.training_env, VecEnv):
            obs = np.array(self.training_env.get_attr('last_observation'))
        else:
            obs = np.array(self.training_env.envs[0].last_observation)
        for key in obs[0].keys():
            values.extend(obs[0][key])

        return values


import json
import dotenv
import logging
import os
import hypercorn.asyncio
import hypercorn.config
import quart.flask_patch # type: ignore

from quart import Quart, Response, request

from .api.gateway import connect

from .api.v3.guilds import channels as channels3, core as guilds_core3, messages as messages3
from .api.v3.users import me as me3, core as users_core3
from .api.v3.rate import rater as rater3, _reset as _reset3
from .api.v3.ui import friends as friends3
from .api.v3.database import loop, _init_indexes
from .api.v3.applications import bots as bots3
from .api.v3.errors import Error

app = Quart(__name__)
dotenv.load_dotenv()
app.config['debug'] = True
logging.basicConfig(level=logging.DEBUG)
rater3.init_app(app)

@app.route('/gateway')
async def health_check():
    d = {
        'url': 'wss://gateway.vincentrps.xyz',
    }
    return Response(json.dumps(d), 200)

@app.errorhandler(404)
async def not_found(*_):
    return json.dumps({'message': '404: Not Found', 'code': 0})

@app.errorhandler(500)
async def internal(*_):
    return json.dumps({'message': '500: Internal Server Error', 'code': 0})

@app.errorhandler(429)
async def ratelimited(*_):
    return json.dumps({'message': '429: Too Many Requests', 'retry_after': rater3.current_limit.reset_at})

@app.errorhandler(Error)
async def handle_errors(err: Error):
    return err._to_json()

@app.after_request
async def set_ratelimit(resp: Response):
    print(f'DEBUG:api:Responded to {request.remote_addr} with {(await resp.get_data())}')
    if rater3.current_limit:
        resp.headers.add('X-RateLimit-Limit', rater3.current_limit.limit)
        resp.headers.add('X-RateLimit-Remaining', rater3.current_limit.remaining)
        resp.headers.add('X-RateLimit-Bucket', rater3._key_func())
        resp.headers.add('X-RateLimit-Reset-After', rater3.current_limit.reset_at)
    resp.headers['Content-Type'] = 'application/json'
    return resp

bps = {
    # v3

    channels3.channels:'/v3/guilds',
    guilds_core3.guilds: '/v3/guilds',
    me3.users_me: '/v3/users/@me',
    users_core3.users: '/v3/users',
    friends3.ui: '/v3/ui/friends',
    bots3.bots: '/v3/bots',
    messages3.msgs: '/v3/channels'
}

for value, suffix in bps.items():
    app.register_blueprint(value, url_prefix=suffix)

cfg = hypercorn.config.Config()
cfg.bind.clear()
cfg.bind.append(f'0.0.0.0:{os.getenv("PORT")}')

loop.create_task(connect())
loop.create_task(_init_indexes())
loop.create_task(_reset3())
loop.run_until_complete(hypercorn.asyncio.serve(app, cfg))
loop.run_forever()


from django.urls import reverse
from rest_framework.test import APITestCase
from model_mommy import mommy
from items.models import Item

class ProductsCreateAPITest(APITestCase):
    def setUp(self):
        super().setUp()
        self.view = reverse('api:create')

    def test_create_product(self):
        """
        Test bulk create
        """
        seller = mommy.make('sellers.Seller',
                            name='TestSeller')
        product = {
            'name': 'some name',
            'description': 'some text',
            'seller': seller.id
        }

        response = self.client.post(self.view, product, format='json')
        self.assertEqual(201, response.status_code, response.data)
        product_count = Item.objects.count()
        self.assertEqual(1, product_count, response.data)


import os
import sys
import json
import shutil
import uuid
from setuptools import setup
from setuptools.command.install import install
from ipython_genutils.path import ensure_dir_exists
from notebook.nbextensions import install_nbextension
from notebook.services.config import ConfigManager
from jupyter_core.paths import jupyter_config_dir
from jupyter_core.application import JupyterApp
try:
    input = raw_input
except NameError:
    pass

NB_DIR = os.path.join(os.path.expanduser('~'), '.ipython/nbextensions')
NB_EXT_DIR = os.path.join(os.path.dirname(__file__),
                          'stepsize/stepsize_nb-ext')
CUR_NB_EXT = os.path.join(NB_DIR, 'stepsize_nb-ext')

SERVER_DIR = os.path.join(os.path.expanduser('~'), '.ipython/extensions')
SERVER_EXT_DIR = os.path.join(os.path.dirname(__file__),
                              'stepsize/stepsize_server-ext')
CUR_SERVER_EXT = os.path.join(SERVER_DIR, 'stepsize_server-ext')
SERVER_EXT_CONFIG = [
    "c.NotebookApp.allow_credentials = True",
    "c.NotebookApp.allow_origin_pat = "
    "'(http\://localhost:[0-9]{4}.*|http\://127\.0\.0\.1:[0-9]{4}.*)'",
    "import os",
    "import sys",
    (
        "sys.path.append(os.path.join(os.path.expanduser('~'), "
        "'.ipython/extensions/stepsize_server-ext'))\n"
        "if isinstance(c.NotebookApp.server_extensions, list):\n"
        "    c.NotebookApp.server_extensions.append('server_ext')\n"
        "else:\n"
        "    c.NotebookApp.server_extensions = []\n"
        "    c.NotebookApp.server_extensions.append('server_ext')"
    )
]

UID_DIR = os.path.join(SERVER_DIR, 'stepsize_version-id')
UID = os.path.join(os.path.dirname(__file__), 'stepsize/stepsize_version-id')
CUR_UID = os.path.join(SERVER_DIR, 'stepsize_version-id/uid.json')

class InstallCommand(install):
    def run(self):
        install.run(self)
        self.setup_extensions(CUR_NB_EXT, NB_EXT_DIR, NB_DIR)
        self.setup_extensions(CUR_SERVER_EXT, SERVER_EXT_DIR, SERVER_DIR)
        self.setup_notebook_config()
        self.setup_server_config(SERVER_EXT_CONFIG)
        self.setup_uid(UID_DIR, CUR_UID, UID)
        self.write_uid(CUR_UID)
        print("Stepsize installation complete")

    def query(self, question, default="yes"):
        valid = {"yes": "yes",   "y": "yes",  "ye": "yes",
                 "no": "no",     "n": "no"}
        if not default:
            prompt = " [y/n] "
        elif default == "yes":
            prompt = " [y/n] "
        elif default == "no":
            prompt = " [y/N] "
        else:
            raise ValueError("invalid default answer: '%s'" % default)

        while 1:
            sys.stdout.write(question + prompt)
            choice = input().lower()
            if default is not None and choice == '':
                return default
            elif choice in valid.keys():
                return valid[choice]
            else:
                sys.stdout.write("Please respond with 'yes' or 'no' "
                                 "(or 'y' or 'n').\n")

    def setup_extensions(self, CUR_EXT, EXT_DIR, DIR):
        # Install the extensions to the required directories
        ensure_dir_exists(DIR)
        if os.path.exists(CUR_EXT):
            question = "ACTION: The Stepsize extension directory already " \
                       "exists, do you want to overwrite %s with the " \
                       "current installation?" % (CUR_EXT)
            prompt = self.query(question)
            if prompt == "yes":
                try:
                    install_nbextension(EXT_DIR, overwrite=True,
                                        nbextensions_dir=DIR)
                    print("OUTCOME: Added the extension to your "
                          "%s directory" % (DIR))
                except:
                    print("WARNING: Unable to install the extension to your "
                          "(nb)extensions folder")
                    print("ERROR: %s" % (sys.exc_info()[0]))
                    raise
            else:
                return
        else:
            try:
                install_nbextension(EXT_DIR, overwrite=True,
                                    nbextensions_dir=DIR)
                print("OUTCOME: Added the extension to your %s directory"
                      % (DIR))
            except:
                print("WARNING: Unable to install the extension to your "
                      "(nb)extensions folder")
                print("ERROR: %s" % (sys.exc_info()[0]))
                raise

    def setup_notebook_config(self):
        # Set the notebook extension to launch on startup by
        # modifying the notebook.json
        ensure_dir_exists(ConfigManager()._config_dir_default())
        # Elements to be added
        version = 2
        data_element = {'stepsize_nb-ext/main': None}
        for i in range(1, version):
            data_element['stepsize_nb-ext/main_v0-%i' % i] = None
        data_element['stepsize_nb-ext/main_v0-%i' % version] = True
        data = {'load_extensions': data_element}
        try:
            cm = ConfigManager()
            cm.update('notebook', data)
            print("OUTCOME: Added the Stepsize notebook extension "
                  "configuration to the notebook.json")
        except:
            print("WARNING: An error occured when trying to add %s to the "
                  "notebook.json" % (data))
            print("ERROR: %s" % (sys.exc_info()[0]))
            raise

    def setup_server_config(self, EXT_CONFIG):
        # Set the server extension to launch on startup by modifying the
        # jupyter_notebook_config.py
        CONFIG = os.path.join(jupyter_config_dir(), 'jupyter_config.py')
        ensure_dir_exists(jupyter_config_dir())
        if os.path.isfile(CONFIG):
            pass
        else:
            c = JupyterApp()
            c.write_default_config()
        with open(CONFIG, 'r+') as fh:
            lines = fh.read()
            for i in EXT_CONFIG:
                if i not in lines:
                    fh.seek(0, 2)
                    fh.write('\n')
                    fh.write(i)
        print("OUTCOME: Added the Stepsize server extension "
              "configuration to the jupyter_config.py")

    def setup_uid(self, UID_DIR, CUR_UID, UID):
        if os.path.isfile(CUR_UID):
            pass
        else:
            shutil.copytree(UID, UID_DIR)
            print("OUTCOME: Added the Stepsize extension installation "
                  "identifier to %s" % (os.path.join(os.path.expanduser('~'),
                                                     '.ipython/extensions')))

    def write_uid(self, CUR_UID):
        # Code to write the uuid to the users uuid.json in their directory
        with open(CUR_UID) as f:
            entry = json.load(f)
        if entry['uid'] == "xxxx":
            id = str(uuid.uuid4())
            entry['uid'] = id
            with open(CUR_UID, 'w') as outfile:
                json.dump(entry, outfile)
                print("OUTCOME: Set the Stepsize extension installation "
                      "identifier")

setup(
    name='Stepsize',
    version='0.1',
    description='Search for Python code snippets directly from code cells',
    long_description='Search for code snippets with a library and function '
                     'name within code cells, the extension will return a '
                     'function signature with a link to the documentation as '
                     'well as a code snippet with a StackOverflow link.',
    platforms='Mac OS X, Windows',
    author='Stepsize',
    author_email='hello@stepsize.com',
    url='http://www.stepsize.com',
    licence='BSD 3-Clause',
    packages=['stepsize'],
    cmdclass={'install': InstallCommand}
)


import re
from functools import lru_cache
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Generator,
    List,
    Optional,
    Pattern,
    Tuple,
    Union,
)

from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import (
    LTChar,
    LTComponent,
    LTContainer,
    LTItem,
    LTPage,
    LTTextContainer,
)
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.psparser import PSLiteral

from . import utils
from ._typing import T_bbox, T_num, T_obj, T_obj_list
from .container import Container
from .table import T_table_settings, Table, TableFinder, TableSettings
from .utils import decode_text, resolve_all, resolve_and_decode
from .utils.text import TextMap

lt_pat = re.compile(r"^LT")

ALL_ATTRS = set(
    [
        "adv",
        "height",
        "linewidth",
        "pts",
        "size",
        "srcsize",
        "width",
        "x0",
        "x1",
        "y0",
        "y1",
        "bits",
        "matrix",
        "upright",
        "fontname",
        "text",
        "imagemask",
        "colorspace",
        "evenodd",
        "fill",
        "non_stroking_color",
        "path",
        "stream",
        "stroke",
        "stroking_color",
    ]
)

if TYPE_CHECKING:  # pragma: nocover
    from .display import PageImage
    from .pdf import PDF

# via https://git.ghostscript.com/?p=mupdf.git;a=blob;f=source/pdf/pdf-font.c;h=6322cedf2c26cfb312c0c0878d7aff97b4c7470e;hb=HEAD#l774   # noqa

CP936_FONTNAMES = {
    b"\xcb\xce\xcc\xe5": "SimSun,Regular",
    b"\xba\xda\xcc\xe5": "SimHei,Regular",
    b"\xbf\xac\xcc\xe5_GB2312": "SimKai,Regular",
    b"\xb7\xc2\xcb\xce_GB2312": "SimFang,Regular",
    b"\xc1\xa5\xca\xe9": "SimLi,Regular",
}

def fix_fontname_bytes(fontname: bytes) -> str:
    if b"+" in fontname:
        split_at = fontname.index(b"+") + 1
        prefix, suffix = fontname[:split_at], fontname[split_at:]
    else:
        prefix, suffix = b"", fontname

    suffix_new = CP936_FONTNAMES.get(suffix, str(suffix)[2:-1])
    return str(prefix)[2:-1] + suffix_new

def separate_pattern(
    color: Tuple[Any, ...]
) -> Tuple[Optional[Tuple[Union[float, int], ...]], Optional[str]]:
    if isinstance(color[-1], PSLiteral):
        return (color[:-1] or None), decode_text(color[-1].name)
    else:
        return color, None

def normalize_color(
    color: Any,
) -> Tuple[Optional[Tuple[Union[float, int], ...]], Optional[str]]:
    if color is None:
        return (None, None)
    elif isinstance(color, tuple):
        tuplefied = color
    elif isinstance(color, list):
        tuplefied = tuple(color)
    else:
        tuplefied = (color,)
    return separate_pattern(tuplefied)

class Page(Container):
    cached_properties: List[str] = Container.cached_properties + ["_layout"]
    is_original: bool = True
    pages = None

    def __init__(
        self,
        pdf: "PDF",
        page_obj: PDFPage,
        page_number: int,
        initial_doctop: T_num = 0,
    ):
        self.pdf = pdf
        self.root_page = self
        self.page_obj = page_obj
        self.page_number = page_number
        _rotation = resolve_all(self.page_obj.attrs.get("Rotate", 0)) or 0
        self.rotation = _rotation % 360
        self.page_obj.rotate = self.rotation
        self.initial_doctop = initial_doctop

        cropbox = page_obj.attrs.get("CropBox")
        mediabox = page_obj.attrs.get("MediaBox")

        self.cropbox = resolve_all(cropbox) if cropbox is not None else None
        self.mediabox = resolve_all(mediabox) or self.cropbox
        m = self.mediabox

        self.bbox: T_bbox = (
            (
                min(m[1], m[3]),
                min(m[0], m[2]),
                max(m[1], m[3]),
                max(m[0], m[2]),
            )
            if self.rotation in [90, 270]
            else (
                min(m[0], m[2]),
                min(m[1], m[3]),
                max(m[0], m[2]),
                max(m[1], m[3]),
            )
        )

        # https://rednafi.github.io/reflections/dont-wrap-instance-methods-with-functoolslru_cache-decorator-in-python.html
        self.get_textmap = lru_cache()(self._get_textmap)

    @property
    def width(self) -> T_num:
        return self.bbox[2] - self.bbox[0]

    @property
    def height(self) -> T_num:
        return self.bbox[3] - self.bbox[1]

    @property
    def layout(self) -> LTPage:
        if hasattr(self, "_layout"):
            return self._layout
        device = PDFPageAggregator(
            self.pdf.rsrcmgr,
            pageno=self.page_number,
            laparams=self.pdf.laparams,
        )
        interpreter = PDFPageInterpreter(self.pdf.rsrcmgr, device)
        interpreter.process_page(self.page_obj)
        self._layout: LTPage = device.get_result()
        return self._layout

    @property
    def annots(self) -> T_obj_list:
        def parse(annot: T_obj) -> T_obj:
            rect = annot["Rect"]

            a = annot.get("A", {})
            extras = {
                "uri": a.get("URI"),
                "title": annot.get("T"),
                "contents": annot.get("Contents"),
            }
            for k, v in extras.items():
                if v is not None:
                    try:
                        extras[k] = v.decode("utf-8")
                    except UnicodeDecodeError:
                        extras[k] = v.decode("utf-16")

            parsed = {
                "page_number": self.page_number,
                "object_type": "annot",
                "x0": rect[0],
                "y0": rect[1],
                "x1": rect[2],
                "y1": rect[3],
                "doctop": self.initial_doctop + self.height - rect[3],
                "top": self.height - rect[3],
                "bottom": self.height - rect[1],
                "width": rect[2] - rect[0],
                "height": rect[3] - rect[1],
            }
            parsed.update(extras)
            # Replace the indirect reference to the page dictionary
            # with a pointer to our actual page
            if "P" in annot:
                annot["P"] = self
            parsed["data"] = annot
            return parsed

        raw = resolve_all(self.page_obj.annots) or []
        return list(map(parse, raw))

    @property
    def hyperlinks(self) -> T_obj_list:
        return [a for a in self.annots if a["uri"] is not None]

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = self.parse_objects()
        return self._objects

    def point2coord(self, pt: Tuple[T_num, T_num]) -> Tuple[T_num, T_num]:
        return (pt[0], self.height - pt[1])

    def process_object(self, obj: LTItem) -> T_obj:
        kind = re.sub(lt_pat, "", obj.__class__.__name__).lower()

        def process_attr(item: Tuple[str, Any]) -> Optional[Tuple[str, Any]]:
            k, v = item
            if k in ALL_ATTRS:
                res = resolve_all(v)
                return (k, res)
            else:
                return None

        attr = dict(filter(None, map(process_attr, obj.__dict__.items())))

        attr["object_type"] = kind
        attr["page_number"] = self.page_number

        for cs in ["ncs", "scs"]:
            # Note: As of pdfminer.six v20221105, that library only
            # exposes ncs for LTChars, and neither attribute for
            # other objects. Keeping this code here, though,
            # for ease of addition if color spaces become
            # more available via pdfminer.six
            if hasattr(obj, cs):
                attr[cs] = resolve_and_decode(getattr(obj, cs).name)

        for color_attr, pattern_attr in [
            ("stroking_color", "stroking_pattern"),
            ("non_stroking_color", "non_stroking_pattern"),
        ]:
            if color_attr in attr:
                attr[color_attr], attr[pattern_attr] = normalize_color(attr[color_attr])

        if isinstance(obj, (LTChar, LTTextContainer)):
            attr["text"] = obj.get_text()

        if isinstance(obj, LTChar):
            # pdfminer.six (at least as of v20221105) does not
            # directly expose .stroking_color and .non_stroking_color
            # for LTChar objects (unlike, e.g., LTRect objects).
            gs = obj.graphicstate
            attr["stroking_color"], attr["stroking_pattern"] = normalize_color(
                gs.scolor
            )
            attr["non_stroking_color"], attr["non_stroking_pattern"] = normalize_color(
                gs.ncolor
            )

            # Handle (rare) byte-encoded fontnames
            if isinstance(attr["fontname"], bytes):
                attr["fontname"] = fix_fontname_bytes(attr["fontname"])

        if "pts" in attr:
            attr["pts"] = list(map(self.point2coord, attr["pts"]))

        if "y0" in attr:
            attr["top"] = self.height - attr["y1"]
            attr["bottom"] = self.height - attr["y0"]
            attr["doctop"] = self.initial_doctop + attr["top"]

        return attr

    def iter_layout_objects(
        self, layout_objects: List[LTComponent]
    ) -> Generator[T_obj, None, None]:
        for obj in layout_objects:
            # If object is, like LTFigure, a higher-level object ...
            if isinstance(obj, LTContainer):
                # and LAParams is passed, process the object itself.
                if self.pdf.laparams is not None:
                    yield self.process_object(obj)
                # Regardless, iterate through its children
                yield from self.iter_layout_objects(obj._objs)
            else:
                yield self.process_object(obj)

    def parse_objects(self) -> Dict[str, T_obj_list]:
        objects: Dict[str, T_obj_list] = {}
        for obj in self.iter_layout_objects(self.layout._objs):
            kind = obj["object_type"]
            if kind in ["anno"]:
                continue
            if objects.get(kind) is None:
                objects[kind] = []
            objects[kind].append(obj)
        return objects

    def debug_tablefinder(
        self, table_settings: Optional[T_table_settings] = None
    ) -> TableFinder:
        tset = TableSettings.resolve(table_settings)
        return TableFinder(self, tset)

    def find_tables(
        self, table_settings: Optional[T_table_settings] = None
    ) -> List[Table]:
        tset = TableSettings.resolve(table_settings)
        return TableFinder(self, tset).tables

    def find_table(
        self, table_settings: Optional[T_table_settings] = None
    ) -> Optional[Table]:
        tset = TableSettings.resolve(table_settings)
        tables = self.find_tables(tset)

        if len(tables) == 0:
            return None

        # Return the largest table, as measured by number of cells.
        def sorter(x: Table) -> Tuple[int, T_num, T_num]:
            return (-len(x.cells), x.bbox[1], x.bbox[0])

        largest = list(sorted(tables, key=sorter))[0]

        return largest

    def extract_tables(
        self, table_settings: Optional[T_table_settings] = None
    ) -> List[List[List[Optional[str]]]]:
        tset = TableSettings.resolve(table_settings)
        tables = self.find_tables(tset)
        return [table.extract(**(tset.text_settings or {})) for table in tables]

    def extract_table(
        self, table_settings: Optional[T_table_settings] = None
    ) -> Optional[List[List[Optional[str]]]]:
        tset = TableSettings.resolve(table_settings)
        table = self.find_table(tset)
        if table is None:
            return None
        else:
            return table.extract(**(tset.text_settings or {}))

    def _get_textmap(self, **kwargs: Any) -> TextMap:
        defaults = dict(x_shift=self.bbox[0], y_shift=self.bbox[1])
        if "layout_width_chars" not in kwargs:
            defaults.update({"layout_width": self.width})
        if "layout_height_chars" not in kwargs:
            defaults.update({"layout_height": self.height})
        full_kwargs: Dict[str, Any] = {**defaults, **kwargs}
        return utils.chars_to_textmap(self.chars, **full_kwargs)

    def search(
        self,
        pattern: Union[str, Pattern[str]],
        regex: bool = True,
        case: bool = True,
        main_group: int = 0,
        return_chars: bool = True,
        return_groups: bool = True,
        **kwargs: Any,
    ) -> List[Dict[str, Any]]:
        textmap = self.get_textmap(**kwargs)
        return textmap.search(
            pattern,
            regex=regex,
            case=case,
            main_group=main_group,
            return_chars=return_chars,
            return_groups=return_groups,
        )

    def extract_text(self, **kwargs: Any) -> str:
        return self.get_textmap(**kwargs).as_string

    def extract_text_simple(self, **kwargs: Any) -> str:
        return utils.extract_text_simple(self.chars, **kwargs)

    def extract_words(self, **kwargs: Any) -> T_obj_list:
        return utils.extract_words(self.chars, **kwargs)

    def extract_text_lines(
        self, strip: bool = True, return_chars: bool = True, **kwargs: Any
    ) -> T_obj_list:
        return self.get_textmap(**kwargs).extract_text_lines(
            strip=strip, return_chars=return_chars
        )

    def crop(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        return CroppedPage(self, bbox, relative=relative, strict=strict)

    def within_bbox(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        """
        Same as .crop, except only includes objects fully within the bbox
        """
        return CroppedPage(
            self, bbox, relative=relative, strict=strict, crop_fn=utils.within_bbox
        )

    def outside_bbox(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        """
        Same as .crop, except only includes objects fully within the bbox
        """
        return CroppedPage(
            self, bbox, relative=relative, strict=strict, crop_fn=utils.outside_bbox
        )

    def filter(self, test_function: Callable[[T_obj], bool]) -> "FilteredPage":
        return FilteredPage(self, test_function)

    def dedupe_chars(self, **kwargs: Any) -> "FilteredPage":
        """
        Removes duplicate chars — those sharing the same text, fontname, size,
        and positioning (within `tolerance`) as other characters on the page.
        """
        p = FilteredPage(self, lambda x: True)
        p._objects = {kind: objs for kind, objs in self.objects.items()}
        p._objects["char"] = utils.dedupe_chars(self.chars, **kwargs)
        return p

    def to_image(
        self,
        resolution: Optional[Union[int, float]] = None,
        width: Optional[Union[int, float]] = None,
        height: Optional[Union[int, float]] = None,
        antialias: bool = False,
    ) -> "PageImage":
        """
        You can pass a maximum of 1 of the following:
        - resolution: The desired number pixels per inch. Defaults to 72.
        - width: The desired image width in pixels.
        - height: The desired image width in pixels.
        """
        from .display import DEFAULT_RESOLUTION, PageImage

        num_specs = sum(x is not None for x in [resolution, width, height])
        if num_specs > 1:
            raise ValueError(
                f"Only one of these arguments can be provided: resolution, width, height. You provided {num_specs}"  # noqa: E501
            )
        elif width is not None:
            resolution = 72 * width / self.width
        elif height is not None:
            resolution = 72 * height / self.height

        return PageImage(
            self, resolution=resolution or DEFAULT_RESOLUTION, antialias=antialias
        )

    def to_dict(self, object_types: Optional[List[str]] = None) -> Dict[str, Any]:
        if object_types is None:
            _object_types = list(self.objects.keys()) + ["annot"]
        else:
            _object_types = object_types
        d = {
            "page_number": self.page_number,
            "initial_doctop": self.initial_doctop,
            "rotation": self.rotation,
            "cropbox": self.cropbox,
            "mediabox": self.mediabox,
            "bbox": self.bbox,
            "width": self.width,
            "height": self.height,
        }
        for t in _object_types:
            d[t + "s"] = getattr(self, t + "s")
        return d

    def __repr__(self) -> str:
        return f"<Page:{self.page_number}>"

class DerivedPage(Page):
    is_original: bool = False

    def __init__(self, parent_page: Page):
        self.parent_page = parent_page
        self.root_page = parent_page.root_page
        self.pdf = parent_page.pdf
        self.page_obj = parent_page.page_obj
        self.page_number = parent_page.page_number
        self.flush_cache(Container.cached_properties)
        self.get_textmap = lru_cache()(self._get_textmap)

def test_proposed_bbox(bbox: T_bbox, parent_bbox: T_bbox) -> None:
    bbox_area = utils.calculate_area(bbox)
    if bbox_area == 0:
        raise ValueError(f"Bounding box {bbox} has an area of zero.")

    overlap = utils.get_bbox_overlap(bbox, parent_bbox)
    if overlap is None:
        raise ValueError(
            f"Bounding box {bbox} is entirely outside "
            f"parent page bounding box {parent_bbox}"
        )

    overlap_area = utils.calculate_area(overlap)
    if overlap_area < bbox_area:
        raise ValueError(
            f"Bounding box {bbox} is not fully within "
            f"parent page bounding box {parent_bbox}"
        )

class CroppedPage(DerivedPage):
    def __init__(
        self,
        parent_page: Page,
        crop_bbox: T_bbox,
        crop_fn: Callable[[T_obj_list, T_bbox], T_obj_list] = utils.crop_to_bbox,
        relative: bool = False,
        strict: bool = True,
    ):
        if relative:
            o_x0, o_top, _, _ = parent_page.bbox
            x0, top, x1, bottom = crop_bbox
            crop_bbox = (x0 + o_x0, top + o_top, x1 + o_x0, bottom + o_top)

        if strict:
            test_proposed_bbox(crop_bbox, parent_page.bbox)

        def _crop_fn(objs: T_obj_list) -> T_obj_list:
            return crop_fn(objs, crop_bbox)

        super().__init__(parent_page)

        self._crop_fn = _crop_fn

        # Note: testing for original function passed, not _crop_fn
        if crop_fn is utils.outside_bbox:
            self.bbox = parent_page.bbox
        else:
            self.bbox = crop_bbox

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = {
            k: self._crop_fn(v) for k, v in self.parent_page.objects.items()
        }
        return self._objects

class FilteredPage(DerivedPage):
    def __init__(self, parent_page: Page, filter_fn: Callable[[T_obj], bool]):
        self.bbox = parent_page.bbox
        self.filter_fn = filter_fn
        super().__init__(parent_page)

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = {
            k: list(filter(self.filter_fn, v))
            for k, v in self.parent_page.objects.items()
        }
        return self._objects


import requests
from fastapi import FastAPI, HTTPException, Depends, status
from fastapi_pagination.ext.sqlalchemy_future import paginate
from fastapi.responses import JSONResponse
from sqlalchemy import select
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session

import crud
import models
import schemas
from database import engine, get_db
from log import init_logging, logger, log
from paginator import Page, Params
from schemas import Notification, DingTalkMessage
from settings import WEBHOOK_URL, ACCESS_TOKEN
from utils import render_message, write_serve

app = FastAPI()

models.Base.metadata.create_all(bind=engine)

init_logging()

@app.get("/")
@log
def root():
    return {"message": "Hello World"}

# Job
@app.post("/job/", response_model=schemas.ScrapeJob)
@log
def create_job(job: schemas.ScrapeJobCreate, db: Session = Depends(get_db)):
    try:
        return crud.create_scrape_job(db, job)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

@app.get("/job/{job_id}/", response_model=schemas.ScrapeJob)
@log
def get_job(job_id: int, db: Session = Depends(get_db)):
    db_job = crud.get_scrape_job(db, job_id)
    if db_job:
        return db_job
    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job Not Found!")

@app.get("/job/", response_model=Page[schemas.ScrapeJob])
@log
def get_jobs(params: Params = Depends(), db: Session = Depends(get_db)):
    return paginate(db, select(models.ScrapeJob).order_by(models.ScrapeJob.id), params)

@app.delete("/job/{job_id}/", response_model=schemas.ScrapeJob)
@log
def delete_job(job_id: int, db: Session = Depends(get_db)):
    try:
        return crud.delete_scrape_job(db, job_id)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

# Module
@app.post("/module/", response_model=schemas.ExporterModule)
@log
def create_module(module: schemas.ExporterModuleCreate, db: Session = Depends(get_db)):
    try:
        return crud.create_blackbox_module(db, module)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

@app.get("/module/{module_id}/", response_model=schemas.ExporterModule)
@log
def get_module(module_id: int, db: Session = Depends(get_db)):
    db_module = crud.get_blackbox_module(db, module_id)
    if db_module:
        return db_module
    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Module Not Found!")

@app.get("/module/", response_model=Page[schemas.ExporterModule])
@log
def get_modules(params: Params = Depends(), db: Session = Depends(get_db)):
    return paginate(db, select(models.ExporterModule).order_by(models.ExporterModule.id), params)

@app.delete("/module/{module_id}/", response_model=schemas.ExporterModule)
@log
def delete_module(module_id: int, db: Session = Depends(get_db)):
    try:
        return crud.delete_blackbox_module(db, module_id)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

# Rule
@app.post("/rule/", response_model=schemas.AlertRule)
@log
def create_rule(rule: schemas.AlertRuleCreate, db: Session = Depends(get_db)):
    try:
        return crud.create_alert_rule(db, rule)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

@app.get("/rule/{rule_id}/", response_model=schemas.AlertRule)
@log
def get_rule(rule_id: int, db: Session = Depends(get_db)):
    db_rule = crud.get_alert_rule(db, rule_id)
    if db_rule:
        return db_rule
    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule Not Found!")

@app.get("/rule/", response_model=Page[schemas.AlertRule])
@log
def get_rules(params: Params = Depends(), db: Session = Depends(get_db)):
    return paginate(db, select(models.AlertRule).order_by(models.AlertRule.id), params)

@app.delete("/rule/{rule_id}/", response_model=schemas.AlertRule)
@log
def delete_rule(rule_id: int, db: Session = Depends(get_db)):
    try:
        return crud.delete_alert_rule(db, rule_id)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

# RuleGroup
@app.post("/rule-group/", response_model=schemas.AlterGroup)
@log
def create_rule_group(rule_group: schemas.AlterGroupCreate, db: Session = Depends(get_db)):
    try:
        return crud.create_alert_group(db, rule_group)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

@app.get("/rule-group/{rule_group_id}/", response_model=schemas.AlterGroup)
@log
def get_rule_group(rule_group_id: int, db: Session = Depends(get_db)):
    db_rule_group = crud.get_alert_group(db, rule_group_id)
    if db_rule_group:
        return db_rule_group
    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Rule Group Not Found!")

@app.get("/rule-group/", response_model=Page[schemas.AlterGroup])
@log
def get_rule_groups(params: Params = Depends(), db: Session = Depends(get_db)):
    return paginate(db, select(models.AlterGroup).order_by(models.AlterGroup.id), params)

@app.delete("/rule-group/{rule_group_id}/", response_model=schemas.AlterGroup)
@log
def delete_rule_group(rule_group_id: int, db: Session = Depends(get_db)):
    try:
        return crud.delete_alert_group(db, rule_group_id)
    except SQLAlchemyError as e:
        logger.exception(e)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))

# http_sd_config
@app.get("/serve/")
@log
def http_sd_config(db: Session = Depends(get_db)):
    ret = list()
    for i in db.scalars(select(models.ScrapeJob).order_by(models.ScrapeJob.id)):
        ret.append(schemas.HttpSdConfig(targets=[j for j in i.target], labels={"module": i.module.name}))
    return ret

@app.get("/start/")
@log
def gen_serve(db: Session = Depends(get_db)):
    if write_serve(db):
        logger.info("服务生成成功")
        return JSONResponse(status_code=status.HTTP_201_CREATED, content={"message": "服务生成成功"})
    logger.error("服务生成失败")
    return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": "服务生成失败"})

# WebHook
@app.post("/webhook")
@log
def webhook(notification: Notification):
    logger.info("告警原始信息：")
    logger.info(notification.dict())
    markdown = render_message(notification)
    message = DingTalkMessage(markdown=markdown)
    logger.info("钉钉告警信息：")
    logger.info(message.dict(by_alias=True))
    payload = {
        "access_token": ACCESS_TOKEN
    }
    response = requests.post(url=WEBHOOK_URL, json=message.dict(by_alias=True), params=payload)
    logger.info("响应信息：")
    logger.info(response.json())
    if response.status_code != 200:
        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Webhook Error!")
    logger.info("-" * 40)
    return message.dict()

@app.get("/ping")
@log
def healthy():
    return {"message": "pong"}


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 29 12:21:49 2018

@author: Kazım
"""
import cv2
import numpy as np
import pandas as pd
camera_height = 75 #cm
camera_theta =26.15

camera_H_angle = 99.64 #degree
camera_W_angle = 100 #degree
compression_factor_H = 1
compression_factor_W = 1
img = cv2.imread('D:\\undistorted.jpg',-1)
#img = cv2.resize(img, (960, 540))

H,W,_ = np.shape(img)
L_ref = 2*camera_height*np.tan(np.radians(camera_H_angle)/5)
h = H/np.radians(camera_H_angle)
w = W/np.radians(camera_W_angle)
x_mid = camera_height/np.tan(np.radians(camera_theta))
#phi_1 = np.arctan(camera_height/(x_mid-L_ref/2)) - np.radians(camera_theta)
#phi_2 = np.radians(camera_theta) - np.arctan(camera_height/(x_mid+L_ref/2))

phi_1 = np.arccos((camera_height*camera_height + x_mid*(x_mid-L_ref/2))/((np.sqrt(np.square(camera_height)+np.square(x_mid)))*(np.sqrt(np.square(camera_height)+np.square(x_mid-L_ref/2)))))
phi_2 = np.arccos((camera_height*camera_height + x_mid*(x_mid+L_ref/2))/((np.sqrt(np.square(camera_height)+np.square(x_mid)))*(np.sqrt(np.square(camera_height)+np.square(x_mid+L_ref/2)))))

beta_1 = np.arctan((L_ref/2)/(np.sqrt(np.square(x_mid-L_ref/2)+np.square(camera_height))))
beta_2 = np.arctan((L_ref/2)/(np.sqrt(np.square(x_mid+L_ref/2)+np.square(camera_height))))

#beta_1 = np.arctan((L_ref/2)/(x_mid-L_ref/2))
#beta_2 = np.arctan((L_ref/2)/(x_mid+L_ref/2))
I_t = H/2 - compression_factor_H*phi_1*h
I_b = H/2 + compression_factor_H*phi_2*h
J_tr = W/2 + compression_factor_W*beta_2*w
J_tl = W/2 - compression_factor_W*beta_2*w
J_br = W/2 + compression_factor_W*beta_1*w
J_bl = W/2 - compression_factor_W*beta_1*w

p_1_prime = (np.round(J_tr).astype(int),np.round(I_t).astype(int))
p_2_prime = (np.round(J_br).astype(int),np.round(I_b).astype(int))
p_3_prime = (np.round(J_bl).astype(int),np.round(I_b).astype(int))
p_4_prime = (np.round(J_tl).astype(int),np.round(I_t).astype(int))

cv2.line(img, p_1_prime, p_2_prime, (0,255,0), thickness=2, lineType=8, shift=0)
cv2.line(img, p_2_prime, p_3_prime, (0,255,0), thickness=2, lineType=8, shift=0)
cv2.line(img, p_3_prime, p_4_prime, (0,255,0), thickness=2, lineType=8, shift=0)
cv2.line(img, p_4_prime, p_1_prime, (0,255,0), thickness=2, lineType=8, shift=0)

#cv2.circle(img, p_3_prime,5, (0,255,0), thickness=15)
#cv2.circle(img, p_4_prime,5, (0,255,0), thickness=15)
#cv2.circle(img, p_2_prime,5, (0,255,0), thickness=1, lineType=8, shift=0)
#cv2.circle(img, p_3_prime,5, (0,255,0), thickness=1, lineType=8, shift=0)
#cv2.circle(img, p_4_prime,5, (0,255,0), thickness=1, lineType=8, shift=0)

pts1 = np.float32([[J_tr,I_t],[J_br,I_b],[J_bl,I_b],[J_tl,I_t]])
pts2 = np.float32([[540,460],[540,540],[460,540],[460,460]])

matrix = cv2.getPerspectiveTransform(pts1,pts2)
result = cv2.warpPerspective(img, matrix, (1000,1000))
#cv2.circle(result, (495,780),5, (0,255,0), thickness=15)

M = cv2.getRotationMatrix2D((1000/2,1000/2),-45,1)
dst = cv2.warpAffine(result,M,(1000,1000))
thres = 50

dst = cv2.cvtColor(dst, cv2.COLOR_BGR2HSV)
sat = dst[:,:,1]
val = dst[:,:,2]
ret,mask = cv2.threshold(val,10,127,cv2.THRESH_BINARY_INV)
ret,bin_inv_dst = cv2.threshold(sat,thres,255,cv2.THRESH_BINARY_INV)
bin_inv_dst = bin_inv_dst - mask

img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
sat = img[:,:,1]
val = img[:,:,2]
ret,mask = cv2.threshold(val,10,127,cv2.THRESH_BINARY_INV)
ret,bin_inv_img = cv2.threshold(sat,thres,255,cv2.THRESH_BINARY_INV)
bin_inv_img = bin_inv_img - mask

result = cv2.cvtColor(result, cv2.COLOR_BGR2HSV)
sat = result[:,:,1]
val = result[:,:,2]
ret,mask = cv2.threshold(val,10,127,cv2.THRESH_BINARY_INV)
ret,bin_inv_result = cv2.threshold(sat,thres,255,cv2.THRESH_BINARY_INV)
bin_inv_result = bin_inv_result - mask

#cv2.imshow('mask',mask)
cv2.imshow('image',bin_inv_result)
cv2.imshow('original',bin_inv_img)
cv2.imshow('original',result)
#cv2.imshow('original',img)
#cv2.imshow('rotated',bin_inv_dst)

#cv2.imwrite('D:\\corner_birdview_rotated_saturation_bin_inv.jpg',bin_inv_dst)

#cv2.imwrite('D:\\corner_withline_saturation_bin_inv.jpg',bin_inv_img)

#cv2.imwrite('D:\\corner_birdview_saturation_bin_inv.jpg',bin_inv_result)

cv2.waitKey(0)
cv2.destroyAllWindows()

#a = (np.sqrt(np.square(camera_height)+np.square(x_mid)))*(np.sqrt(np.square(camera_height)+np.square(x_mid-L_ref/2)))
#b = (np.sqrt(np.square(camera_height)+np.square(x_mid)))*(np.sqrt(np.square(camera_height)+np.square(x_mid+L_ref/2)))

driveME/Written but not used codes/Solidworks_Ortam_simulasyonu/Perspective transform_sunum.py


from django.shortcuts import render,redirect
from .models import *
from django.contrib import messages
from django.contrib.auth.models import User
from django.core.mail import send_mail
# Create your views here.
def inquiry(request):
    if request.method == 'POST':
        car_id=request.POST['car_id']
        car_title = request.POST['car_title']
        user_id = request.POST['user_id']
        first_name = request.POST['first_name']
        last_name=request.POST['last_name']
        customer_need = request.POST['customer_need']
        city = request.POST['city']
        state = request.POST['state']
        email = request.POST['email']
        phone = request.POST['phone']
        message=request.POST['message']

        if request.user.is_authenticated:
            userid=request.user.id
            has_contacted=Contact.objects.all().filter(car_id=car_id,user_id=userid)
            if has_contacted:
                messages.error(request,"You had already registered. We will contact you soon")
                return redirect('car')

        customer=Contact(car_id=car_id,car_title=car_title,user_id=user_id,first_name=first_name,last_name=last_name,customer_need=customer_need,city=city,state=state,email=email,phone=phone,message=message)
        admin_detail=User.objects.get(is_superuser=True)
        admin_email=admin_detail.email
        send_mail(
            'New Car inquiry',
            'You have a new inquiry for the car with ID: ' + car_id+ ' and car Name: '+ car_title+ ' Please login to admin pannel for more detail' ,
            'myweddingcorp23@gmail.com', #from email address
            [admin_email], #To email address
            fail_silently=False,
                 )

        customer.save()
        messages.success(request,"YOU HAVE SUBMITTED THE FORM AND WE WILL CONTACT YOU SOON")
        return redirect('car')
        # return redirect('/car/',+ car_id)


class Solution:
    def productExceptSelf(self, nums: List[int]) -> List[int]:
        n = len(nums)
        prod = [1] * n
        for i in range(1, n):
            prod[i] = prod[i - 1] * nums[i - 1]
        acc = 1
        for i in range(n - 1, -1, -1):
            prod[i] = prod[i] * acc
            acc *= nums[i]
        return prod


'''
WAP to read content of a text file named
    student.txt and print the number of C.S.
        student
'''
f = open("students.txt","r")
data = f.readlines()
c = 0
for i in data:
    i = i.split()
    if i[1] == 'C.S.' or i[1] == 'c.s.':
        c += 1
print("Number of Students Having C.S. are : ",c)
f.close()

323.) to count num of stu having cs through file handling.py


"""Test :py:mod:phyre_engine.component.system."""
import textwrap
import unittest

import phyre_engine.component.system as system

class TestPython(unittest.TestCase):
    """Test phyre_engine.component.syste.Python component."""

    SCRIPT = textwrap.dedent("""\
    data["result"] = data["input"] * 2
    """)

    def test_execution(self):
        """Execute arbitrary code."""
        py = system.Python(self.SCRIPT)
        result = py.run({"input": 1})
        self.assertEqual(result["result"], 2)

    def test_adds(self):
        """Test ADDS fields."""
        py = system.Python(self.SCRIPT, adds="['a', 'b', 'c']")
        self.assertEqual(py.ADDS, ["a", "b", "c"])

    def test_removes(self):
        """Test REMOVES fields."""
        py = system.Python(self.SCRIPT, removes="['a', 'b', 'c']")
        self.assertEqual(py.REMOVES, ["a", "b", "c"])

    def test_required(self):
        """Test REQUIRED fields."""
        py = system.Python(self.SCRIPT, required="['a', 'b', 'c']")
        self.assertEqual(py.REQUIRED, ["a", "b", "c"])


# Created with Pyto
import numpy as np
import random
import matplotlib.pyplot as plt
from sklearn.datasets import make_regression
from dataclasses import dataclass
import math

def initial_2D_condition_random_by_array(_domain,_source_dim):
    irow_center =random.randint(0, len(_domain)-1)
    icol_center =random.randint(0, len(_domain[0])-1)
    for i in range (irow_center-_source_dim-1,irow_center+_source_dim):
        if(i<0 or i>=len(_domain)): continue
        for j in range (icol_center-_source_dim-1,icol_center+_source_dim):
            if(j<0 or j>=len(_domain[i])): continue
            _domain[i][j]=100

def initial_2D_condition_random_by_size(size,_source_dim):
    domain = np.full((size,size),0.00)
    initial_2D_condition_random_by_array(domain,_source_dim)
    return domain

def diffusion_equation(_domain,_k,_t,_dx):
    domain_next = np.full((len(_domain), len(_domain[0])), 0.0)
    alpha = _k * _t*_dx
    for i in range(0, len(_domain)):
        for j in range(0, len(_domain[0])):
            if (i>0): domain_next[i][j]+=_domain[i-1][j]*alpha -_domain[i][j]*alpha
            if (i<len(_domain)-1): domain_next[i][j]+=_domain[i+1][j]*alpha-_domain[i][j]*alpha
            if (j>0): domain_next[i][j]+=_domain[i][j-1]*alpha-_domain[i][j]*alpha
            if (j<len(_domain[0])-1): domain_next[i][j]+=_domain[i][j+1]*alpha-_domain[i][j]*alpha
            domain_next[i][j] = _domain[i][j]+domain_next[i][j]
#    for i in range(0, len(_domain)-1):
#        for j in range(0, len(_domain[0])-1):
#            _domain[i][j]=domain_next[i][j]
    return domain_next

def get_nodeid(i,j,length):
    return i*length+length

def iterator(_duration,_timestepsize,_domain):
    for i in range (0, int(_duration/_timestepsize)):
        _domain = diffusion_equation(_domain, 0.01, _timestepsize, 1)
    return _domain

def result_writer(_pfile,_domain):
    for i in range(0, len(_domain)):
        swrite=''
        for j in range(0, len(_domain[0])):
            swrite+='%.1f' %_domain[i][j]
            swrite+=' '
        swrite+='\n'
        _pfile.write(swrite);
    return 0

def result_reader(_pfile,_oresult_list):
    bIsInput = True
    size=0
    irowcount = 0
    iNresult_pair = 0
    while(True):
        sread=_pfile.readline()
        if(sread=="Input \n"):
            bIsInput= True
            oresult_pair=result_pair(size)
            _oresult_list.data.append(oresult_pair)
            irowcount=0
            #print("input")
        elif(sread=="Output \n"):
            bIsInput= False
            irowcount=0
            #print("output")
        elif(sread=="domain data size\n"):
            sread=_pfile.readline()
            size=int(sread)
            #print("size %d\n" %size)
        elif(not sread): break #end of file
        else:
            if (bIsInput==True):
                sread_list=sread.split(" ")[0:-1]
                for i in range (0,size-1):
                    _oresult_list.data[-1].input[irowcount][i]=float(sread_list[i])
                irowcount+=1
            else:
                sread_list=sread.split(" ")[0:-1]
                for i in range (0,size-1):
                    _oresult_list.data[-1].output[irowcount][i]=float(sread_list[i])
                irowcount+=1
    return _oresult_list

def Two_D_Array_Contour_Show(domain):
    plt.contourf(domain)
    plt.colorbar()
    plt.show()
    plt.clf()

def reconstruct_2D_array(array_1D):
    size= round(math.sqrt(len(array_1D)))
    domain = np.full((size, size), 0.0)
    for i in range (0,size):
        for j in range (0,size):
            domain[i][j]=float(array_1D[i*size+j])
    return domain

class result_pair():
    def __init__(self,size):
        self.input=np.full((size,size),0)
        self.output=np.full((size,size),0)

class result_list():
    def __init__(self):
        self.data=[]


import robot
import polygon
import obstacle

def block(reference_point, color):
    other_vertex_list = [(0,75), (75,75), (75,10)]
    quadrilateral = polygon.Polygon(reference_point, other_vertex_list)
    quadrilateral.color = color
    block = obstacle.Obstacle([quadrilateral])
    return block

def complex_block(reference_point, color):
    other_vertex_list = [(0,75), (75,75), (75,10)]
    #other_vertex_list = [(0,90), (90,100), (90,80), (85, 80)]
    six_sided_figure = polygon.Polygon(reference_point, other_vertex_list)
    six_sided_figure.color = color
    block = obstacle.Obstacle([six_sided_figure])
    return block


from datetime import datetime
import csv
import matplotlib.pyplot as plt
import matplotlib.dates as md

class Quote:
    def __init__(self, timestamp=None, open=None, close=None, dividend=None, splitCoefficient=None):
        self.timestamp = datetime.strptime(timestamp, "%Y-%m-%d")
        self.open = float(open)
        self.close = float(close)
        self.dividend = float(dividend)
        self.splitCoefficient = float(splitCoefficient)
        self.realOpen = self.open
        self.realClose = self.close

    def __str__(self):
        return "%s\t%s\t%s" % (self.timestamp.strftime("%Y-%m-%d"), self.open, self.close)

    def printRealQuote(self):
        return "%s\t%s\t%s" % (self.timestamp.strftime("%Y-%m-%d"), self.realOpen, self.realClose)

    @staticmethod
    def parseCSV(filePath):
        with open(filePath, 'r') as csvfile:
            #print("Opening file: " + filePath)
            reader = csv.DictReader(csvfile)
            quotes = []
            for row in reader:
                #print row['timestamp']+" "+row['close']
                quotes.append(Quote(row['timestamp'], row['open'], row['close'], row['dividend_amount'], row['split_coefficient']))
        return quotes

    @staticmethod
    def sort(quotes, reverse=False):
        quotes.sort(key=lambda x: x.timestamp, reverse=reverse) #Sorting

    @staticmethod
    def adjustSplits(quotes):
        # Rebasing stock prices with splits
        # TODO: Should handle reverse split
        indx = 0
        indx2 = 0
        while indx < len(quotes):
            if quotes[indx].splitCoefficient != 1.0:
                indx2 = indx
                while indx2 < len(quotes):
                    quotes[indx2].open *= quotes[indx].splitCoefficient
                    quotes[indx2].close *= quotes[indx].splitCoefficient
                    indx2 += 1
            indx += 1

    @staticmethod
    def fullCSVParse(filePath):
        with open(filePath, 'r') as csvfile:
            #print("Opening file: " + filePath)
            reader = csv.DictReader(csvfile)
            quotes = []
            for row in reader:
                #print row['timestamp']+" "+row['close']
                quotes.append(Quote(row['timestamp'], row['open'], row['close'], row['dividend_amount'], row['split_coefficient']))

        Quote.sort(quotes, False)
        Quote.adjustSplits(quotes)
        return quotes

    @staticmethod
    def findDividendDates(quotes):
        dividendsDates = []
        for q in quotes:
            if float(q.dividend) > 0.0:
                dividendsDates.append(q.timestamp)
        return dividendsDates

    @staticmethod
    def plotQuotes(config, quotes, show=True, figureNumber=1):
        # print("Config: " + str(config))
        times = list(map(lambda q: q.timestamp, quotes))
        opens = list(map(lambda q: q.open, quotes))
        closes = list(map(lambda q: q.close, quotes))

        plt.figure(figureNumber)

        if config['plotOpen']:
            plt.plot(times, opens, 'g-', label='Open')
            plt.legend(loc='best')

        if config['plotClose']:
            plt.plot(times, closes, 'b-', label='Close')
            plt.legend(loc='best')

        if config['dividendMarker']:
            dividendsDates = Quote.findDividendDates(quotes) #Search dates with dividends
            for dD in dividendsDates:
                plt.axvline(x=dD, c='r', label='Dividend date')
                plt.legend(loc='best')

        if 'verticalMarkers' in config:
            for vm in config['verticalMarkers']:
                plt.axvline(x=vm['date'], c=vm['color'], label=vm['label'])

            #print("Dividend dates: " + str(dividendsDates))

        if show:
            plt.show()

    @staticmethod
    def plotMultipleQuotes(config, *quotesSet):
        i = 1
        for q in quotesSet:
            Quote.plotQuotes(config, q, False, i) #False = doesn't call show
            i = i + 1
        plt.show()

    # config = {
    #     "daysBefore": 2,
    #     "daysAfter": 2,
    #     "dividendsNumber": 4
    # }
    @staticmethod
    def plotDividendDates(config, quotes):
        dividendsDates = Quote.findDividendDates(quotes) #Search dates with dividends
        latestDividendsDates = dividendsDates[-config['dividendsNumber']:]
        print(latestDividendsDates)

        subPlotNumber = 421
        for d in latestDividendsDates: # for each of the latest quote dates
            index = [ q.timestamp for q in quotes ].index(d) # finding the index
            subQuotes = quotes[ index-config['daysBefore'] : index+config['daysAfter']+1 ] # slicing the array

            print("Date of interest: " + str(d))
            #print([q.timestamp for q in subQuotes])

            # ---------- PLOT ----------
            plt.figure(1)

            ax = plt.subplot(subPlotNumber)
            xfmt = md.DateFormatter('%Y-%m-%d')
            ax.xaxis.set_major_formatter(xfmt)

            dates =[q.timestamp for q in subQuotes]
            plt.plot( dates , [q.open for q in subQuotes], 'g-', label='Open' )
            plt.plot( dates , [q.close for q in subQuotes], 'b-', label='Close' )
            plt.axvline(x=quotes[index].timestamp, c='r', label='Dividend date')
            plt.legend(loc='best')
            subPlotNumber = subPlotNumber + 1
            plt.xticks( dates, rotation=35 )

        plt.show()