import os, re
import pandas as pd
from collections import OrderedDict
import numpy as np
from typing import Any
from datetime import datetime as dt
from datetime import timedelta
import plotly.graph_objs as go

# ---------------------- AUX METHODS FOR PARSER -------------------------- #
def parse_test_results(fname):
    '''
    '''
    min_pattern = ['response bad', 'polarity bad', 'rub+buzz bad', 'thd bad', 'created_at 17.10.2018 13.07.41', 'unit n. f072-00737 bad']
    main_features = ['response', 'polarity', 'rub+buzz', 'thd']
    timestamp = re.compile(r'(\d+\.){2}\d+')

    with open('Data/{}'.format(fname), 'r') as f:
        from_file = f.read()

    tmp = [x.strip().lower() for x in from_file.split('\n') if x][1:]
    tmp = ['created_at {}'.format('.'.join(x.split())) if timestamp.match(x) else x for x in tmp]

    try:
        if len(tmp) < 6:
            missing_features = list(set([x.split(' ',1)[0] for x in min_pattern]) - set([x.split(' ',1)[0] for x in tmp]))
            print('MISSING:', tmp)

            # if unit is missing and all other 4 features are present
            if (('unit' in missing_features) and (not[x for x in missing_features if x in main_features])):
                if any('bad' in x for x in tmp):
                    overall = 'bad'
                else:
                    overall = 'good'
                tmp.append('unit n. {} {}'.format(fname.split('.',1)[0], overall))

            # else if any of the 4 features is missing and overall is 'good'
            elif 'good' == ''.join([re.search('good|bad', x).group(0) for x in tmp if 'unit' in x]):
                tmp = ['{} good'.format(x.split(' ',1)[0]) if x.split(' ',1)[0] in main_features else x for x in main_features+tmp]
                tmp = list(OrderedDict.fromkeys(tmp))

                print('FIXED: ',tmp)

            # else if overall is 'bad' and main_features values are missing, infer 'bad' for them
            elif 'bad' == ''.join([re.search('good|bad', x).group(0) for x in tmp if 'unit' in x]):
                tmp = ['{} bad'.format(x.split(' ',1)[0])
                        if x.split(' ',1)[0] in (main_features and missing_features)
                        else x for x in main_features+tmp]

                tmp = list(OrderedDict.fromkeys(tmp))
                tmp = [x for x in tmp if re.search('good|bad', x)!=None]
                print('FIXED:', tmp)

            elif len(tmp)==0:
                tmp = ['{} {}'.format(feature, np.nan) if ('unit' not in feature) else '{} n. {} {}'.format(feature, fname, np.nan) for feature in missing_features]
                print('FIXED:', tmp)

            else:
                raise ValueError('Input file {} has missing values that can\'t be handled yet'.format(fname))

    except Exception as e:
        print('ERROR:', e)
        print(tmp, len(tmp))

    return tmp

def extend_test_results(list_of_strings):
    '''
    '''
    tmpd = {}
    # defines valid columns
    columns = ['response', 'polarity', 'rub+buzz', 'thd', 'created_at', 'id', 'overall']

    # compiles regular expression patters
    result = re.compile('good|bad')
    model_no = re.compile(r'\w\d{3}-\d{5}')

    for entry in list_of_strings:
        if entry:
            key, value = entry.split(' ',1)

            if key in columns:
                tmpd[key] = ''.join(value.split())

            # parses the date/timestamp entry (subs any non-digit character with '.')
            elif re.match('created_at', key):
                tmpd['created_at'] = re.sub(r'\D', '.', entry)

            # parse the device id and overall result
            elif 'unit' in key:
                tmpd['overall'] = result.search(''.join(value.split())).group(0) if result.search(value)!=None else 'nan'
                tmpd['id'] = model_no.search(''.join(value.split())).group(0) if model_no.search(value)!=None else 'nan'

    return tmpd

def clean_timestamp_series(series):
    '''
    '''
    return series.apply(lambda x: np.nan if ('nan' in str(x)) else re.search(r'(\d+\.){5}\d+', str(x)).group(0))

def pad_timestamp_series(series):
    '''
    '''
    return series.apply(lambda x: np.nan if ('nan' in str(x)) else '.'.join([y.zfill(2) for y in str(x).split('.')]))

def parse_timestamp_series(series):
    '''
    Finds entries in the given Series that do not match the standard format and attempts to fix them
    This method requires that the dataframe where the series comes from be sorted by ['id'].
    '''
    tmp = []

    def find_nearest_neighbor(index, series):
        # finds the nearest neighbor with a valid date
        neighbor_index = np.nan
        i = index + 1
        j = index - 1

        while(True):
            if (i < len(series)-1) and (i + (1 if pd.isnull(series[i]) else 0) == i):
                return i
            elif i < len(series)-1:
                i += 1

            elif (j - (1 if pd.isnull(series[j]) else 0) == j):
                return j
            elif j > 1:
                j -= 1

        return np.nan

    for index, value in series.iteritems():
        NaN_FLAG = pd.isnull(value)

        if not NaN_FLAG:
            try:
                tmp.append(pd.to_datetime(value, format='%d.%m.%Y.%H.%M.%S', errors='raise'))

            except ValueError as e:
                print('TIMESTAMP_PARSE_WARNING:', e)
                print('... attempting to fix the format:')

                date_pattern = re.compile(r'(\d+\.){5}\d+')
                date_to_fix = date_pattern.search(str(e)).group(0)

                # grabs the alleged day and month
                first, second, tail = date_to_fix.split('.',2)

                if ((int(first) <= 12) and (int(second) > 12)):
                    try:
                        tmp.append(pd.to_datetime('{}.{}.{}'.format(second, first, tail), format='%d.%m.%Y.%H.%M.%S'))

                    except Exception as ex:
                        print('\tFAILED to handle timestamp format exception: ', ex)

                    else:
                        print('\tSUCCEEDED in handling exception for timestamp format.')
                        pass

                elif ((int(first) > 12) and (int(second) > 12)):

                    try:
                        # find the nearest neighbor with a valid date
                        neighbor_index = find_nearest_neighbor(index, series)

                        if pd.isnull(neighbor_index):
                            raise IndexError('Could not find neighbor with valid timestamp')

                    except IndexError as ex:
                        print(e, series[index])
                        pass

                    else:
                        new_date = '{}.{}'.format(str(series[neighbor_index]).rsplit('.',3)[0], value.split('.', 3)[-1])

                    try: # to approxiamte the date from a neighbor, the original dataframe must be sorted by ['id'], though.
                        tmp.append(pd.to_datetime(new_date, format='%d.%m.%Y.%H.%M.%S'))

                    except Exception as ex:
                        print('\tFAILED to handle timestamp format exception: ', ex)

                    else:
                        print('\tSUCCEEDED in handling exception for timestamp format.')
                        pass

        else:
            print('TIMESTAMP_PARSE_WARNING: time data \'{}\' does not match format \'%d.%m.%Y.%H.%M.%S\' (index: {})'.format(value, index))
            print('... attempting to fix the format:')

            # find the next neighbor with a valid timestamp to approximate
            try:
                neighbor_index = find_nearest_neighbor(index, series)

                if pd.isnull(neighbor_index):
                    raise IndexError('Could not find neighbor with valid timestamp')

            except IndexError as ex:
                print(e, series[index])
                pass

            else:
                new_date = str(series[neighbor_index])

            try: # to approxiamte the date from a neighbor, the original dataframe must be sorted by ['id'], though.
                tmp.append(pd.to_datetime(new_date, format='%d.%m.%Y.%H.%M.%S'))

            except Exception as ex:
                print('\tFAILED to handle timestamp format exception: ', ex)

            else:
                print('\tSUCCEEDED in handling exception for timestamp format.')
                pass

    return tmp

def parse_benchmark_state(df):
    '''
    Parses the number of failed tests for each Orion device in the dataframe
    Returns a list of states that can be used to create a new column in the dataframe
    '''
    list_of_states = []
    for index, value in df.iterrows():
        try:
            if 'good' in str(value.overall):
                list_of_states.append('passed')

            elif 'bad' in str(value.overall):
                list_of_states.append('failed_{}'.format((value.value_counts().bad - 1)))

            elif 'nan' in str(value.overall):
                list_of_states.append('nan')

        except Exception as e:
            print('BENCHMARK_STATE_PARSER_ERROR:', value)
            print(pd.isnull(value).value_counts())

    return list_of_states

def load_testresults_todataframe(path, is_csv=False):
    '''
    FUTURE feature:
        handle single files (txt, csv and json)
    '''
    # parses the text files into a list of dictionaries that will be used to create the dataframe
    list_of_dicts = []
    for txt_file in os.listdir(path):
        from_file = parse_test_results(txt_file)

        list_of_dicts.append(extend_test_results(from_file))

    # creates the dataframe
    df = pd.DataFrame(list_of_dicts)

    # sort by ['id'] to enable extra error handling in parse_timestamp_series()
    df.sort_values(by=['id'], inplace=True, ascending=False, kind='mergesort')
    df.reset_index(inplace=True, drop=True)

    # standardizes the timestamp format in 'created_at' and transforms to datetime
    df.created_at = clean_timestamp_series(df.created_at)
    df.created_at = pad_timestamp_series(df.created_at)
    df.created_at = parse_timestamp_series(df.created_at)

    # finds and drop rows that have no valid field
    df.replace(to_replace='nan', value=np.nan, inplace=True)
    df.dropna(how='all', inplace=True)

    # extends the dataframe to have columns states (the number of failed tests if any, or passed if none)
    df['state'] = parse_benchmark_state(df)

    return df

# ---------------------- AUX METHODS FOR DASHBOARD -------------------------- #

# testing out typing (https://docs.python.org/3/library/typing.html)
def update_windroseplot(in_df: pd.DataFrame, start: dt.date, end: dt.date, in_focus: Any) -> dict:
    '''
    '''
    colors = dict(passed='#009f00', failed_1='#ebaca2', failed_2='#e0907a', failed_3='#ce6a6b', failed_all='#ba4c49', nan='#4a919e')

    end = end + timedelta(days=1)

    if not in_focus:
        df = in_df.copy()
        df = df.loc[start : end]

    else:
        df = in_df[(in_df.id.isin(in_focus))].copy()
        df = df.loc[start : end]

    return {
        'data': [
            go.Scatterpolar(
                r = [x*2 for x in range(5, len(df.loc[df.state==state])+5)],
                theta = [360*x for x in np.random.rand(len(df.loc[df.state==state]))],
                text=['Id: {}'.format(str(x)) for x in df.loc[df.state==state, 'id'].values],
                hoverinfo = 'text',
                name=state,
                mode = 'markers',
                marker = dict(
                    color = colors[state],
                    size = 20,
                    opacity=0.75,
                    line = dict(
                        color = "black"
                    ),
                ),
            ) for state in df.state.unique()
        ],
        'layout': go.Layout(
            title='Orion Devices',
            font=dict(size=18),
            hoverlabel=dict(namelength=20),
            showlegend=True,
            legend=dict(font=dict(size=16)),
            autosize=True,
            #width=1700,
            #height=800,
            polar = dict(
                domain = dict(
                    x = [0, 1],
                    y = [0, 1]
                ),
                bgcolor = 'rgba(74, 145, 158, 0)',
                angularaxis = dict(
                    tickwidth = 0,
                    linewidth = 3,
                    layer = "below traces",
                    #tickvals=,
                    #ticktext=,
                    #nticks= len(np.unique(df[start : end].index.date)),
                    showticklabels=False,
                    showgrid=False,
                    showline=False,
                    direction = 'clockwise',
                    rotation=180,

                ),
                radialaxis = dict(
                    autorange=True,
                    showline = False,
                    linewidth = 0,
                    tickwidth = 0,
                    gridwidth = 2,
                    showgrid=True,
                    showticklabels=False,
                ),
                    sector = [0, 360],
            ),
            paper_bgcolor = 'rgba(0,0,0,0)',
        )
    }

def update_barplot(in_df: pd.DataFrame, start: dt.date, end: dt.date, in_focus: Any) -> dict:
    '''
    '''
    colors = dict(passed='#bed3c3', failed_1='#ebaca2', failed_2='#e0907a', failed_3='#ce6a6b', failed_all='#ba4c49', nan='#4a919e')

    end = end + timedelta(days=1)

    if not in_focus:
        df = in_df.copy()
        df = df.loc[start : end]

    else:
        df = in_df[(in_df.state.isin(in_focus))].copy()
        df = df.loc[start : end]

    # for each state in 'in_focus', we plot 'td' days around the pole
    overall_s = df.state.unique()
    overall_series = {}

    for feat in overall_s:
        overall_series[feat] = df[(df.state==feat)].state.resample('D').count().replace(0, np.nan).dropna()

    # for each day, calculate the failure rate
    if not in_focus or (any(['failed' in x for x in in_focus]) and any(['passed' in x for x in in_focus])):
        fail_rate = {}
        for date in np.unique(df.index.date):
            d = date.strftime('%Y-%m-%d')
            fail_n = sum([y for (x,y) in zip(df[d].state.value_counts().index, df[d].state.value_counts().values) if 'fail' in x])

            if df[d].state.value_counts()['passed'] > 0:
                fail_rate[date.strftime('%b %d')] = (fail_n  / (df[d].state.value_counts()['passed'] + fail_n))*100

            else:
                fail_rate[date] = 1

        # calculate the standard deviation for the series
        std = np.std(list(fail_rate.values()))

    # define data properties for our chart
    my_data = [
        go.Bar(
                #x = overall_series[state].index,
                x = [x.strftime('%b %d') for x in np.unique(overall_series[state].index.date)],
                y = overall_series[state].values,
                text = [str(x) for x in overall_series[state].values],
                hoverinfo = 'text',
                textposition = 'auto',
                opacity=1,
                marker=dict(color=colors[state]),
                name=state,

            ) for state in overall_series.keys()
    ]
    if not in_focus or (any(['failed' in x for x in in_focus]) and any(['passed' in x for x in in_focus])):
        my_data.extend([
            go.Scatter(
                mode='lines',
                x = [x for x in fail_rate.keys()],
                y = list(fail_rate.values()),
                name='Fail Rate ± Standard Deviation',
                hoverinfo = 'text',
                text=[(re.search(r'(?<=\.)(\d){2}', str(x)).group(0)+'% ± '+str(y)) for (x, y) in zip(list(fail_rate.values()), [std]*len(list(fail_rate.values())))],
                yaxis='y2',
                error_y=dict(
                    type='percent',
                    value=std,
                    #type='data',
                    #array=[std]*len(list(fail_rate.values())),
                    visible=True
                ),
                marker=dict(color='#c97b42')

            ),

        ])

    # define layout properties for our chart
    layout = go.Layout(
        title= ('Overview'),
        titlefont=dict(
            family='Courier New, monospace',
            size=15,
            color='#7f7f7f'
        ),
        autosize=False,
        height=800,
        width=800,
        hoverlabel=dict(namelength=35),
        paper_bgcolor='rgba(0,0,0,0)',
        plot_bgcolor='rgba(0,0,0,0)',
        #barmode='group',
        margin={'l': 30, 'r': 30},
        hovermode='closest',
        bargap=0.025,
        bargroupgap=0.005,
        legend=dict(
            x=0,
            y=1.0,
            bgcolor='rgba(255, 255, 255, 0)',
            bordercolor='rgba(255, 255, 255, 0)'
        ),

        yaxis=dict(
            tickfont=dict(
                color='#7f7f7f'
            ),
            domain=[0, .7],
            type='log',
            autorange=True,
            showgrid=False,
            showline=False,
            showticklabels=False,
        ),
        yaxis2=dict(
            side='right',
            #overlaying='y',
            domain=[.7,.9],
            titlefont=dict(
                color='#7f7f7f'
            ),
            tickfont=dict(
                color='#7f7f7f'
            ),
            autorange=True,
            #range=[0,1],
            showgrid=False,
            showline=False,
            showticklabels=False,
            zeroline=False,
        )
    )

    return go.Figure(data=my_data, layout=layout)

# debugging purposes -------------------------------------------------------------------
if __name__ == '__main__':
    df = load_testresults_todataframe('Data/')
    update_barplot(df, dt(2018,9,4), dt(2018,10,17), ['passed', 'failed_2'])

'''
SAVING INTERESTING PATTERNS

[1] - dict of (feature: series) where the series is a resampled count of the occurance of the feature per day (only ones with occurances)
It could be useful in a pie chart:

features = df.state.np.unique()
    feature_series = {}

    for feat in features:
        feature_series[feat] = df[(df.state==feat)].state.resample('D').count().replace(0, np.nan).dropna()

[2] - data field for a graph figure

'data': [
            go.SOMEPLOT(
                r = feature_series[feature].values,
                theta = [x.strftime('%d %b') for x in feature_series[feature].index],
                text= [feature]*len(td),
                name=feature,
                mode = 'markers',
                marker = dict(
                    color = color,
                    size = 15,
                    line = dict(
                        color = "white"
                    ),
                    opacity=0.7,
                ),
            ) for (feature, color) in zip(features, colors)
        ],
'''


import datetime
import time
from urllib.request import urlopen

isGame = True

#get all games for the day
url = 'http://live.nhle.com/GameData/GCScoreboard/'+str(datetime.date.today())+'.jsonp'
#url = 'http://live.nhle.com/GameData/GCScoreboard/2015-10-08.jsonp'
socket = urlopen(url)
html = socket.read().decode()
socket.close()
if '"id"' not in html:
    print("No Games")
    isGame = False
else:
    if '"CHI"' not in html: #abbr
        isGame = False
        print("No Blackhawks Game")
    lines = html.split("},{")
    i = 0

    while i < len(lines):
        if '"CHICAGO"' in lines[i]:
            iid = lines[i].index('"id":')
            id = lines[i][iid+5:iid+15]
            print("GID: " + id)
            break
        else:
            i = i+1

#import pyglet
#sound = pyglet.media.load('gh.mp3', streaming=False)
#sound.play()
#pyglet.app.run()

while isGame:
#get scoreboard for selected game
    url = 'http://live.nhle.com/GameData/20152016/'+id+'/gc/gcsb.jsonp'
    socket = urlopen(url)
    html = socket.read().decode()
    socket.close()
    print(html)
#get box score for selected game
    url = 'http://live.nhle.com/GameData/20152016/'+id+'/gc/gcbx.jsonp'
    socket = urlopen(url)
    html = socket.read().decode()
    socket.close()
    print(html)

    url = 'http://live.nhle.com/GameData/GCScoreboard/'+str(datetime.date.today())+'.jsonp'
    #url = 'http://live.nhle.com/GameData/GCScoreboard/2015-10-08.jsonp'
    socket = urlopen(url)
    html = socket.read().decode()
    socket.close()
    print(html)
#refresh delay
    time.sleep(30)


# -*- coding:utf-8 -*-
"""
cron: 0 30 0 * * *
new Env('微信小程序-口味王');
"""

import time
import os
try:
    import requests
except Exception as e:
    print(e, "\n缺少requests 模块，请执行命令安装：python3 -m pip install requests")
    exit(3)
'''
版本 v1.0.1
====================== Cookie 配置===========================
'''
mycookies = [
]
try:
    mycookies = os.environ["KWW_COOKIE"].split("&")
except:
    print("【提示】请先获取微信小程序[口味王]cookie,环境变量添加 KWW_COOKIE ,如有不懂加群：212796668、681030097、743744614")
    exit(3)
'''
#2|“口味王”槟榔曾冠名过《这！就是灌篮3》吗？——正确
#3|“口味王”槟榔曾赞助过综艺《这！就是街舞4》吗？——正确
#4|“口味王”是槟榔行业领导品牌；——正确
#6|口味王已连续4年举办“我要上大学”公益活动？——正确
#7|游戏上分时，使用“口味王”槟榔可以秒回状态？——正确
#8|开车犯困的时候，使用“口味王”槟榔可以秒回状态？——正确
#9|熬夜加班的时候，使用“口味王”槟榔可以秒回状态？——正确
#10|槟榔品牌中，“口味王”槟榔销量全国领先吗？——正确
#11|“口味王”槟榔曾赞助过电视剧《鬼吹灯之云南虫谷》吗？——正确
#12|“口味王”槟榔曾赞助过电视剧《将夜2》吗？——正确
#13|“口味王”槟榔曾赞助过电视剧《鬼吹灯之龙岭迷窟》吗？——正确
#15|“口味王”槟榔曾赞助过电视剧《巡回检察组》吗？——正确
#16|“口味王”槟榔曾赞助《欢乐喜剧人5》吗？——正确
#17|2019年“口味王”冠名虎牙英雄联盟S9全球总决赛直播吗？——正确
#18|口味王已连续6年开展“温暖回家”公益活动——正确
#19|“口味王”槟榔曾赞助过电视剧《巡警之海外行动》吗？——正确
#20|“口味王”槟榔是_____的槟榔品牌。——全国销量领先 "全国销量领先","全国高端销量领先","湖南销量领先","湖南高端销量领先"
#21|以下哪个是“和成天下”槟榔的特点？——以上全是 "10包高端槟榔，7包和成天下","全国高端销量领先","海南嫩青果，味美更耐嚼","以上全是"
#22|“和成天下”槟榔是全国高端销量领先的槟榔品牌吗？——正确
#23|10包高端槟榔，有几包是“和成天下”？——7 4,5,6,7
#24|10包高端槟榔，有7包是“和成天下”对吗？——正确
#27|“和成天下”槟榔生产需要多少道工艺？——22道
#28|“和成天下”槟榔生产需要22道工艺？——正确
#30|为什么“和成天下”坚持使用海南槟榔果生产？——以上都是 "海南槟榔果汁多肉厚","海南槟榔果果形匀称","海南槟榔果更耐嚼","以上都是"
#31|高端槟榔市场占有率最高的品牌是“和成天下”吗？——正确
#32|“和成天下”品牌采用海南原果的比例是多少？——100% "50%","80%","70%","100%"
#33|“和成天下”是哪一类槟榔的引领者和典型代表？——高端槟榔
#34|“和成天下”是高端槟榔槟榔的引领者和典型代表？——正确
#35|“和成天下”是湖南人更爱吃的槟榔吗？——正确
#36|“和成天下”曾赞助过芒果TV《野生厨房》吗？——正确
#38|“和成天下”采用的是_____槟榔？——海南嫩青果 海南嫩青果,湖南嫩青果,河南嫩青果,云南嫩青果
#39|“和成天下”品牌广告语：“海南_______，味美更耐嚼”——嫩青果 好果子,万宁果,嫩青果,秋前果
#41|“和成天下”品牌广告语：“_____嫩青果，味美更耐嚼”——海南
#42|“和成天下”槟榔原果来自全球槟榔黄金产区——海南吗？——正确
#43|“和成天下”槟榔是_____的槟榔品牌。——全国高端销量领先 "全国销量领先","全国高端销量领先","湖南销量领先","湖南高端销量领先"
#44|海南的槟榔果有什么特点——以上都是 "汁多肉厚","果形匀称","耐嚼","以上都是"
#45|全球槟榔的黄金产区在哪里——中国海南 "印度尼西亚","中国台湾","菲律宾","中国海南"
#46|海南的三大支柱产业是什么？——槟榔、椰子、橡胶   椰子、橡胶、芒果,槟榔、椰子、橡胶,槟榔、椰子、芒果,槟榔、橡胶、芒果
#48|与东南亚槟榔相比，海南槟榔更适合加工成干果进行咀嚼吗？——正确
#50|中国“四大南药”之首是什么？——槟榔 "益智仁","槟榔","砂仁","巴戟天"
#51|白居易的千古名句：“戴花红石竹，帔晕紫_____”——槟榔 "枸杞","黄芪","人参","槟榔"
#52|《本草纲目》中描述槟榔有哪些功效？——以上都是 解油,驱虫,除胀,以上都是
#53|邓丽君的歌曲：“高高的树上结_____，谁先爬上谁先尝”——槟榔 槟榔,椰子,榴莲,菠萝蜜
#55|“可疗饥怀香自吐，能消瘴疠暖如熏。”这句诗描写的是哪种植物？——槟榔树 "椰子树","橡胶树","槟榔树","橄榄树"
#56|西汉汉武帝曾制定_____作为皇室贡品进献——以上都是 "荔枝","槟榔","珍珠","以上都是"
#57|槟榔起源于哪里？——马来西亚 "菲律宾","越南","印度尼西亚","马来西亚"
#59|海南万宁被称为什么——槟榔之乡 槟榔之乡,芒果之乡,椰子之乡,凤梨之乡
#60|为什么海南是全球槟榔的黄金产区——以上都是 温差小,日照长,空气湿度高,以上都是
#61|海南万宁的槟榔果是槟榔中的黄金圣果吗？——正确
#62|国内嚼食槟榔以果肉为主，越南等东南亚国家则以带核槟榔为主——正确
#63|与东南亚槟榔相比，海南槟榔纤维细腻更适合咀嚼吗？——正确
#65|李白的千古名句：“何时黄金盘，一斛荐_____”——槟榔 "槟榔","黄芪","人参","枸杞"
#66|“若笋竹生竿，种之精硬，引茎直上，不生枝叶。”描写的是哪种植物？——槟榔树 "椰子树","橡胶树","槟榔树","橄榄树"
#67|“高高的树上结槟榔，谁先爬上谁先尝”是哪首歌的歌词？——《采槟榔》 "《采槟榔》","《摘槟榔》","《尝槟榔》","《品槟榔》"
#68|“开花树杪翻青箨，结子苞中皱锦纹。”这句诗描写的是哪种植物？——槟榔树 "槟榔树","橡胶树","椰子树","橄榄树"
#70|考古发现最早的槟榔遗骸是哪年？——公元前10,000年 "公元前1,000年","公元前3,000年","公元前7,000年","公元前10,000年"
#71|“朗是宾门篓似妾，结成一对好鸳鸯。”这句诗描写的是哪种植物？——槟榔树 椰子树,橡胶树,橄榄树,槟榔树
#72|槟榔果一般在什么时间采摘？——8~10月 8~10月,11~1月,2~4月,5~7月
====================== 题库 ===========================
'''
ktList = {'1':'',#题目未知
             '2':'正确',
             '3':'正确',
             '4':'正确',
             '5':'',#
             '6':'正确',
             '7':'正确',
             '8':'正确',
             '9':'正确',
             '10':'正确',
             '11':'正确',
             '12':'正确',
             '13':'正确',
             '14':'',#
             '15':'正确',
             '16':'正确',
             '17':'正确',
             '18':'正确',
             '19':'正确',
             '20':'全国销量领先',
             '21':'以上全是',
             '22':'正确',
             '23':'7',
             '24':'正确',
             '25':'',#
             '26':'',#
             '27':'22道',
             '28':'正确',
             '29':'',#
             '30':'以上都是',
             '31':'正确',
             '32':'100%',
             '33':'高端槟榔',
             '34':'正确',
             '35':'正确',
             '36':'正确',
             '37':'',#
             '38':'海南嫩青果',
             '39':'嫩青果',
             '40':'',#
             '41':'海南',
             '42':'正确',
             '43':'全国高端销量领先',
             '44':'以上都是',
             '45':'中国海南',
             '46':'槟榔、椰子、橡胶',
             '47':'',#
             '48':'正确',
             '49':'',#
             '50':'槟榔',
             '51':'槟榔',
             '52':'以上都是',
             '53':'槟榔',
             '54':'',#
             '55':'槟榔树',
             '56':'以上都是',
             '57':'马来西亚',
             '58':'',#
             '59':'槟榔之乡',
             '60':'以上都是',
             '61':'正确',
             '62':'正确',
             '63':'正确',
             '64':'',#
             '65':'槟榔',
             '66':'槟榔树',
             '67':'《采槟榔》',
             '68':'槟榔树',
             '69':'',#
             '70':'公元前10,000年',
             '71':'槟榔树',
             '72':'8~10月'
             }

'''
====================== 请求 ===========================
'''
def getApiBody(url, body):
    headers = {
        'Content-Type': 'application/json',
    }
    return requests.get(url=url,body=body, headers=headers, timeout=300).json()

def getApi(url, headers):
    return requests.get(url=url, headers=headers, timeout=300).json()

def getApiR(url, headers):
    return requests.get(url=url, headers=headers, allow_redirects=False, timeout=300)

if __name__ == '__main__':
    for i in range(len(mycookies)):
        print("用户【" + mycookies[i] + "】")
        url = 'https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=select-member-score&formName=searchForm&memberId=' + mycookies[i]
        headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
        }
        res = getApi(url, headers)
        print("积分剩余 : " + str(res['rows'][0]))
        if int(res['rows'][0]) >= 888:
            print("积分大于 888 分，请去微信公众号【口味王】兑换红包")
        print("【每日签到】")
        res = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=selectSignInfo&formName=searchForm&memberId=' + mycookies[i], headers)
        propList = res['rows']['data']
        for prop in range(len(propList)):
            if propList[prop]['flag'] == "1":
                print(propList[prop]['formulaDesc'] + " " + ("未签到" if propList[prop]['flag'] == "0" else  "已签到") + "  签到奖励 " + propList[prop]['paramNo'])
                if propList[prop]['flag'] == "0":
                    body = {
                    "pageName": "AddSignSvmInfo",
                                "formName": "addForm",
                    'orderNo': propList[prop]['orderNo'],
                    'paramNo': propList[prop]['paramNo'],
                    'cateId': propList[prop]['cateId'],
                    'memberId': mycookies[i] ,
                    'memberName': "JDWXX" ,
                    }
                    jf = getApiBody('https://member.kwwblcj.com/member/api/submit/?userKeys=v1.0', body)
                    print(jf['msg'])
                    break
        print("【任务】")
        res = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=select-task-list&formName=searchForm&memberId=' + mycookies[i],headers)
        propList = res['rows']
        for prop in range(len(propList)):
            print(propList[prop]['taskTitle'] + "  " + "奖励积分:" +propList[prop]['rewardScore'] + " ->  " + ("待完成" if propList[prop]['complete'] == '0' else "已完成"))
            if propList[prop]['taskTitle'] == "开启签到提醒" and propList[prop]['complete'] == "0":
                print("【开启签到提醒】")
                jf = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=setOpenSignTaskFlag&formName=addForm&memberId=' + mycookies[i] + '&userCname=JDWXX&openId=o_V6_5Yo3mET81MVAQw4yYebL3zE', headers)
                print('完成' if jf['rows'][0] == 'T' else '失败，去手动完成任务')
                jf = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=setOpenSignTaskFlag&formName=addForm&memberId=' + mycookies[i] + '&userCname=JDWXX&openId=0', headers)
                print('完成' if jf['rows'][0] == 'T' else'失败，去手动完成任务')
            if propList[prop]['taskTitle'] == "订阅活动通知" and propList[prop]['complete'] == "0":
                print("【订阅活动通知】")
                jf = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=setOpenSubscribeTaskFlag&formName=addForm&memberId=' + mycookies[i] + '&userCname=JDWXX&openId=o_V6_5Yo3mET81MVAQw4yYebL3zE', headers)
                print('完成' if jf['rows'][0] == 'T' else '失败，去手动完成任务')
                jf = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=setOpenSubscribeTaskFlag&formName=addForm&memberId=' + mycookies[i] +'&userCname=JDWXX&openId=0', headers)
                print('完成' if jf['rows'][0] == 'T' else '失败，去手动完成任务')
        print("【每日阅读】")
        jf = getApi('https://member.kwwblcj.com/member/api/list/?userKeys=v1.0&pageName=setNewsReadTaskFlag&formName=addForm&memberId=' + mycookies[i] + '&userCname=JDWXX&articleTitle=undefined', headers)
        print("阅读日期：" + jf['rows'][0])
        print("【答题任务】")
        url = 'https://member.kwwblcj.com/member/api/info/?userKeys=v1.0&pageName=loginFreePlugin&formName=searchForm&uid=' + mycookies[i] +'&levelCode=K1&redirect=https%3A%2F%2F89420.activity-20.m.duiba.com.cn%2Fprojectx%2Fp129446ea%2Findex.html%3FappID%3D89420'
        res = getApi(url, headers)
        cookie = getApiR(res['result'], headers).headers.get('Set-Cookie')
        cookieList = cookie.split(";")
        cookie_wdata4 = ''
        cookie_w_ts = ''
        cookie__ac = ''
        for ii in range(len(cookieList)):
            if cookieList[ii].find('wdata4') != -1:
                cookie_wdata4 = cookieList[ii]
            if cookieList[ii].find('w_ts') != -1:
                cookie_w_ts = cookieList[ii]
            if cookieList[ii].find('_ac') != -1:
                cookie__ac = cookieList[ii]
        Cookie = cookie_wdata4 + ";" + cookie_w_ts + ";" + cookie__ac
        Cookie = Cookie.replace("HttpOnly,","")
        headers = {
            'Cookie': Cookie,
        }
        res = getApi('https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/start.do?user_type=0&is_from_share=1&_t=' + str(time.time()), headers)
        startId = str(res['data'])
        if startId == 'None':
            print("今日已答题")
            continue
        i = 1
        while i < 5:
            i += 1
            url = 'https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/getQuestion.do?startId=' + startId + '&user_type=0&is_from_share=1&_t=' + str(time.time())
            res = getApi(url, headers)
            print("题目ID：" + res['data']['id'] + " > " + res['data']['content'])
            answerList = res['data']['answerList']
            print(str(answerList))
            dt = False
            for jj in range(len(answerList)):
                if answerList[jj].find(ktList.get(res['data']['id'])) != -1:
                    print('找到答案 >' + ktList.get(res['data']['id']))
                    print('提交答案 > 提交值 >' + str(jj + 1))
                    url = 'https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/submit.do?answer=' + str(jj + 1) + '&startId=' + startId + '&user_type=0&is_from_share=1&_t=' + str(time.time())
                    res = getApi(url, headers)
                    print(res)
                    dt = True
                    break
            if dt == False:
                for jj in range(len(answerList)):
                    print(answerList[jj])
                    print(ktList.get(res['data']['id']))
                    if answerList[jj].find("正确") != -1 \
                            or answerList[jj].find("槟榔") != -1 \
                            or answerList[jj].find("海南") != -1 \
                            or answerList[jj].find("100%") != -1 \
                            or answerList[jj].find("中国") != -1 \
                            or answerList[jj].find("全国") != -1 \
                            or answerList[jj].find("领先") != -1:
                        print('未找到答案 找到默认配置规则')
                        print('提交答案 > 提交值 >' + str(jj + 1))
                        url = 'https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/submit.do?answer=' + str(jj + 1) + '&startId=' + startId + '&user_type=0&is_from_share=1&_t=' + str(time.time())
                        res = getApi(url, headers)
                        print(res)
                        dt = True
                        break
            if dt == False:
                print('未找到配置')
                print('提交答案 > 提交默认值 1')
                url = 'https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/submit.do?answer=1&startId=' + startId + '&user_type=0&is_from_share=1&_t=' + str(time.time())
                res = getApi(url, headers)
                print(res)
        # 领取奖励
        url = 'https://89420.activity-20.m.duiba.com.cn/projectx/p129446ea/answer/complete.do?startId=' + startId + '&user_type=0&is_from_share=1&_t=' + str(time.time())
        res = getApi(url, headers)
        print(res)


import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

from sklearn.datasets import load_iris

iris = load_iris()

# df = pd.DataFrame(iris,columns=['SepalLength','SepalWidth','PetalLength','PetalWidth'])
# df['class'] = iris['target']
# df['SepalLength'] = iris.data[:,0]
# df['SepalWidth'] = iris.data[:,1]
# df['PetalLength'] = iris.data[:,2]
# df['PetalWidth'] = iris.data[:,3]
# df.to_csv('D:/资料/数据挖掘/iris.csv',index=None)

data = pd.read_csv('D:/资料/数据挖掘/iris.csv',encoding='gbk')
# print(data)

# plt.rcParams['font.sans-serif'] = ['SimHei']
#
# plt.figure(figsize=(10,8))
# colors = ['red','blue','green']
# labels = ['setosa','versicolor','Vinginica']
#
# for i in range(3):
#     plt.scatter(x=data['SepalLength'][i*50:(i+1)*50],
#                 y=data['SepalWidth'][i*50:(i+1)*50],
#                 c=colors[i],
#                 label=labels[i])
#
#
# plt.legend()
# plt.show()

data['scale'] = data['SepalLength'] / data['SepalWidth']
print(data)

# 标准化
from sklearn.preprocessing import MinMaxScaler
scale = MinMaxScaler()
data1 = scale.fit_transform(data)
data1 = data1[:,:4]
# print(data1)

# 创建PCA对象
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
pca = pca.fit(data1)

# 绘制贡献度曲线
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('Number of components')
plt.ylabel('Cumulayive explained variance')

# 根据贡献度选择合适的维度
threshold = 0.95
n_components = np.argmax(np.cumsum(pca.explained_variance_ratio_) >= threshold) + 1
print('Number of omponents to retain 95% varvance', n_components)

# 进行PCA降维
pca = PCA(n_components=n_components)
data_pca = pca.fit_transform(data1)
print(data_pca)

colors = ['red','blue','yellow']
labels = ['setosa','versicolor','Vinginica']
plt.figure()
for i in range(3):
    plt.scatter(x=data_pca[i*50:(i+1)*50, 0],
                y=data_pca[i*50:(i+1)*50, 1],
                alpha=1,
                c=colors[i],
                label=labels[i])

plt.title('Scatter Plot')
plt.show()

# movie_data = pd.read_csv("D:/python爬虫/douban.csv",encoding='gbk')
# # movie_data.dropna(axis='index',how='any',inplace=False)
# data = movie_data.loc[:,['score','num']]
#
# # print(data)
#
# from sklearn.preprocessing import StandardScaler
#
# tool = StandardScaler()
# data1 = tool.fit_transform(data)

# # axis=0  返回的是一列的均值
# mean_val = np.mean(data1,axis=0)
# # 取中心化
# mean_removed = data1 - mean_val
# # 获取协方差矩阵
# cov_mat = np.cov(mean_removed, rowvar=0)
# # 获取矩阵特征值与特征向量（计算贡献率）
# # eigen_vals特征根  eigen_vecs特征向量
# eigen_vals, eigen_vecs = np.linalg.eig(cov_mat)
# percentage = eigen_vals/sum(eigen_vals)
# # print(percentage)
# # 特征根排序，接着删除解释量小的特征根
# eigen_val_ind = np.argsort(eigen_vals)
# # print(eigen_val_ind)
# topNfeat = 1
# eigen_val_ind = eigen_val_ind[-1:(-topNfeat+1):-1]
# # print(eigen_val_ind)
# # 特征向量由高到低排序 取最大的n个值对应的特征向量
# red_eigen_vecs = eigen_vecs[:,eigen_val_ind]
# # print(red_eigen_vecs)
# # 将原始数据投影到主成分上得到新的低维数据low_data
# # 矩阵相乘 @
# low_data = mean_removed @ red_eigen_vecs
# # 得到重构数据reconMat
# recon_mat = (low_data @ red_eigen_vecs.T) + mean_val
# print(recon_mat)
#
# # 绘图
# # plt.figure(figsize=(8,6))
# # x1 = data1[:,0]
# # y1 = data1[:,1]
# # plt.scatter(x=x1,y=y1,color='green',alpha=0.6,s=30,label='original data')
#
# x2 = data1[:,0]
# y2 = data1[:,1]
# plt.scatter(x=x2,y=y2,color='purple',alpha=0.6,s=30,label='after z-score')
#
# x3 = np.array(recon_mat)[:,0]
# y3 = np.array(recon_mat)[:,1]
# plt.scatter(x=x3,y=y3,color='yellow',alpha=0.5,s=30,label='recon_mat')
#
# plt.legend()
# plt.show()


from pydriller import Repository
import xlsxwriter
from urlRepo import repo
from random import *
import os

workbook = xlsxwriter.Workbook('hello.xlsx')
worksheet = workbook.add_worksheet()

i = 1
listaCommit = []

for r in repo:
    try:
        for commit in Repository(r).traverse_commits():
            listaCommit.append(commit.msg+" , "+r)
    except:
        listaCommit.append("####")

for i in range(2001):
    rand = randint(1, 684589)
    worksheet.write(f'A{i}', listaCommit[rand])
    print(listaCommit[rand])

workbook.close()
os.system("pause")


import selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
import time

chrome_driver_path = "E:\setups\chromedriver_win32\chromedriver.exe"

# Create a new instance of the Chrome driver
driver = webdriver.Chrome(chrome_driver_path)

driver.get("http://20.204.138.86/login")

time.sleep(5)

loginForm = driver.find_element(By.CSS_SELECTOR, '.login-form')
uf = loginForm.find_element(By.XPATH, '//input[1]')
pf = loginForm.find_element(By.XPATH, '//input[2]')
loginButton = loginForm.find_element(By.XPATH, '//button')

uf.send_keys("hrishikesh214")
pf.send_keys("hrishi")

loginButton.click()

time.sleep(2)

if driver.current_url == "http://20.204.138.86/":
    print("Test Passed")
else:
    print("Test Failed")

driver.close()


# from django.forms.models import modelformset_factory
from django.forms.formsets import formset_factory
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render, render_to_response
from django.core.urlresolvers import reverse
from polls.models import *
import json

input_dict = {
    1: 'radio',     #тип 1.2
    2: 'checkbox',  #тип 1.3
    3: 'radio',     #тип 1.1
}

def index(request):
    return HttpResponse("Hello, world. You're at the polls index.")

def detail(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    return render(request, 'detail.html', {'question': question})

def make_context(amount=10):
    quest_list = []
    index = 0
    AllQuestion = Question.objects.all()[:amount]
    for quest in AllQuestion:
        ChoiceForQuestion = Choice.objects.filter(question=quest.pk)
        chois_list = []
        quest_dict = {}
        index += 1
        if ChoiceForQuestion:
            for chois in ChoiceForQuestion:
                chois_dict = {}
                chois_dict['choice_text'] = chois.choice_text
                chois_dict['choice_cost_positive'] = chois.choice_cost_positive
                chois_dict['choice_cost_negative'] = chois.choice_cost_negative
                chois_dict['choice_is_true'] = chois.choice_is_true
                chois_dict['choice_id'] = chois.id
                chois_list.append(chois_dict)
        quest_dict['question_text'] = quest.question_text
        quest_dict['question_choice'] = chois_list
        quest_dict['question_input_type'] = input_dict[quest.question_type]
        quest_dict['question_type'] = quest.question_type
        quest_dict['question_cost_positive'] = quest.question_cost_positive
        quest_dict['question_cost_negative'] = quest.question_cost_negative
        quest_dict['question_id'] = quest.id
        quest_dict['index'] = index
        quest_list.append(quest_dict)
    return quest_list

def make_context_js(amount=10):
    quest_list = []
    index = 0
    AllQuestion = Question.objects.all()[:amount]
    for quest in AllQuestion:
        ChoiceForQuestion = Choice.objects.filter(question=quest.pk)
        chois_list = []
        quest_dict = {}
        index += 1
        if ChoiceForQuestion:
            for chois in ChoiceForQuestion:
                chois_dict = {}
                chois_dict['choice_text'] = chois.choice_text
                chois_dict['choice_id'] = chois.id
                chois_list.append(chois_dict)
        quest_dict['question_text'] = quest.question_text
        quest_dict['question_choice'] = chois_list
        quest_dict['question_input_type'] = input_dict[quest.question_type]
        quest_dict['question_id'] = quest.id
        quest_dict['index'] = index
        quest_list.append(quest_dict)
    return quest_list

c = make_context()

c_js = make_context_js()

# def test(request):
#
#     return render_to_response('test.html', {
#         'formset': c_js,
#     })

#для jsona
def test(request):

    context_js = json.dumps(c_js, ensure_ascii=False)

    return render_to_response('jstext.html', {
        'formset': context_js,
    })

def vote(request):

    if request.method == 'POST':
        kwargs = [k for k in request.POST.lists()]
        keys_c = [k for k in request.POST.dict().keys()]
        BSum = 0 #Базовая оценка
        YSum = 0 #Продуктивная оценка
        for i in range(0, len(keys_c)):
            quest_id = keys_c[i]
            choice_list = request.POST.getlist(keys_c[i])
            for quest_index in c:
                if quest_index['question_id'] == int(keys_c[i]):
                    quest = quest_index
            asda = 'asd'
            # тип вопроса 1.2
            if quest['question_type'] == 1:
                BSum = BSum + max([v['choice_cost_positive'] for v in quest['question_choice']])
                for choice_index in quest['question_choice']:
                    if choice_index['choice_id'] == int(choice_list[0]):
                        choice_cost = choice_index['choice_cost_positive']
                YSum = YSum + choice_cost
            #остальные типы вопросов
            else:
                BSum = BSum + quest['question_cost_negative']
                if quest['question_type'] == 3:
                    for choice_index in quest['question_choice']:
                        if choice_index['choice_id'] == int(choice_list[0]):
                            if choice_index['choice_is_true']:
                                BSum = BSum - quest['question_cost_negative'] + quest['question_cost_positive']
                                YSum = YSum + quest['question_cost_positive']
                if quest['question_type'] == 2:
                    true_index = 0
                    true_choice = 0
                    for choice_index in quest['question_choice']:
                        if choice_index['choice_is_true']:
                            true_index += 1
                        for choice_selected in choice_list:
                            if int(choice_selected) == choice_index['choice_id']:
                                if choice_index['choice_is_true']:
                                    true_choice += 1

                    if true_choice == true_index:
                        YSum = YSum + quest['question_cost_positive']
        rating = 100 * (YSum/BSum)
        ok = 'неудача'
        if rating > 80:
            ok = 'Отлично'
        elif rating > 65:
            ok = 'Хорошо'
        elif rating > 50:
            ok = 'Удовлетворительно'
        return render_to_response("vote.html", {
            "vote": ok,
        })


import logging

from django.utils.translation import ugettext_lazy as _  # noqa

from horizon import exceptions
from horizon import forms
from horizon import messages

from wildcard import api

LOG = logging.getLogger(__name__)

class BaseQueueForm(forms.SelfHandlingForm):

    name = forms.CharField(label=_("Name"))
    description = forms.CharField(
        required=False,
        label=_("Description"),
        widget=forms.widgets.Textarea,
    )

class CreateQueueForm(BaseQueueForm):

    def handle(self, request, data):
        try:
            queue = api.payload.queue_create(request, **data)
            messages.success(
                request,
                _('Queue "%s" was successfully created.') % data['name']
            )
            return queue
        except Exception:
            exceptions.handle(request, _('Unable to create queue.'))

class UpdateQueueForm(BaseQueueForm):

    uuid = forms.CharField(label=_("UUID"), widget=forms.HiddenInput)

    def handle(self, request, data):
        uuid = data.pop('uuid')
        try:
            api.payload.queue_update(request, uuid, **data)
            messages.success(
                request,
                _('Queue has been updated successfully.')
            )
        except Exception:
            exceptions.handle(request, ignore=True)
            messages.error(request, _('Unable to update the queue.'))
        return True


import gi

gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

class ErrorDialog(Gtk.Dialog):
    def __init__(self, error_message, parent):
        Gtk.Dialog.__init__(self, title="ERROR", transient_for=parent, flags=0)
        self.add_buttons(Gtk.STOCK_OK, Gtk.ResponseType.OK)
        self.set_default_size(150, 100)
        label = Gtk.Label(label=error_message)
        box = self.get_content_area()
        box.add(label)
        self.show_all()


#uvicorn --host 0.0.0.0 app.main:app --reload
#docker build -t api-greenhouse .
#docker run --env-file env_file -d --restart always --name greenhouse-container -p 8000:8000 api-greenhouse
import uvicorn
from fastapi import FastAPI
from motor.motor_asyncio import AsyncIOMotorClient

from app.routers import router
from app.config import settings

app = FastAPI()

print(f"Connection server:{settings.DB_URL}")

@app.on_event("startup")
async def startup_db_client():
    app.mongodb_client = AsyncIOMotorClient(settings.DB_URL)
    app.mongodb = app.mongodb_client[settings.DB_NAME]

@app.on_event("shutdown")
async def shutdown_db_client():
    app.mongodb_client.close()

app.include_router(router, tags=["values"], prefix="/greenhouse")

if __name__ == "__main__":
    uvicorn.run(
        "main:app",
        host = settings.HOST,
        reload = settings.DEBUG_MODE,
        port = settings.PORT
    )


#!/usr/bin/env python

# Dump protobuf descriptors from binary images of the MAIN modem app for Samsung
# modems used by Google Pixel devices. The code in those binary images uses nanopb
# for handling protobuf, and the descriptors store some type information for
# decoding the messages, which is also useful for almost fully reconstructing the
# original protobuf specification files. However, some information, such as enums
# and the actual type and signness of fixed-size types, is unfortunately lost.
# These can only be recovered by reverse engineering the code that uses the decoded
# data.

# NOTE: This code only works with descriptors generated by nanopb versions in the
# range 0.2.5 to 0.3.9.3 (0.3.9.4 added an explicit bool type and broke compatiblity
# by adding it to the beginning of the types "enum", and 0.4.0 completely changed
# the format of the descriptor). Make sure that the binary you're analysing uses
# a nanopb version that is supported by this script by manually inspecting the
# generated output.

from __future__ import print_function
import enum  # Requires enum34 on Python 2
import io
import os
import struct

class NanoDesc(object):
    _FMT = struct.Struct(
        '<I'  # Tag
        'B'  # Type
        'I'  # Data offset
        'i'  # Size offset
        'I'  # Data size
        'I'  # Array size
        'I'  # Ptr (for sub-messages, extensions, and default values)
        )
    SIZE = _FMT.size

    # Needed for cleanly checking if we got an empty descriptor on Python 2
    ZERO = b''.join(b'\x00' for _ in range(SIZE))

    class Type(enum.IntEnum):
        VarInt = 0
        UVarInt = 1
        SVarInt = 2
        Fixed32 = 3
        Fixed64 = 4
        Bytes = 5
        String = 6
        Submessage = 7
        Extension = 8
        FixedLengthBytes = 9

    TYPE_SHIFT = 0
    TYPE_MASK = 0x0f

    class Attribute(enum.IntEnum):
        required = 0
        optional = 1
        repeated = 2
        oneof = 3

    ATTR_SHIFT = 4
    ATTR_MASK = 0x3

    class Allocation(enum.IntEnum):
        static = 0
        callback = 1
        pointer = 2

    ALLOC_SHIFT = 6
    ALLOC_MASK = 0x3

    def __init__(self, dio, ref_base):
        data = dio.read(self.SIZE)
        if len(data) != self.SIZE:
            raise EOFError('Unexpected EOF')

        if data == self.ZERO:
            self.tag = 0
            self.type = 0
            self.alloc = 0
            self.attr = 0
            self.data_offset = 0
            self.data_size = 0
            self.size_offset = 0
            self.array_size = 0
            self.ptr_or_def = None

            return

        tag, typ_attr_alloc, doff, szoff, dsz, arsz, ptr = self._FMT.unpack(data)

        alloc = self.Allocation((typ_attr_alloc >> self.ALLOC_SHIFT) & self.ALLOC_MASK)
        attr = self.Attribute((typ_attr_alloc >> self.ATTR_SHIFT) & self.ATTR_MASK)
        typ = self.Type((typ_attr_alloc >> self.TYPE_SHIFT) & self.TYPE_MASK)

        if typ == self.Type.Submessage:
            if ptr == 0:
                raise ValueError('Expected a pointer for submessage')
        if ptr != 0:
            if typ in (self.Type.Bytes, self.Type.String, self.Type.FixedLengthBytes) and alloc != self.Allocation.static:
                raise ValueError(
                    'Unexpected pointer field for bytes/string without static allocation (type={}, alloc={})'.format(
                        typ.name, alloc.name))
            if ptr < ref_base:
                raise ValueError('ptr pointing before ref base ({:08x} < {:08x})'.format(ptr, ref_base))
            ptr -= ref_base

        if szoff != 0:
            if ((attr == self.Attribute.oneof) or
                (attr == self.Attribute.optional and alloc == self.Allocation.static) or
                (attr == self.Attribute.repeated and alloc in (self.Allocation.static, self.Allocation.pointer))):
                if szoff > 0:
                    raise ValueError('Size offset must be negative, found {}'.format(szoff))
            else:
                raise ValueError('Found size offset for a field that is not optional or repeated')

        if arsz != 0 and attr != self.Attribute.repeated and alloc != self.Allocation.static:
            raise ValueError('Found array size for a field that is not repeated static')

        if alloc == self.Allocation.callback:
            if attr == self.Attribute.oneof:
                raise ValueError('Found OneOf with callback allocation')

            if dsz != 8:
                raise ValueError('Callback with invalid size = {}'.format(dsz))
        # elif typ == self.Type.Bool:
        #     if dsz == 0 or dsz > 4:
        #         raise ValueError('Bool with invalid size = {}'.format(dsz))
        elif typ in (self.Type.VarInt, self.Type.UVarInt, self.Type.SVarInt):
            if dsz == 0 or dsz > 8:
                raise ValueError('Int with invalid size = {}'.format(dsz))
        elif typ == self.Type.Fixed32:
            if dsz != 4:
                raise ValueError('Fixed32 with invalid size = {}'.format(dsz))
        elif typ == self.Type.Fixed64:
            if dsz != 8:
                raise ValueError('Fixed64 with invalid size = {}'.format(dsz))
        elif typ in (self.Type.String, self.Type.Bytes, self.Type.FixedLengthBytes):
            if dsz == 0:
                raise ValueError('Invalid {} with size 0'.format(typ.name))
            if (alloc == self.Allocation.callback or attr not in (self.Attribute.repeated, self.Attribute.oneof)) and szoff != 0:
                raise ValueError('Invalid {} with size offset {}'.format(typ.name, szoff))

        # Default values are only decoded by nanopb for static allocations, so
        # ignore the pointer if that's not the case and if this isn't a submessage
        # or extension
        if typ not in (self.Type.Submessage, self.Type.Extension) and alloc != self.Allocation.static:
            ptr = None

        self.tag = tag
        self.type = typ
        self.alloc = alloc
        self.attr = attr
        self.data_offset = doff
        self.data_size = dsz
        self.size_offset = szoff
        self.array_size = arsz
        self.ptr_or_def = ptr or None

    def empty(self):
        return (
            self.tag == 0 and self.type == 0 and self.alloc == 0 and
            self.attr == 0 and self.data_offset == 0 and self.data_size == 0 and
            self.size_offset == 0 and self.array_size == 0 and not self.ptr_or_def)

    def __repr__(self):
        return '{}(tag={}, type={}, alloc={}, attr={}, data_offset={}, data_size={}, size_offset={}, array_size={}, ptr={})'.format(
            type(self).__name__, self.tag, self.type.name, self.alloc.name,
            self.attr.name, self.data_offset, self.data_size, self.size_offset,
            self.array_size, self.ptr_or_def)

def decode_default_value(dio, field):
    INT_SIZES_FMT_MAP = {
        1: '<B', 2: '<H', 4: '<I', 8: '<Q',
    }
    TYPE_DECODER_MAP = {
        NanoDesc.Type.VarInt: lambda ds, s: struct.unpack(INT_SIZES_FMT_MAP[ds].lower(), s)[0],
        NanoDesc.Type.UVarInt: lambda ds, s: struct.unpack(INT_SIZES_FMT_MAP[ds].upper(), s)[0],
        NanoDesc.Type.SVarInt: lambda ds, s: struct.unpack(INT_SIZES_FMT_MAP[ds].lower(), s)[0],
        # The following two might actually be signed, or even floating point,
        # but alas we have no way of telling
        NanoDesc.Type.Fixed32: lambda ds, s: struct.unpack(INT_SIZES_FMT_MAP[ds].upper(), s)[0],
        NanoDesc.Type.Fixed64: lambda ds, s: struct.unpack(INT_SIZES_FMT_MAP[ds].upper(), s)[0],
        # Bytes have a size field tucked in front of them
        NanoDesc.Type.Bytes: lambda ds, s: s[4:],
        # Fixed length bytes aren't encoded in any special way
        NanoDesc.Type.FixedLengthBytes: lambda ds, s: s,
        # Strings are UTF-8 encoded
        NanoDesc.Type.String: lambda ds, s: s.rstrip(b'\x00').decode('utf-8'),
    }
    data = dio.read(field.data_size)
    return TYPE_DECODER_MAP[field.type](field.data_size, data)

def unpack_desc(dio, ref_base, desc):
    start = dio.tell()
    try:
        pack = NanoDesc(dio, ref_base)
    except ValueError as e:
        return start + 4
    if pack.empty():
        fields = []
    else:
        fields = [pack]
        while True:
            try:
                pack = NanoDesc(dio, ref_base)
            except ValueError as e:
                return start + 4
            if pack.empty():
                break
            if pack.tag < fields[-1].tag + 1:
                return start + 4
            fields.append(pack)

        for field in fields:
            if field.type == NanoDesc.Type.Submessage:
                sub = desc.get(field.ptr_or_def)
                if not sub:
                    dio.seek(field.ptr_or_def, os.SEEK_SET)
                    s = unpack_desc(dio, ref_base, desc)
                    try:
                        sub = desc[field.ptr_or_def]
                    except KeyError:
                        return start + 4
            elif field.type == NanoDesc.Type.Extension:
                raise NotImplementedError('Extension decoding is not supported')
            elif field.ptr_or_def:
                dio.seek(field.ptr_or_def, os.SEEK_SET)
                field.ptr_or_def = decode_default_value(dio, field)
    desc[start] = fields
    return start + ((len(fields) + 1) * NanoDesc.SIZE)

def read_descriptors(dio, ref_base):
    desc = {}
    DESCRIPTROR_START = b'\x01\x00\x00\x00'

    CHUNK_SIZE = 4 * 1024 * 1024

    offset = 0
    for chunk in iter(lambda: dio.read(CHUNK_SIZE), b''):
        start = 0
        while start < CHUNK_SIZE:
            start = chunk.find(DESCRIPTROR_START, start)
            if start < 0:
                start = CHUNK_SIZE - (len(DESCRIPTROR_START) - 1)
                break
            d = desc.get(offset + start)
            if d:
                start += (len(d) + 1) * NanoDesc.SIZE
            else:
                dio.seek(offset + start, os.SEEK_SET)
                start = unpack_desc(dio, ref_base, desc)
        offset += start
        dio.seek(offset, os.SEEK_SET)

    return desc

def dump_nanopb_descriptors(desc):
    NANO_TO_PROTO_TYPES = {
        NanoDesc.Type.VarInt: lambda f: ('int64' if f.data_size == 8 else 'bool' if field.data_size == 1 else 'int32', ' // enum?'),
        NanoDesc.Type.UVarInt: lambda f: ('uint64' if f.data_size == 8 else 'uint32', ' // enum?'),
        NanoDesc.Type.SVarInt: lambda f: ('sint64' if f.data_size == 8 else 'sint32', ''),
        NanoDesc.Type.Fixed32: lambda f: ('fixed32', ' // sfixed32? float?'),
        NanoDesc.Type.Fixed64: lambda f: ('fixed64', ' // sfixed64? double?'),
        NanoDesc.Type.Bytes: lambda f: ('bytes', ''),
        NanoDesc.Type.String: lambda f: ('string', ''),
        NanoDesc.Type.Submessage: lambda f: ('m{:x}'.format(f.ptr_or_def), ''),
        NanoDesc.Type.FixedLengthBytes: lambda f: ('bytes', ''),
    }
    NANO_TO_PROTO_ATTR = {
        NanoDesc.Attribute.required: 'required ',
        NanoDesc.Attribute.optional: 'optional ',
        NanoDesc.Attribute.repeated: 'repeated ',
        NanoDesc.Attribute.oneof: '',
    }
    print('// Automatically discovered protobuf descriptors ({} messages)\n'.format(len(desc)))
    print('syntax = "proto2";')
    for off, fields in sorted(desc.items()):
        print('\nmessage m{:x} {{'.format(off))
        lines = []
        one_of_indent = ''
        expected_field_tag = 1
        for field in fields:
            while expected_field_tag < field.tag:
                print('    {}reserved {};'.format(one_of_indent, expected_field_tag))
                expected_field_tag += 1

            if one_of_indent:
                if field.attr != NanoDesc.Attribute.oneof or field.data_offset != 0xffffffff:
                    print('    }')
                    one_of_indent = ''

            if field.attr == NanoDesc.Attribute.oneof:
                if field.data_offset == 0xffffffff:
                    if not one_of_indent:
                        raise ValueError('OneOf continuation when not inside oneof')
                elif not one_of_indent:
                    print('    oneof oneof{} {{'.format(field.tag))
                    one_of_indent = '    '

            tp, comment = NANO_TO_PROTO_TYPES[field.type](field)
            opts = []
            can_be_static = False
            if field.attr == NanoDesc.Attribute.repeated:
                if field.type not in (NanoDesc.Type.Bytes, NanoDesc.Type.String, NanoDesc.Type.Submessage, NanoDesc.Type.FixedLengthBytes):
                    opts.append('packed = true')
                if field.array_size > 0:
                    can_be_static = True
                    opts.append('(nanopb).max_count = {}'.format(field.array_size))
                    if field.size_offset == 0:
                        opts.append('(nanopb).fixed_count = true')
            elif field.type != NanoDesc.Type.Submessage and field.ptr_or_def is not None:
                opts.append('default = {}'.format(field.ptr_or_def))

            if field.alloc != NanoDesc.Allocation.static:
                opts.append('(nanopb).type = FT_{}'.format(field.alloc.name.upper()))

            if field.type in (NanoDesc.Type.VarInt, NanoDesc.Type.UVarInt, NanoDesc.Type.SVarInt):
                if tp != 'bool' and int(tp[-2:]) // 8 != field.data_size:
                    opts.append('(nanopb).int_size = {}'.format(field.data_size))
            elif field.type in (NanoDesc.Type.FixedLengthBytes, NanoDesc.Type.String):
                opts.append('(nanopb).max_size = {}'.format(field.data_size))
                if field.type == NanoDesc.Type.FixedLengthBytes:
                    opts.append('(nanopb).fixed_length = true')
            elif field.type == NanoDesc.Type.Bytes:
                opts.append('(nanopb).max_size = {}'.format(field.data_size - 4))
            if opts:
                opts = ' [{}]'.format(', '.join(opts))
            else:
                opts = ''
            print('    {}{}{} f{} = {}{};{}'.format(one_of_indent, NANO_TO_PROTO_ATTR[field.attr], tp, field.tag, field.tag, opts, comment))
            expected_field_tag += 1
        if one_of_indent:
            print('    }')
        print('}')

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('binary', type=argparse.FileType('rb'), help='Path to the extracted MAIN part from the modem firmware')
    parser.add_argument('-l', '--load_address', type=lambda s: int(s, 16), required=True, help='The load address of MAIN')
    args = parser.parse_args()

    desc = read_descriptors(args.binary, args.load_address)
    dump_nanopb_descriptors(desc)


import scrobble

from google.appengine.api import users
from google.appengine.ext import webapp

class ChangeSecretPage(webapp.RequestHandler):
    """Handle a request to change a secret for a user"""

    def post(self):
        user = users.get_current_user()
        secret = self.request.get('secret')
        if user and secret:
            scrobble.put_secret(user, secret)
            self.redirect('/user')
        else:
            self.redirect('/')


import sqlite3
from requests_html import HTMLSession
import requests
from bs4 import BeautifulSoup

conn = sqlite3.connect("webscraper.db")
c = conn.cursor()

s = HTMLSession()

number_of_articles = 100
list_of_urls = []
title = []
main_content = []
date = []

page_number_index = 1

while len(list_of_urls) < number_of_articles:
    url = f'https://www.theguardian.com/uk/london?page={page_number_index}'
    result = requests.get(url)
    doc = BeautifulSoup(result.text, "html.parser")
    articles_url = doc.find_all(class_='fc-item__container')

    for a in articles_url:
        list_of_urls.append(a.a['href'])
        date.append(" ".join(a.a['href'].rsplit('/', 4)[1:-1]))
        title.append(a.a['href'].rsplit('/', 1)[-1].replace('-', ' '))

    page_number_index += 1

#extracted text // following each urls extraceted in step 1

for news in list_of_urls:
    result = requests.get(news)
    doc = BeautifulSoup(result.text, "html.parser")
    articles = doc.find_all(id='maincontent')

    for p in articles:
        art = p.find_all("p")
        body_article = []
        for a in art:
            body_article.append(a.text)

        main_content.append(" ".join(body_article))

c.execute("""CREATE TABLE news (
            article title,
            text,
            date of article,
            source URL
    )""")

for i in range(0,number_of_articles):
    c.execute("INSERT INTO news VALUES (?, ?, ?, ?)", (title[i], main_content[i], date[i], list_of_urls[i]))
    conn.commit()

conn.close()


import socket
from struct import pack,unpack
from json import loads, dumps
from base64 import b64encode, b64decode
from binascii import hexlify,unhexlify
from cmac import CMAC
import time

"""GWMP Identifiers"""
PUSH_DATA = 0
PUSH_ACK = 1
PULL_DATA = 2
PULL_RESP = 3
PULL_ACK = 4
TX_ACK = 5

class Rxpk(object):
    """A Gateway Rxpk (upstream) JSON object.

    The root JSON object shall contain zero or more rxpk
    objects. See Gateway to Server Interface Definition
    Section 6.2.2.

    Attributes:
        tmst (int): value of the gateway time counter when the
                    frame was received (us precision).
        freq (float): Centre frequency of recieved signal (MHz).
        chan (int): Concentrator IF channel on which the frame
                    was received.
        rfch (int): Concentrator RF chain on which the frame
                    was received.
        stat (int): The result of the gateway's CRC test on the
                    frame - 1 = correct, -1 = incorrect, 0 = no test.
        modu (str): Modulation technique - "LORA" or "FSK".
        datr (str): Datarate identifier. For Lora, comprised of
                    "SFnBWm where n is the spreading factor and
                    m is the frame's bandwidth in kHz.
        codr (str): ECC code rate as "k/n" where k is carried
                    bits and n is total bits received.
        rssi (int): The measured received signal strength (dBm).
        lsnr (float): Measured signal to noise ratio (dB).
        data (str): Frame payload encoded in Base64.
        time (str): UTC time of the LoRa frame (us precision).
        size (int): Number of octects in the received frame.

    """

    def __init__(self, tmst=None, freq=None, chan=None, rfch=None,
                 stat=None, modu=None, datr=None, codr=None, rssi=None,
                 lsnr=None, data=None, time=None, size=None):
        """Rxpk initialisation method.

        """
        self.tmst = tmst
        self.freq = freq
        self.chan = chan
        self.rfch = rfch
        self.stat = stat
        self.modu = modu
        self.datr = datr
        self.codr = codr
        self.rssi = rssi
        self.lsnr = lsnr
        self.data = data
        self.time = time
        self.size = size

    @classmethod
    def decode(cls, rxp):
        """Decode Rxpk JSON dictionary.

        Args:
            rxp (dict): Dict representation of rxpk JSON object.

        Returns:
            Rxpk object if successful, None otherwise.

        """

        rkeys = rxp.keys()
        # Check mandatory fields exist
        mandatory = ('tmst', 'freq', 'chan', 'rfch',
                     'stat', 'modu', 'datr', 'codr',
                     'rssi', 'lsnr', 'data')
        if not all (rkeys for k in mandatory):
            return None
        # Mandatory attributes
        tmst = int(rxp['tmst'])
        freq = float(rxp['freq'])
        chan = int(rxp['chan'])
        rfch = int(rxp['rfch'])
        stat = int(rxp['stat'])
        modu = rxp['modu']
        datr = rxp['datr']
        codr = rxp['codr']
        rssi = int(rxp['rssi'])
        lsnr = float(rxp['lsnr'])
        data = b64decode(rxp['data'])
        # Optional attributes
        time = rxp['time'] if 'time' in rkeys else None
        size = int(rxp['size']) if 'size' in rkeys else None
        a = Rxpk(tmst=tmst, freq=freq, chan=chan, rfch=rfch, stat=stat,
                    modu=modu, datr=datr, codr=codr, rssi=rssi, lsnr=lsnr,
                    data=data, time=time, size=size)

        for m, v in a.__dict__.items():
            try:
                for attr, value in v.__dict__.items():
                    print('    '+attr, value)
            except:
                print(m, v)
                continue

        return a

class Stat(object):
    """A Gateway Stat (upstream) JSON object.

    The root JSON object shall contain zero or one stat
    objects. See Gateway to Server Interface Definition
    Section 6.2.1.

    Attributes:
        time (str): UTC time of the LoRa frame (us precision).
        lati (float): Gateway latitude in degress north of the equator.
        long (float): Gateway longitude in degress north of the equator.
        alti (int): Altitude of the gateway's position in metres above sea
                    level
        rxnb (int): Number of radio frames received since gateway start.
        rxok (int): Number of radio frames received with correct CRC since
                    gateway start.
        rwfw (int): Number of radio frames forwarded to the network server
                    since gateway start.
        ackr (int): Percentage of radio frames forwarded to the network
                    server, and acknowledged by the server since gateway
                    start.
        dwnb (int): Number of radio frames received from the network server
                    since gateway start.
        txnb (int): Number of radio frames transmitted since gateway start.

    """

    def __init__(self):
        """Stat initialisation method.

        """
        self.time = None
        self.lati = None
        self.long = None
        self.alti = None
        self.rxnb = None
        self.rxok = None
        self.rwfw = None
        self.ackr = None
        self.dwnb = None
        self.txnb = None

    @classmethod
    def decode(cls, stp):
        """Decode Stat JSON dictionary.

        Args:
            stp (dict): Dict representation of stat JSON object.

        Returns:
            Stat object.

        """

        skeys = stp['stat'].keys()
        s = Stat()

        # Set the attributes
        s.time = stp['stat']['time'] if 'time' in skeys else None
        s.lati = float(stp['stat']['lati']) if 'lati' in skeys else None
        s.long = float(stp['stat']['long']) if 'long' in skeys else None
        s.alti = int(stp['stat']['alti']) if 'alti' in skeys else None
        s.rxnb = int(stp['stat']['rxnb']) if 'rxnb' in skeys else None
        s.rxok = int(stp['stat']['rxok']) if 'rxok' in skeys else None
        s.rwfw = int(stp['stat']['rwfw']) if 'rwfw' in skeys else None
        s.ackr = int(stp['stat']['ackr']) if 'ackr' in skeys else None
        s.dwnb = int(stp['stat']['dwnb']) if 'dwnb' in skeys else None
        s.txnb = int(stp['stat']['txnb']) if 'txnb' in skeys else None

        print('STAT PACKET')
        for attr, value in s.__dict__.items():
            print(attr, value)
        return s

class GatewayMessage():
    """A Gateway Message.

    Messages sent between the LoRa gateway and the LoRa network
    server. The gateway message protocol operates over UDP and
    occupies the data area of a UDP packet. See Gateway to Server
    Interface Definition.

    Attributes:
        version (int): Protocol version - 0x01 or 0x02
        token (str): Arbitrary tracking value set by the gateway.
        id (int): Identifier - see GWMP Identifiers above.
        gatewayEUI (str): Gateway device identifier.
        payload (str): GWMP payload.
        remote (tuple): Gateway IP address and port.
        ptype (str): JSON protocol top-level object type.
    """

    def __init__(self, version=2, token=0, identifier=None,
                 gatewayEUI=None, txpk=None, remote=None,
                 ptype=None):
        """GatewayMessage initialisation method.

        Args:
            version (int): GWMP version.
            token (str): Message token.
            id: GWMP identifier.
            gatewayEUI: gateway device identifier.
            payload: GWMP payload.
            ptype (str): payload type
            remote: (host, port)

        Raises:
            TypeError: If payload argument is set to None.

        """
        self.version = version
        self.token = token
        self.id = identifier
        self.gatewayEUI = gatewayEUI
        self.payload = ''
        self.ptype = ptype
        self.remote = remote

        self.rxpk = None
        self.txpk = txpk
        self.stat = None

    @classmethod
    def decode(cls, data, remote):
        """Create a Message object from binary representation.

        Args:
            data (str): UDP packet data.
            remote (tuple): Gateway address and port.

        Returns:
            GatewayMessage object on success.

        """
        # Check length
        if len(data) < 4:
            raise DecodeError("Message too short.")
        # Decode header
        (version, token, identifer) = unpack('<BHB', data[:4])
        #print('Received Token', token)
        m = GatewayMessage(version=version, token=token, identifier=identifer)
        m.remote = remote
        # Test versions (1 or 2) and supported message types
        if ( m.version not in (1, 2) or
             m.version == 1 and m.id not in (PUSH_DATA, PULL_DATA) or
             m.version == 2 and m.id not in (PUSH_DATA, PULL_DATA, TX_ACK)
             ):
                print('Version',m.version,'ID',m.id)
                pass
                #raise UnsupportedMethod()

        # Decode gateway EUI and payload
        if m.id == PUSH_DATA:
            print('PUSH DATA')
            if len(data) < 12:
                raise DecodeError("PUSH_DATA message too short.")
            m.gatewayEUI = unpack('<Q', data[4:12])[0]
            m.payload = data[12:]
        elif m.id == PULL_DATA:
            print('PULL DATA')
            if len(data) < 12:
                raise DecodeError("PULL_DATA message too short.")
            m.gatewayEUI = unpack('<Q', data[4:12])[0]
            #print('Gateway EUI: ',m.gatewayEUI)
        elif m.id == TX_ACK:
            m.payload = data[4:]

        # Decode PUSH_DATA payload
        if m.id == PUSH_DATA:
            try:
                jdata = loads(m.payload)
            except ValueError:
                raise DecodeError("JSON payload decode error")
            m.ptype = list(jdata.keys())[0]
            # Rxpk payload - one or more.

            if  m.ptype == 'rxpk':
                m.rxpk = []

                for r in jdata['rxpk']:
                    rx = Rxpk.decode(r)
                    if rx is not None:
                        m.rxpk.append(rx)
                if not m.rxpk:
                    raise DecodeError("Rxpk payload decode error")

            # Stat payload
            elif m.ptype == 'stat':
                m.stat = Stat.decode(jdata)
                if m.stat is None:
                    raise DecodeError("Stat payload decode error")
            # Unknown payload type
            else:
                raise DecodeError("Unknown payload type")

        '''for a, v in m.__dict__.items():
            try:
                for attr, value in v.__dict__.items():
                    print('    '+attr,value)
            except:
                print(a, v)
                continue
        '''
        return m

    def encode(self):
        """Create a binary representation of message from Message object.

        Returns:
            String of packed data.

        """
        data = ''
        if self.id == PUSH_ACK:
            data = pack('<BHB', self.version, self.token, self.id)
        elif self.id == PULL_ACK:
            data = pack('<BHB', self.version, self.token, self.id)
        elif self.id == PULL_RESP:
            if self.version == 1:
                self.token = 0
            self.payload = self.txpk.encode()
            print(str(self.payload))
            data = pack('<BHB', self.version, self.token, self.id) + \
                    bytearray(self.payload,'utf-8')
        return data

def sendPullResponse(remote, request, txpk,sock):
    """"Send a PULL_RESP message to a gateway.

    The PULL_RESP message transports its payload, a JSON object,
    from the LoRa network server to the LoRa gateway. The length
    of a PULL_RESP message shall not exceed 1000 octets.

    Args:
        request (GatewayMessage): The decoded Pull Request
        txpk (Txpk): The txpk to be transported
    """
    remote = (remote[0], remote[1])

    m = GatewayMessage(version=request.version, token=request.token,
                identifier=PULL_RESP, gatewayEUI=b'9079494338994186168',
                remote=remote, ptype='txpk', txpk=txpk)
    print("Sending PULL_RESP message to %s:%d" % remote)

    sock.sendto(m.encode(), remote)
    data, address = sock.recvfrom(4096)
    print(data)
    print('received %s bytes from %s' % (len(data), address))

def sendPushAck(remote, request,sock):
    """"Send a PULL_ACK message to a gateway.

    """
    remote = (remote[0], remote[1])

    m = GatewayMessage(version=request.version, token=request.token,
                identifier=PUSH_ACK, gatewayEUI=b'9079494338994186168',
                remote=remote, ptype=None)
    print("Sending PULL_ACK message to %s:%d" % remote)

    sock.sendto(m.encode(), remote)

def sendPullAck(remote, request,sock):
    """"Send a PULL_ACK message to a gateway.

    """
    remote = (remote[0], remote[1])

    m = GatewayMessage(version=request.version, token=request.token,
                identifier=PULL_ACK, gatewayEUI=b'9079494338994186168',
                remote=remote, ptype=None)
    print("Sending PULL_ACK message to %s:%d" % remote)

    sock.sendto(m.encode(), remote)

class Txpk(object):
    """A Gateway Txpk (downstream) JSON object.

    The root JSON object shall contain zero or more txpk
    objects. See Gateway to Server Interface Definition
    Section 6.2.4.

    Attributes:
        imme (bool): If true, the gateway is commanded to
                     transmit the frame immediately
        tmst (int): If "imme" is not true and "tmst" is present,
                    the gateway is commanded to transmit the frame
                    when its internal timestamp counter equals the
                    value of "tmst".
        time (str): UTC time. The precision is one microsecond. The
                    format is ISO 8601 compact format. If "imme" is
                    false or not present and "tmst" is not present,
                    the gateway is commanded to transmit the frame at
                    this time.
        freq (float): The centre frequency on when the frame is to
                    be transmitted in units of MHz.
        rfch (int): The antenna on which the gateway is commanded
                    to transmit the frame.
        powe (int): The output power which what the gateway is
                    commanded to transmit the frame.
        modu (str): Modulation technique - "LORA" or "FSK".
        datr (str): Datarate identifier. For Lora, comprised of
                    "SFnBWm where n is the spreading factor and
                    m is the frame's bandwidth in kHz.
        codr (str): ECC code rate as "k/n" where k is carried
                    bits and n is total bits received.
        ipol (bool): If true, commands gateway to invert the
                    polarity of the transmitted bits. LoRa Server sets
                    value to true when "modu" equals "LORA", otherwise
                    the value is omitted.
        size (int): Number of octets in the received frame.
        data (str): Frame payload encoded in Base64. Padding characters
                    shall not be not added
        ncrc (bool): If not false, disable physical layer CRC generation
                    by the transmitter.
    """

    def __init__(self, imme=False, tmst=None, time=None, freq=None,
                 rfch=None, powe=None, modu=None, datr=None, codr=None,
                 ipol=None, size=None, data=None, ncrc=None):
        """Txpk initialisation method.

        """
        self.imme = imme
        self.tmst = tmst
        self.time = time
        self.freq = freq
        self.rfch = rfch
        self.powe = powe
        self.modu = modu
        self.datr = datr
        self.codr = codr
        self.ipol = ipol
        self.size = size
        self.data = data
        self.ncrc = ncrc
        self.keys = ['imme', 'tmst', 'time', 'freq', 'rfch',
                    'powe', 'modu', 'datr', 'codr', 'ipol',
                    'size', 'data', 'ncrc']
        # Base64 encode data, no padding
        if self.data is not None:
            self.size = len(self.data)
            self.data = b64encode(self.data)
            # Remove padding
            if self.data[-2:] == '==':
                self.data = self.data[:-2]
            elif self.data[-1:] == '=':
                self.data = self.data[:-1]
        else:
            self.size = 0

    def encode(self):
        """Create a JSON string from Txpk object

        """
        # Create dict from attributes. Maintain added order
        #jd = {'txpk': collections.OrderedDict()}
        jd = {'txpk':{}}

        for key in self.keys:
            val = getattr(self, key)

            if val is not None:
                if key == 'data':
                    jd['txpk'][key] = val.decode('utf-8')
                else:
                    jd['txpk'][key] = val
            #print('key',key)
            #print('valtype',type(val),val)
        #print(jd)

        return dumps(jd, separators=(',', ':'))

def intPackBytes(n, length, endian='big'):
    """Convert an integer to a packed binary string representation.

    Args:
        n (int: Integer to convert
        length (int): converted string length
        endian (st)r): endian type: 'big' or 'little'

    Returns:
        A packed binary string.
    """

    if length == 0:
        return ''
    h = '%x' % n
    # There must be a better way to do this
    s = unhexlify(str.zfill(('0'*(len(h) % 2) + h), length*2))
    if endian == 'big':
        return s
    else:
        #return s[::-1]

        return int.to_bytes(n, length, 'little')
    return n

def aesEncrypt(key, data, mode='CMAC'):
    """AES encryption function

    Args:
        key (str): packed 128 bit key
        data (str): packed plain text data
        mode (str): Optional mode specification (CMAC)

    Returns:
        Packed encrypted data string
    """
    dataorder='big'
    keyorder='big'

    if mode == 'CMAC':
        cipher = CMAC()
        # there must be a better way to do this
        key=(int.from_bytes(key[0:4], 'big'),
             int.from_bytes(key[4:8], 'big'),
             int.from_bytes(key[8:12], 'big'),
             int.from_bytes(key[12:16], 'big'))
        if len(data) <= 16:
            length=len(data)*8
            data=data+bytearray((16-len(data)))
            data=[(int.from_bytes(data[0:4], 'big'),
                   int.from_bytes(data[4:8], 'big'),
                   int.from_bytes(data[8:12], 'big'),
                   int.from_bytes(data[12:16], 'big'))]
        elif len(data) > 16:
            length = (len(data)-16)*8
            data=data+bytearray((32-len(data)))
            data=[(int.from_bytes(data[0:4], 'big'),
                   int.from_bytes(data[4:8], 'big'),
                   int.from_bytes(data[8:12], 'big'),
                   int.from_bytes(data[12:16], 'big')),
                  (int.from_bytes(data[16:20], 'big'),
                   int.from_bytes(data[20:24], 'big'),
                   int.from_bytes(data[24:28], 'big'),
                   int.from_bytes(data[28:32], 'big'))]
        else:
            print('Data greater than 32 bytes')
        mic=cipher.cmac(key, data, length)
        mic = mic[0].to_bytes(4, 'big')
        return mic

    else:
        try:
            cipher = AES.new(key,AES.MODE_ECB)
        except:
            cipher = AES(key,AES.MODE_ECB)
        return cipher.encrypt(data)

def buildJoinRequest(dev_eui,app_eui,dev_nonce,app_key):
    data=bytes([0x00])+app_eui+dev_eui+dev_nonce
    mic = aesEncrypt(intPackBytes(app_key, 16), data,
                     mode='CMAC')[0:4]
    return data+mic

APP_KEY=0x00000000000000000000000000000000
APP_EUI=0x0000000000000000
DEV_EUI=0x0000000000000000

def main():
    try:
        f = open('nonce.txt', 'r')
        DEV_NONCE = int(f.readline()) # can be used only once and it appears Helium console drops packets with repeated dev nonce
    except:
        DEV_NONCE = 0x0000
        f=open('nonce.txt', 'w')
        f.write(str(DEV_NONCE))
        f.close()

    app_key=APP_KEY
    app_eui=intPackBytes(APP_EUI,8)
    dev_eui=intPackBytes(DEV_EUI,8)
    dev_nonce=int(DEV_NONCE)
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    # Bind the socket to the port
    server_address = ('', 1680)
    print('Starting up on%s port %s' % server_address)
    sock.bind(server_address)

    while True:
        print('\n===================================================================')
        data, remote = sock.recvfrom(4096)
        print('received %s bytes from %s' % (len(data), remote))

        # Decode the data from the gateway even though we don't really care
        gm=GatewayMessage()
        message = gm.decode(data, remote)

        # build data payload for OTAA join
        packet=hexlify(buildJoinRequest(dev_eui,app_eui,intPackBytes(dev_nonce,2),app_key))

        if message.id==PULL_DATA:

            txpk = Txpk(imme=True, freq=904.3,
                                   rfch=0, powe=27,
                                   modu="LORA", datr='SF10BW125',
                                   codr="4/5", ipol=False, ncrc=False, data=unhexlify(packet))
            sendPullResponse(remote,message,txpk, sock)

            # update the nonce and write it to file
            f=open('nonce.txt', 'w')
            dev_nonce=dev_nonce+1
            f.write(str(dev_nonce))
            f.close()

        elif message.id==PUSH_DATA:
            sendPushAck(remote, message,sock)

if __name__ == "__main__":
    main()


"""JobRecommend URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from calc import views as calc_views
from zhilianjob import views as job_views

urlpatterns = [
    path('', job_views.index, name="index"),

    path('posin/', job_views.posin, name="posin"),
    path('workn/', job_views.workn, name="workn"),
    path('compn/', job_views.compn, name="compn"),
    path('sizen/', job_views.sizen, name="sizen"),
    path('attributen/', job_views.attributen, name="attributen"),
    path('fieldn/', job_views.fieldn, name="fieldn"),
    path('workedun/', job_views.workedun, name="workedun"),
    path('edusaln/', job_views.edusaln, name="edusaln"),
    path('expsaln/', job_views.expsaln, name="expsaln"),

    path('upload/', job_views.upload, name='upload'),
    path('result/', job_views.result, name='result'),

    path('test/', job_views.test, name="test"),
    path('new_add/<int:a>/<int:b>/', calc_views.add2, name='add2'),
    path('add/', calc_views.add, name='add'),

    path('admin/', admin.site.urls),
]


import torch
import os
from PIL import Image
from torch.utils.data import Dataset
import pandas as pd
import random
from PIL import Image
from torchvision import transforms

class TripletAntsDataset(Dataset):
    """
    Dataset class for the triplet ants dataset. Returns a triplet of images and the label of the
    anchor image.
    """

    def __init__(self, csv_file, root_dir, transform=None):
        """
        Arguments:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        if isinstance(csv_file, pd.DataFrame):
            self.labels = csv_file
        elif isinstance(csv_file, str):
            self.labels = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        anchor_path = os.path.join(self.root_dir,               # Directory to all ant folders
                                self.labels.iloc[idx, 0],       # Directory to specific ant folder
                                self.labels.iloc[idx, 1])       # Filename of anchor image

        anchor_image = Image.open(anchor_path)

        positive_path = anchor_path
        while positive_path == anchor_path:
            pos_img_name = random.choice(os.listdir(os.path.join(self.root_dir, self.labels.iloc[idx, 0])))
            positive_path = os.path.join(self.root_dir,
                                         self.labels.iloc[idx, 0],
                                         pos_img_name)

        positive_image = Image.open(positive_path)

        positive_dir=  os.path.join(self.root_dir,               # Directory to all ant folders
                                self.labels.iloc[idx, 0])       # Directory to specific ant folder
        negative_dir = positive_dir

        while negative_dir == positive_dir:
            negative_dir = random.choice(os.listdir(self.root_dir))

            negative_dir_path = os.path.join(self.root_dir, negative_dir)
            if os.path.isfile(negative_dir_path):
                negative_dir = positive_dir
                continue

            neg_img_name = random.choice(os.listdir(negative_dir_path))
            negative_path = os.path.join(negative_dir_path, neg_img_name)

        negative_image = Image.open(negative_path)

        if self.transform:
            anchor_image = self.transform(anchor_image)
            positive_image = self.transform(positive_image)
            negative_image = self.transform(negative_image)

        return anchor_image, positive_image, negative_image, str(self.labels.iloc[idx, 0])

class AntsDataset(Dataset):
    """
    Dataset class for the ants dataset. Returns an image and the label of the image.
    """

    def __init__(self, csv_file, root_dir, transform=None):
        """
        Arguments:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        if isinstance(csv_file, pd.DataFrame):
            self.labels = csv_file
        elif isinstance(csv_file, str):
            self.labels = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        anchor_path = os.path.join(self.root_dir,               # Directory to all ant folders
                                self.labels.iloc[idx, 0],       # Directory to specific ant folder
                                self.labels.iloc[idx, 1])       # Filename of anchor image

        anchor_image = Image.open(anchor_path)

        if self.transform:
            anchor_image = self.transform(anchor_image)

        # Get ant id number
        ant_id = torch.tensor(int(self.labels.iloc[idx, 0].split("_")[-1]))
        return anchor_image, (self.labels.iloc[idx, 0])


import torch.nn as nn
import numpy as np
import torch
from pathlib2 import Path
from .denseunet import DenseUNet
from .dataset.kit import KitDataset
from .dataset.transform import MedicalTransform
from .checkpoint import load_params
from torch.utils.data import DataLoader, SequentialSampler

class Eval:

    def __init__(self, bach_size, img_size, data_path, resume):
        self._batch_size = bach_size
        self._img_size = img_size
        self._data_path = Path(data_path)
        self._resume = Path(resume)
        self._dataset = KitDataset(data_path, stack_num=3, img_size=img_size, transform=MedicalTransform(img_size))
        self._net = DenseUNet(in_ch=self._dataset.stack_num, out_ch=self._dataset.img_channels)
        self._sampler = SequentialSampler(self._dataset)
        self._data_loader = DataLoader(self._dataset, batch_size=self._batch_size, sampler=self._sampler, num_workers=1, pin_memory=True)
        self.load_params()

    def load_params(self):

        assert self._resume.exists()
        data = {'net': self.net}
        load_params(data, self._resume)
        print(f'{" Start evaluation ":-^40s}\n')
        msg = f'Net: {self._net.__class__.__name__}\n' + \
            f'Batch size: {self._batch_size}\n'
        print(msg)
        self._net.eval()
        torch.set_grad_enabled(False)

    def __call__(self):
        for batch_idx, data in enumerate(self._data_loader):
            imgs = data['image'].cpu()
            outputs = self._net(imgs)
            predicts = outputs['output']
            predicts = predicts.argmax(dim=1)
            predicts = predicts.cpu().detach().numpy()
            data['predict'] = predicts
            yield data

    @property
    def dataset(self):
        return self._dataset

    @property
    def img_size(self):
        return self._img_size

    @property
    def net(self):
        return self._net

if __name__ == '__main__':
    e = Eval(3, 512, './dataset/data/0/imaging/', './best.pth')
    for data in e():
        print(data['predict'].shape)


# -*- coding: utf-8 -*-
import os

"""
    1. --parallel-mode：使Coverage监测被测代码子进程的覆盖率，如果被测代码是多进程的，必须使用此参数；
    2. --branch：统计分支代码覆盖率，加上这个参数可使统计更精确,
         branch=True 选项开启分支覆盖分析
    3.--include: 限定要统计代码的路径，如果不限定，Coverage会把请求涉及到的所有代码，
                 包括系统库和Tornado框架的代码都分析一遍，会大大拉低代码覆盖率；
        include 选项用来限制程序包中文件的分析范围，只对这些文件中的代码进行覆盖 检测。
"""
COV = None
if os.environ.get('FLASK_COVERAGE'):
    import coverage
    # 启动覆盖检测引擎
    COV = coverage.coverage(branch=True, include='app/*')
    COV.start()

from app import create_app, db
from app.models import User, Role
from flask_script import Manager, Shell
from flask_migrate import Migrate, MigrateCommand

env = os.getenv('FLASK_CONFIG') or 'default'
app = create_app(env)
manager = Manager(app)
migrate = Migrate(app, db)

def make_shell_context():
    return dict(app=app, db=db, User=User, Role=Role)
manager.add_command("shell", Shell(make_context=make_shell_context))
manager.add_command('db', MigrateCommand)

@manager.command
def test(coverage=False):
    """Run the unit tests."""
    if coverage and not os.environ.get('FLASK_COVERAGE'):
        import sys
        os.environ['FLASK_COVERAGE'] = '1'
        os.execvp(sys.executable, [sys.executable] + sys.argv)
    import unittest
    tests = unittest.TestLoader().discover('tests')
    unittest.TextTestRunner(verbosity=2).run(tests)
    if COV:
        COV.stop()
        COV.save()
        print('Coverage Summary:')
        COV.report()
        basedir = os.path.abspath(os.path.dirname(__file__))
        covdir = os.path.join(basedir, 'tmp/coverage')
        COV.html_report(directory=covdir)
        print('HTML version: file://%s/index.html' % covdir)
        COV.erase()

if __name__ == '__main__':
    manager.run()