# coding: utf-8

import os, subprocess
import datetime, time
import pygrib
import numpy as np
import pandas as pd
from util import StopWatch

##################################################
# MSMファイル名を取得する
##################################################
def get_msm_pall_file_name(year, month, day, hh):
    # ファイル名を設定する
    #   ex) Z__C_RJTD_20170101000000_MSM_GPV_Rjp_L-pall_FH00-15_grib2.bin
    filename = 'Z__C_RJTD_{0:04d}{1:02d}{2:02d}{3:02d}0000_MSM_GPV_Rjp_L-pall_FH00-15_grib2.bin'.format(
        year, month, day, hh)

    return filename

##################################################
# GSM指定気圧面データのファイル名を取得する
##################################################
def get_gsm_pall_file_name(year, month, day, hh):
    # ファイル名を設定する
    #   ex) Z__C_RJTD_20170102000000_GSM_GPV_Rjp_L-pall_FD0000-0312_grib2.bin
    filename = 'Z__C_RJTD_{0:04d}{1:02d}{2:02d}{3:02d}0000_GSM_GPV_Rjp_L-pall_FD0000-0312_grib2.bin'.format(
        year, month, day, hh)

    return filename

##################################################
# GSM地表データのファイル名を取得する
##################################################
def get_gsm_surf_file_name(year, month, day, hh):
    # ファイル名を設定する
    #   ex) Z__C_RJTD_20170102000000_GSM_GPV_Rjp_L-pall_FD0000-0312_grib2.bin
    filename = 'Z__C_RJTD_{0:04d}{1:02d}{2:02d}{3:02d}0000_GSM_GPV_Rjp_Lsurf_FD0000-0312_grib2.bin'.format(
        year, month, day, hh)

    return filename

##################################################
# URLを取得する
##################################################
def get_url(year, month, day, filename):

    # ディレクトリ名を設定する
    date_dir = '{0:04d}/{1:02d}/{2:02d}'.format(year, month, day)

    # URLを設定する
    url = 'http://database.rish.kyoto-u.ac.jp/arch/jmadata/data/gpv/original/{0:s}/{1:s}'.format(
        date_dir, filename)

    return url

##################################################
# MSMの初期時刻を取得する
##################################################
def get_msm_initial_hours():
    return [9, 12, 16, 18, 21, 0, 3, 6]

##################################################
# GSMの初期時刻を取得する
##################################################
def get_gsm_initial_hours():
    return [18, 0, 6, 12]

##################################################
# MSMの指定気圧面を取得する
##################################################
def get_msm_mandatory_levels():
    return [850, 700, 500]

##################################################
# GSMの指定気圧面を取得する
##################################################
def get_gsm_mandatory_levels():
    return [850, 700, 500]

##################################################
# 地表データのうち予測に使用する積算量を取得する
##################################################
def get_time_precipitation_time_ranges():
    return [3, 6, 12, 24]

##################################################
# GSMの緯度,経度の範囲
##################################################
def get_gsm_latlons():
    #   和歌山〜福島 (34,135)〜(38,141)
    #   静岡〜いわき (35,138)〜(37,141)
    #   沼津〜日立 (35,138.8)〜(36.6,140.7)
    #   和歌山〜いわき (34,135)〜(38,141)
    # lat_min, lat_max, lon_min, lon_max

    return (34, 38, 135, 141)

##################################################
# パラメータ名を日本語に変換する
##################################################
def paramet_name_to_japanese(param_name, total_precipitation_time_range=None):

    param_name_dict = {
        'Pressure reduced to MSL'       : '海面更正気圧',
        'Pressure'                      : '地上気圧',
        '10 metre U wind component'     : '東西風',
        '10 metre V wind component'     : '南北風',
        '2 metre temperature'           : '気温',
        '2 metre relative humidity'     : '相対湿度',
        'Low cloud cover'               : '下層雲量',
        'Medium cloud cover'            : '中層雲量',
        'High cloud cover'              : '上層雲量',
        'Total cloud cover'             : '全雲量',
        'Total precipitation'           : '積算降水量',
        'Geopotential height'           : '高度',
        'u-component of wind'           : '東西風',
        'v-component of wind'           : '南北風',
        'Temperature'                   : '気温',
        'Vertical velocity (pressure)'  : '上昇流',
        'Relative humidity'             : '相対湿度'
    }

    if param_name in param_name_dict:
        param_name = param_name_dict[param_name]

        # 積算降水量の場合は何時間の積算かを付与する
        if total_precipitation_time_range is not None:
            param_name = "{0:s}_{1:02d}h".format(param_name, total_precipitation_time_range)

        return param_name
    else:
        return None

##################################################
# GSM地表データの中で、指定したパラメータが重要か否かを返す
##################################################
def is_parameter_important_in_gsm_surf(param_name):

    param_important = [
        'Pressure reduced to MSL'       ,   # 海面更正気圧
        'Pressure'                      ,   # 地上気圧
        'u-component of wind'           ,   # 東西風
        'v-component of wind'           ,   # 南北風
        'Temperature'                   ,   # 気温
        'Relative humidity'             ,   # 相対湿度
        'Low cloud cover'               ,   # 下層雲量
        'Medium cloud cover'            ,   # 中層雲量
        'High cloud cover'              ,   # 上層雲量
        'Total cloud cover'             ,   # 全雲量
        'Total precipitation'           ,   # 積算降水量
    ]

    return param_name in param_important

##################################################
# GSM指定気圧面データの中で、指定したパラメータが重要か否かを返す
##################################################
def is_parameter_important_in_gsm_pall(param_name):

    param_important = [
        'Geopotential height'           ,   # 高度
        'u-component of wind'           ,   # 東西風
        'v-component of wind'           ,   # 南北風
        'Temperature'                   ,   # 気温
        'Vertical velocity (pressure)'  ,   # 上昇流
        'Relative humidity'             ,   # 相対湿度
    ]

    return param_name in param_important

##################################################
# GSMの過去データをダウンロードする
##################################################
def download_gsm_files(output_dir, start_year, start_month, start_day, days):

    # 格納先ディレクトリを用意する
    os.makedirs(output_dir, exist_ok=True)

    # 取得を開始する年月日を設定
    start_date = datetime.date(start_year, start_month, start_day)

    # 日本時間の最初の方(0時〜9時)は、UTCだと前日になるため
    # 日付を前日に変更してから処理を開始する
    start_date = start_date - datetime.timedelta(days=1)

    # 初期時刻(UTC)
    initial_hours = get_gsm_initial_hours()

    # 指定した日数分のデータをダウンロードする
    date = start_date
    for i in range(days):

        # 年月日を取得
        year = date.year
        month = date.month
        day = date.day

        for hh in initial_hours:

            # 日を跨いだらdayを1加算する
            if hh == 0:
                tmp_date = date + datetime.timedelta(days=1)
                year = tmp_date.year
                month = tmp_date.month
                day = tmp_date.day

            # ファイル名を取得する
            filenames = [
                get_gsm_pall_file_name(year, month, day, hh),
                get_gsm_surf_file_name(year, month, day, hh)
            ]

            for filename in filenames:

                # ファイルが存在しなければダウンロードする
                output_filepath = os.path.join(output_dir, filename)
                if not os.path.isfile(output_filepath):

                    # URLを設定する
                    url = get_url(year, month, day, filename)

                    # 指定したURLからダウンロードする
                    result = subprocess.run(['curl', '-O', url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=output_dir)
                    print(url)

                    # 1秒ディレイ
                    time.sleep(1)

        # 日付を更新する
        date = date + datetime.timedelta(days=1)

##################################################
# GSMの指定気圧面データをGRIB2からCSVに変換する
##################################################
def gsm_pall_grib2_to_csv(input_dir, output_dir, start_date):

    # 出力ファイル名
    output_filename = 'GSM_pall_{0:04d}_{1:02d}_{2:02d}.csv'.format(
        start_date.year, start_date.month, start_date.day)
    output_filepath = os.path.join(output_dir, output_filename)

    # ファイルが存在する場合は終了
    if os.path.isfile(output_filepath):
        return

    # 日本時間の最初の方(0時〜9時)は、UTCだと前日になるため
    # 日付を前日に変更してから処理を開始する
    date = start_date - datetime.timedelta(days=1)

    # 格納用のDataFrameを用意する
    hh_df = None

    # 処理時間計測クラス
    stop_watch = StopWatch()

    # 初期時刻ごとに処理する
    for hh in get_gsm_initial_hours():

        # DEBUG
        stop_watch.start()

        # 日を跨いだらdayを1加算する
        if hh == 0:
            date = date + datetime.timedelta(days=1)
            #year = date.year
            #month = date.month
            #day = date.day

        # 年月日を取得
        year = date.year
        month = date.month
        day = date.day

        # DEBUG
        print("==================================================")
        print('{0:04d}/{1:02d}/{2:02d} {3:02d}:00 PALL'.format(year, month, day, hh))

        # ファイル名を取得する
        filename = get_gsm_pall_file_name(year, month, day, hh)
        filepath = os.path.join(input_dir, filename)

        # GRIB2ファイルを読み込む
        grbs = pygrib.open(filepath)

        # 列名リストとndarrayを準備する
        column_names = []
        values_array = np.empty(0)

        # 指定気圧面のうち学習に用いるデータのみを取り出し、
        # values_arrayに追加する
        for grb in grbs:

            # 指定気圧面のデータを取り出す
            levels = get_gsm_mandatory_levels()
            for level in levels:

                # 指定気圧面、初期時刻のデータ以外の場合は飛ばす
                if (grb.level != level) or (grb.forecastTime != 0):
                    continue

                # 重要パラメータかを判定し、重要でないパラメータはCSVに出力しない
                if is_parameter_important_in_gsm_pall(grb.parameterName):
                    pass
                else:
                    continue

                # パラメータ名を日本語に変換する
                param_name = paramet_name_to_japanese(grb.parameterName)
                if param_name is None:
                    continue

                # DEBUG
                #print("==============================")
                #print('Level:{0:d}, Parameter name: {1:s}'.format(level, grb.parameterName))

                # 指定した(緯度,経度)に含まれる格子点のデータを抽出する
                lat_min, lat_max, lon_min, lon_max = get_gsm_latlons()
                data, latitudes, longitudes = grb.data(lat1=lat_min, lat2=lat_max, lon1=lon_min, lon2=lon_max)
                #print(data.shape, latitudes.min(), latitudes.max(), longitudes.min(), longitudes.max())

                # 物理量, 緯度, 経度を一次元化する
                values = data.reshape(-1,)
                latitudes = latitudes.reshape(-1,)
                longitudes = longitudes.reshape(-1,)

                # 列名を作成し、リストに追加する
                for i in range(latitudes.shape[0]):
                    column_name = '{0:d}hPa_lat{1:.2f}_long{2:.3f}_{3:s}'.format(
                        level, latitudes[i], longitudes[i], param_name)
                    column_names.append(column_name)

                # 物理量をndrrayに追加する
                values_array = np.append(values_array, values)

        # GRIBファイルを閉じる
        grbs.close()

        # 物理量と列名からDataFrameを作成する
        values_array = values_array.reshape(1,values_array.shape[0])
        df = pd.DataFrame(data=values_array, columns=column_names)

        # 時刻データを追加する(UTCから日本時間に変更する)
        hh = hh + 9
        if hh >= 24: hh = hh - 24
        df['時'] = hh

        # 1時刻のDataFrameを、1日分のDataFrameに追加する
        if hh_df is None:
            hh_df = df
        else:
            hh_df = hh_df.append(df, ignore_index=True)

        stop_watch.stop().print_elapsed_sec('proc time')

    # 日付のデータを追加する
    hh_df['日付'] = start_date

    # 1日分のデータをCSVファイルに出力する
    hh_df = move_datetime_column_to_top(hh_df)
    hh_df.to_csv(output_filepath)

    print(output_filepath)

##################################################
# GSMの地表データをGRIB2からCSVに変換する
##################################################
def gsm_surf_grib2_to_csv(input_dir, output_dir, start_date):

    # 出力ファイル名
    output_filename = 'GSM_surf_{0:04d}_{1:02d}_{2:02d}.csv'.format(
        start_date.year, start_date.month, start_date.day)
    output_filepath = os.path.join(output_dir, output_filename)

    # ファイルが存在する場合は終了
    if os.path.isfile(output_filepath):
        return

    # 日本時間の最初の方(0時〜9時)は、UTCだと前日になるため
    # 日付を前日に変更してから処理を開始する
    date = start_date - datetime.timedelta(days=1)

    # 格納用のDataFrameを用意する
    hh_df = None

    # 処理時間計測クラス
    stop_watch = StopWatch()

    # 初期時刻ごとに処理する
    for hh in get_gsm_initial_hours():

        # DEBUG
        stop_watch.start()

        # 日を跨いだらdayを1加算する
        if hh == 0:
            date = date + datetime.timedelta(days=1)
            #year = date.year
            #month = date.month
            #day = date.day

        # 年月日を取得
        year = date.year
        month = date.month
        day = date.day

        # DEBUG
        print("==================================================")
        print('{0:04d}/{1:02d}/{2:02d} {3:02d}:00 SURF'.format(year, month, day, hh))

        # ファイル名を取得する
        filename = get_gsm_surf_file_name(year, month, day, hh)
        filepath = os.path.join(input_dir, filename)

        # GRIB2ファイルを読み込む
        grbs = pygrib.open(filepath)

        # 列名リストとndarrayを準備する
        column_names = []
        values_array = np.empty(0)

        # 学習に用いるデータのみを取り出し、values_arrayに追加する
        for grb in grbs:

             # 初期時刻のデータ以外であれば飛ばす
            if grb.forecastTime != 0:
                 continue

            # 重要パラメータかを判定し、重要でないパラメータはCSVに出力しない
            if is_parameter_important_in_gsm_surf(grb.parameterName):
                pass
            else:
                continue

            # 積算降水量のうち、指定した時間積算量のみCSVに出力する
            total_precipitation_time_range = None
            if grb.parameterName == 'Total precipitation':
                if grb.lengthOfTimeRange in get_time_precipitation_time_ranges():
                    total_precipitation_time_range = grb.lengthOfTimeRange
                else:
                    continue

            # DEBUG
            #print("==============================")
            #print('Parameter name: {0:s}'.format(grb.parameterName))

            # パラメータ名を日本語に変換する
            param_name = paramet_name_to_japanese(grb.parameterName, total_precipitation_time_range)
            if param_name is None:
                continue

            # 指定した(緯度,経度)に含まれる格子点のデータを抽出する
            lat_min, lat_max, lon_min, lon_max = get_gsm_latlons()
            data, latitudes, longitudes = grb.data(lat1=lat_min, lat2=lat_max, lon1=lon_min, lon2=lon_max)

            # 物理量, 緯度, 経度を一次元化する
            values = data.reshape(-1,)
            latitudes = latitudes.reshape(-1,)
            longitudes = longitudes.reshape(-1,)

            # 列名を作成し、リストに追加する
            for i in range(latitudes.shape[0]):
                column_name = 'Surf_lat{0:.2f}_long{1:.3f}_{2:s}'.format(
                    latitudes[i], longitudes[i], param_name)
                column_names.append(column_name)

            # 物理量をndrrayに追加する
            values_array = np.append(values_array, values)

        # GRIBファイルを閉じる
        grbs.close()

        # 物理量と列名からDataFrameを作成する
        values_array = values_array.reshape(1,values_array.shape[0])
        df = pd.DataFrame(data=values_array, columns=column_names)

        # 時刻データを追加する(UTCから日本時間に変更する)
        hh = hh + 9
        if hh >= 24: hh = hh - 24
        df['時'] = hh

        # 1時刻のDataFrameを、1日分のDataFrameに追加する
        if hh_df is None:
            hh_df = df
        else:
            hh_df = hh_df.append(df, ignore_index=True)

        stop_watch.stop().print_elapsed_sec('proc time')

    # 日付のデータを追加する
    hh_df['日付'] = start_date

    # 1日分のデータをCSVファイルに出力する
    hh_df = move_datetime_column_to_top(hh_df)
    hh_df.to_csv(output_filepath)

    print(output_filepath)

##################################################
# GSMの指定気圧面・地表データをGRIB2からCSVに変換する
##################################################
def gsm_grib2_to_csv(input_dir, output_dir, start_year, start_month, start_day, days):

    # 格納先ディレクトリ
    output_pall_dir = os.path.join(output_dir, 'pall')
    output_surf_dir = os.path.join(output_dir, 'surf')

    # 格納先ディレクトリを用意する
    #os.makedirs(output_dir, exist_ok=True)
    os.makedirs(output_pall_dir, exist_ok=True)
    os.makedirs(output_surf_dir, exist_ok=True)

    # 取得を開始する年月日を設定
    date = datetime.date(start_year, start_month, start_day)

    # 初期時刻(UTC)
    initial_hours = get_gsm_initial_hours()

    # 指定した日数分のデータを読み込みCSCファイルに出力する
    for i in range(days):

        # GSMの指定気圧面データをGRIB2からCSVに変換する
        gsm_pall_grib2_to_csv(input_dir, output_pall_dir, date)

        # GSMの地表データをGRIB2からCSVに変換する
        gsm_surf_grib2_to_csv(input_dir, output_surf_dir, date)

        # 日付を更新する
        date = date + datetime.timedelta(days=1)

##################################################
# 日付・時刻の列を先頭に移動する
##################################################
def move_datetime_column_to_top(df):
    """ 日付・時刻の列を先頭に移動する

    Args:
        df(DataFrame) : 変換対象のDataFrame

    Returns:
        DataFrame : 変換後のDataFrame
    """
    new_columns = ['日付', '時']
    for col in df.columns:
        if (col != '日付') and (col != '時'):
            new_columns.append(col)

    new_df = df.loc[:, new_columns]

    return new_df

##################################################
# メイン
##################################################
if __name__ == '__main__':

    #
    # 1日単位でGSMのデータをCSVに出力する
    #

    # GSMデータの格納ディレクトリ
    cwd = os.getcwd()
    input_dir = os.path.join(cwd, 'input3')

    # 取得開始日付、取得する日数を設定する
    year = 2016
    month = 1
    day = 1
    days = 1

    # GSMの過去データをダウンロードする
    download_gsm_files(input_dir, year, month, day, days)

    # CSVファイルの出力先ディレクトリ
    cwd = os.getcwd()
    output_dir = os.path.join(cwd, 'input6')

    # GSMのデータをGRIB2からCSVに変換する
    gsm_grib2_to_csv(input_dir, output_dir, year, month, day, days)


import tensorflow as tf
import numpy as np
from PIL import Image
from skimage.transform import resize
from hdf5storage import loadmat, savemat
from os.path import join

from util import *
from convert_to_tree_labels import make_mapping, convert_terminal_img_to_tree_labels as convert_to_tree_labelmap

def resize_func(arr, h, w):
	arr = resize(arr, (CROP_SIZE, CROP_SIZE), preserve_range=True, order=1, anti_aliasing=False)

	if h < CROP_SIZE or w < CROP_SIZE:
		arr = arr[:min(h, CROP_SIZE), :min(w, CROP_SIZE)]

	if h > CROP_SIZE or w > CROP_SIZE:
		arr = resize(arr, (h, w), preserve_range=Tru, order=1, anti_aliasing=False)

	return arr

import sys
imset = sys.argv[1].lower().capitalize()

GRAPH_PATH = 'E:/lerner/deeplab/model_trained/deeplabv3_pascal_train_aug/frozen_inference_graph.pb'
DS_PATH = 'D:/datasets/processed/voc2012'
RGB_PATH = join(DS_PATH, 'rgb', imset)
OUTPUT_PATH = join(DS_PATH, 'Deeplab_Prediction', imset)

INPT_FMT = imset.lower()+'_%06d_rgb.jpg'
PRED_MAT_NAME = 'pred_img'
LGT_FMT = imset.lower()+'_%06d_logits.mat'
LGT_MAT_NAME = 'logits_img'

OUTPUT_TENSOR_NAME = 'ResizeBilinear_2:0'
INPUT_TENSOR_NAME = 'ImageTensor:0'
CROP_SIZE = 513
PIX_MEAN = 128

tf.reset_default_graph()

with tf.gfile.GFile(GRAPH_PATH, 'rb') as f:
	graph_def = tf.GraphDef.FromString(f.read())

with tf.Graph().as_default() as graph:
	tf.import_graph_def(graph_def, name='')

ds_info = loadmat(join(DS_PATH, 'dataset_info.mat'))

num_img = num_img_for(imset.lower())

orig_labelmap = ds_info['class_labels']
orig_labelmap[orig_labelmap.index('potted plant')] = 'pottedplant'
tree_labelmap = loadmat(join(DS_PATH, 'Decision_Tree', 'concept_tree.mat'))['all_node_labels']
mapping = make_mapping(orig_labelmap, tree_labelmap)

ckpt_graph = graph
#with tf.Session(graph=graph) as sess:
if __name__ == '__main__':
	for im_idx in range(55, num_img+1):
		print('Performing inference on image %d' % im_idx)
		sys.stdout.flush()

		inpt_im = np.array(Image.open(join(RGB_PATH, INPT_FMT % im_idx)), dtype=np.float32)
		if inpt_im.shape[2] == 4:
			inpt_im = inpt_im[:,:,:-1]
		h, w, _ = inpt_im.shape

		pad_h = max(0, CROP_SIZE-h)
		pad_w = max(0, CROP_SIZE-w)

		inpt_im = np.pad(
			inpt_im,
			[(0, pad_h), (0, pad_w), (0, 0)],
			'constant',
			constant_values=PIX_MEAN)

		inpt_im = np.expand_dims(inpt_im, axis=0)

		with tf.Session(graph=ckpt_graph) as sess:
			graph = tf.get_default_graph()
			output_tensor = graph.get_tensor_by_name(OUTPUT_TENSOR_NAME)

			output_tensor = tf.image.resize_images(output_tensor,
												[CROP_SIZE, CROP_SIZE],
												method=tf.image.ResizeMethod.BILINEAR,
												align_corners=True)
			#output_tensor.set_shape([1, CROP_SIZE, CROP_SIZE, 21])

			if h < CROP_SIZE or w < CROP_SIZE:
				output_tensor = tf.slice(output_tensor,
									[0, 0, 0, 0],
									[1, min(h, CROP_SIZE), min(w, CROP_SIZE), 21])

			if h > CROP_SIZE or w > CROP_SIZE:
				output_tensor = tf.image.resize_images(output_tensor,
												[h, w],
												method=tf.image.ResizeMethod.BILINEAR,
												align_corners=True)

			logits = sess.run(output_tensor, feed_dict={INPUT_TENSOR_NAME: inpt_im})

		logits = np.squeeze(logits)
		#softmax = resize_func(softmax, h, w).astype(np.float32)

		savemat(join(OUTPUT_PATH, LGT_FMT % im_idx),
				{LGT_MAT_NAME: logits})


"""
Implemented algorithm
Kruskal minimal spanning tree for finding the least correlated path for a set of stocks
This project can be used to building a low-correlation stock portfolio
author: Yi Rong
updated on 1/25/21
reference: https://miscj.aut.ac.ir/article_490.html
"""

import numpy as np
import pandas as pd
import datetime as dt
import pdb
import sys
import os

sys.path.append(os.path.abspath(
    "F:\\YI RONG\\BU\\academic\\CS 566 ANALYSIS OF ALGORITHMS\\Project\\CS566_Project\\TradeAssistant.py"))
import TradeAssistant as ta

class Graph_MST():

    def __init__(self, num_vertex):
    # initialize the vertex number and the graph
        self.v = num_vertex
        self.graph = []

    def addEdge(self, u, v, w):
    # intent: add edge to self.graph
    # Pre1: u and v are non-negative integers, w is a non-negative number
    # e.g. [u, v, w] = [0, 1, 0.5], in which u is node 0, v is node 1 and their edge weight is 0.5
    # Pre2: u != v
    # Post1: self.graph is a list of edges, which is defined as [u, v, w]

        self.graph.append([u, v, w])

    def findParent(self, parent, i):
    # intent: find root parent node of node i
    # Pre1: parent is a list of |V| integers and its range is among [0, len(parent))
    # e.g. parent[i] is a node index, which is the parent node of node i
    # Pre2: i is among [0, len(parent))
    # Post1: for final return_result, parent[return_result] = return_result

        # Sa: if parent of node i is not i
        # XOR
        # i is returned
        if parent[i] == i:
            return i

        # Sb: find parent of node parent[i]
        return self.findParent(parent, parent[i])

    def union(self, parent, rank, x, y):
    # intent: union two nodes through setting one as the other's parent
    # Pre1: parent is a list of integers and its range is among [0, len(parent))
    # Pre2: rank a list of integers and its range is among [0, len(parent))
    # e.g. rank[2] = 3 means the node 2 has 3 nodes as children
    # Pre3: x, y are integers among [0, len(parent)) and x != y
    # Post1: x_root becomes the parent of y_root xor y_root becomes the parent of x_root

        # Sa: find the root parent node for node x and y
        x_root = self.findParent(parent, x)
        y_root = self.findParent(parent, y)

        # Sb: compare rank[x_root] and rank[y_root] and the bigger one becomes the parent of the other one
        # XOR
        # if two ranks are equal, set y_root as the parent
        if rank[x_root] > rank[y_root]:
            parent[y_root] = x_root
            rank[x_root] += 1
        else:
            parent[x_root] = y_root
            rank[y_root] += 1

    def kruskalMST(self):
    # intent: find MST for an undirected graph using Kruskal
    # Pre1: self.graph is a list of edges and each edge is defined as [u, v, w]
    # Post1: return_result is a tree, containing (num_vertex - 1) edges
    # Post2: all vertices of self.graph are included in return_result
    # Post3: edges in return_result are sorted

        return_result = []
        parent = [i for i in range(self.v)] # Initializing parent, each vertex is the root parent of itself
        rank = [0] * self.v # Initializing rank with zeros

        i = 0  # index for each edge in self.graph
        e = 0  # index for checking how many edges invovled in result

        # Sa: all edges in self.graph is sorted based on its w
        self.graph = sorted(self.graph, key=lambda edge: edge[2])

        # Sb: there are (num_vertex - 1) edges in return_result
        # XOR
        # return_result is a forest of parts of an self.graph MST,
        # add an edge with minimal w that doesn’t create a cycle to return_result
        while e < self.v - 1:

            u, v, w = self.graph[i]
            i += 1

            u_root = self.findParent(parent, u)
            v_root = self.findParent(parent, v)

            # if there is not cycle, add the minimal w edge to return_result
            if u_root != v_root:
                e += 1
                return_result.append([u, v, w])
                self.union(parent, rank, u_root, v_root)

        # Sc: return_result is one tree with all nodes of self.graph
        return return_result

class portfolio:

    def __init__(self, L_symbol, start, end):
        self.L_symbol = L_symbol
        self.start = start
        self.end = end
        obj_ta = ta.TradeAssistant(L_symbol=L_symbol, start=start, end=end)
        L_data = obj_ta.get_raw_data()
        self.dict_raw_data = L_data[0]
        self.L_ts = L_data[1]
        self.num_symbol = len(L_symbol)

    def get_dict_rt(self):
        dict_rt = {}
        for key in self.dict_raw_data.keys():
            dict_rt[key] = self.dict_raw_data[key].pct_change()

        return dict_rt

    def get_dist_graph(self):
        df_rt = pd.DataFrame(self.get_dict_rt())
        df_corr = df_rt.corr()

        def _corr_to_dist(corr):
            return np.sqrt(2 * (1 - corr))

        df_dist = df_corr.apply(_corr_to_dist)

        return df_dist

    def get_MST(self):

        g = Graph_MST(len(self.L_ts))

        df_dist_graph = self.get_dist_graph()

        for u in range(self.num_symbol):
            for v in range(u + 1, self.num_symbol):
                g.addEdge(u, v, df_dist_graph.iloc[u, v])

        mst_result = g.kruskalMST()

        return mst_result

    def print_mst(self):
        mst = self.get_MST()
        print("The correlated stock network is listed below:")
        for u, v, dist in mst:
            print(self.L_symbol[u] + " -- " + self.L_symbol[v] + " == %.5f" % (dist))

if __name__ == "__main__":
    L_symbol = ["TSLA", "FB", "V", "JPM", "WORK", "UNH", "BABA", "MCD", "NKE"]
    port = portfolio(L_symbol=L_symbol, start=dt.datetime(2018, 1, 1), end=dt.datetime(2020, 7, 1))
    port.print_mst()


"""payday.py

This program processes the two-dimensional list containing the day
of the week and the hours worked for that day.  It recognizes that
Saturday hours are paid at time and a half and Sunday hours are
paid at double time.  It prints the total of the results.
"""

pay_rate = 27
week = [['Sunday', 2],
        ['Monday', 8.5],
        ['Tuesday', 6.75],
        ['Wednesday', 9],
        ['Thursday', 9.25],
        ['Friday', 7.75],
        ['Saturday', 7.5]]


import pandas as pd

DEFAULT_SCORE_TYPE_LIST = ['chargeScore', 'distanceScore', 'serviceScore', 'locationScore', 'roomScore','bathScore', 'equipmentScore', 'mealScore']
InputDir='../data'
InputType='cont'+len(DEFAULT_SCORE_TYPE_LIST)
OutputDir='../exp_out'+'_'+InputType
RESULT_FILE_PATH = OutputDir+'/all/'
USER_FILE_PATH=OutputDir+'/per_user/'
TRUE_PATH=OutputDir+'/truedata/'
FALSE_PATH=OutputDir+'/falsedata/'

ALL_ITEMS = pd.read_json(InputDir+"/all_items.json")


from get_web_page import get_web_page_from_disk
from bs4 import BeautifulSoup

def get_democracy_now_links():
    test_html = get_web_page_from_disk('democracynow_home_page.html')
    bs = BeautifulSoup(test_html, features='lxml')
    all_anchor_elements = bs.findAll('a')
    return [(e.get('href'), e.get_text()) for e in all_anchor_elements if
        len(e.get_text()) > 0]

if __name__ == '__main__':
    for [uri, text] in get_democracy_now_links():
        print(uri, ':', text)
    _hy_anon_var_2 = None
else:
    _hy_anon_var_2 = None


from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist, FieldError
from django.http import Http404, HttpResponseServerError, HttpResponseForbidden

def accepts_instance(qs, lookup_keywords, output_name='instance', remove_keywords=True):
    def instance_view(original_fn):
        def get_instance(request, **kwargs):
            try:
                lookup = dict((k, kwargs[k]) for k in lookup_keywords)
            except KeyError:
                if settings.DEBUG:
                    return HttpResponseServerError("Incorrect arguments (%s) for instance lookup (options were: %s)" % (lookup_keywords, kwargs.keys()))
                else:
                    return HttpResponseServerError("Incorrect arguments for instance lookup.")
            try:
                instance = qs.get(**lookup)
            except ObjectDoesNotExist:
                raise Http404("The requested resource could not be found.")
            except FieldError:
                if settings.DEBUG:
                    return HttpResponseServerError("Invalid fields specified for instance lookup (%s)." % lookup.keys())
                else:
                    return HttpResponseServerError("Unable to retrieve the requested resource.")

            new_kwargs = dict(kwargs)

            new_kwargs[output_name] = instance

            if remove_keywords:
                for k in lookup_keywords:
                    new_kwargs.pop(k)

            return original_fn(request, **new_kwargs)

        return get_instance

    return instance_view


# -*- coding: utf-8 -*-

import discord
import datetime
from discord.ext import commands
from config import settings

import httplib2
import apiclient.discovery
from oauth2client.service_account import ServiceAccountCredentials

intents = discord.Intents.default()
intents.members = True
intents.reactions = True

FILE_DATA = "DATA.txt"
FILE_LOGINS = "LOGINS.txt"

DATA = dict()
STATES = dict()
LOGINS = dict()
ALL_LOGINS = []
LOGGED_IN = set()

ADMIN_ROLE_ID = "write here admin role id"

STR_NO = '🅾'
STR_YES = '✅'
CONFIRMATIONS = ['!delete', '!lock', '!unlock']
EXCEPTION = 'Incorect input'

CREDENTIALS_FILE = 'write name of json file'

credentials = ServiceAccountCredentials.from_json_keyfile_name(CREDENTIALS_FILE, ['https://www.googleapis.com/auth/spreadsheets',
                                                                                'https://www.googleapis.com/auth/drive'])
httpAuth = credentials.authorize(httplib2.Http())
service = apiclient.discovery.build('sheets', 'v4', http = httpAuth)

spreadsheetId = 'write spreadsheet id here'

def ReadAll():
    global DATA, STATES, LOGINS
    fin = open(FILE_DATA, "r", encoding='utf-8')
    DATA = dict()
    STATES = dict()
    GroupName = ""
    for line in fin:
        if line[0] == "!":
            GroupName = line[1:-1]
            DATA[GroupName] = []
        else:
            Value, State = line[:-1].split()
            DATA[GroupName].append(Value)
            STATES[(GroupName, len(DATA[GroupName]))] = int(State)
    fin.close()
    fin = open(FILE_LOGINS, "r", encoding='utf-8')
    LOGINS = dict()
    for line in fin:
        UserId, UserLogin = line[:-1].split()
        LOGINS[int(UserId)] = UserLogin
        LOGGED_IN.add(UserLogin)
    fin.close()

def ReadLogins():
    global ALL_LOGINS
    sheet_metadata = service.spreadsheets().get(spreadsheetId=spreadsheetId).execute()
    sheets = sheet_metadata.get('sheets')
    title = sheets[0].get("properties").get("title")
    ranges = [title + "!B2:B999"]
    results = service.spreadsheets().values().batchGet(spreadsheetId = spreadsheetId,
                                         ranges = ranges,
                                         valueRenderOption = 'FORMATTED_VALUE',
                                         dateTimeRenderOption = 'FORMATTED_STRING').execute()
    sheet_values = results['valueRanges'][0]['values']
    ALL_LOGINS = []
    for elem in sheet_values:
        ALL_LOGINS.append(elem[0])

def ReadResults(SheetName, RowNumber):
    sheet_metadata = service.spreadsheets().get(spreadsheetId=spreadsheetId).execute()
    sheets = sheet_metadata.get('sheets')
    ranges = [SheetName + "!G" + RowNumber + ":" + chr(ord('G') + len(DATA[SheetName]) - 1) + RowNumber]
    results = service.spreadsheets().values().batchGet(spreadsheetId = spreadsheetId,
                                         ranges = ranges,
                                         valueRenderOption = 'FORMATTED_VALUE',
                                         dateTimeRenderOption = 'FORMATTED_STRING').execute()
    if 'values' in results['valueRanges'][0]:
        return results['valueRanges'][0]['values'][0] + [''] * (len(DATA[SheetName]) - len(results['valueRanges'][0]['values'][0]))
    else:
        return [''] * len(DATA[SheetName])

def WriteAll():
    fout = open(FILE_DATA, "w", encoding='utf-8')
    for GroupName in DATA:
        print("!" + GroupName, file=fout)
        for i in range(len(DATA[GroupName])):
            print(DATA[GroupName][i], str(STATES[(GroupName, i + 1)]), file=fout)
    fout.close()
    fout = open(FILE_LOGINS, "w", encoding='utf-8')
    for UserId in LOGINS:
        print(UserId, LOGINS[UserId], file=fout)
    fout.close()

bot = commands.Bot(command_prefix = settings['prefix'], intents = intents)
bot.remove_command('help')

@bot.event
async def on_reaction_add(reaction, user):
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(user.id).roles
    if user == bot.user or not is_admin:
        return
    if user != reaction.message.author:
        return
    emoji = reaction.emoji
    if emoji == STR_YES and reaction.message.content in CONFIRMATIONS:
        ReadAll()
        if reaction.message.content == '!delete':
            Groups = []
            for GroupName in DATA:
                for i in range(len(DATA[GroupName])):
                    del STATES[(GroupName, i + 1)]
                Groups.append(GroupName)
            for GroupName in Groups:
                del DATA[GroupName]
        elif reaction.message.content == '!lock':
            for GroupName in DATA:
                for i in range(len(DATA[GroupName])):
                    STATES[(GroupName, i + 1)] = 1
        elif reaction.message.content == '!unlock':
            for GroupName in DATA:
                for i in range(len(DATA[GroupName])):
                    STATES[(GroupName, i + 1)] = 0
        WriteAll()
        await reaction.message.delete()

@bot.command()
async def login(ctx, *args):
    ReadAll()
    if len(args) == 1:
        if ctx.message.author.id not in LOGINS:
            ReadLogins()
            if args[0] in ALL_LOGINS and args[0] not in LOGGED_IN:
                LOGINS[ctx.message.author.id] = args[0]
                await ctx.send('Successfully logged in ' + ctx.message.author.mention)
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def add(ctx, *args):
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    if len(args) == 1:
        GroupName = args[0]
        if GroupName not in DATA:
            DATA[GroupName] = []
        else:
            await ctx.send(EXCEPTION)
    elif len(args) > 1:
        GroupName = args[0]
        if GroupName in DATA:
            for i in range(1, len(args)):
                DATA[GroupName].append(args[i])
                STATES[(GroupName, len(DATA[GroupName]))] = 0
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def delete(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    if (len(args) == 0):
        await ctx.message.add_reaction(STR_YES)
        await ctx.message.add_reaction(STR_NO)
    elif len(args) == 1:
        GroupName = args[0]
        if GroupName in DATA:
            for i in range(len(DATA[GroupName])):
                del STATES[(GroupName, i + 1)]
            del DATA[GroupName]
        else:
            await ctx.send(EXCEPTION)
    elif len(args) > 1:
        GroupName = args[0]
        FLAG = True
        for i in range(1, len(args)):
            if not args[i].isdigit() or not(1 <= int(args[i]) <= len(DATA[GroupName])):
                FLAG = False
        if FLAG:
            items = sorted(list(map(int, args[1:])))[::-1]
            if GroupName in DATA:
                GroupStates = []
                for i in range(len(DATA[GroupName])):
                    GroupStates.append(STATES[(GroupName, i + 1)])
                    del STATES[(GroupName, i + 1)]
                for i in range(len(items)):
                    DATA[GroupName].pop(items[i] - 1)
                    GroupStates.pop(items[i] - 1)
                for i in range(len(DATA[GroupName])):
                    STATES[(GroupName, i + 1)] = GroupStates[i]
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def edit(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    if len(args) == 2:
        GroupName = args[0]
        NewGroupName = args[1]
        if GroupName in DATA:
            if NewGroupName not in DATA:
                DATA[NewGroupName] = []
                for i in range(len(DATA[GroupName])):
                    DATA[NewGroupName].append(DATA[GroupName][i])
                    STATES[(NewGroupName, i + 1)] = STATES[(GroupName, i + 1)]
                    del STATES[(GroupName, i + 1)]
                del DATA[GroupName]
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    elif len(args) == 3:
        GroupName = args[0]
        Index = args[1]
        Value = args[2]
        if GroupName in DATA:
            if Index.isdigit() and (1 <= int(Index) <= len(DATA[GroupName])):
                DATA[GroupName][int(Index) - 1] = Value
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def lock(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    if (len(args) == 0):
        await ctx.message.add_reaction(STR_YES)
        await ctx.message.add_reaction(STR_NO)
    elif len(args) == 1:
        GroupName = args[0]
        if GroupName in DATA:
            for i in range(len(DATA[GroupName])):
                STATES[(GroupName, i + 1)] = 1
        else:
            await ctx.send(EXCEPTION)
    elif len(args) == 2:
        GroupName = args[0]
        Index = args[1]
        if GroupName in DATA:
            if Index.isdigit() and (1 <= int(Index) <= len(DATA[GroupName])):
                STATES[(GroupName, int(Index))] = 1
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def unlock(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if (not is_admin):
        return
    if (len(args) == 0):
        await ctx.message.add_reaction(STR_YES)
        await ctx.message.add_reaction(STR_NO)
    elif len(args) == 1:
        GroupName = args[0]
        if GroupName in DATA:
            for i in range(len(DATA[GroupName])):
                STATES[(GroupName, i + 1)] = 0
        else:
            await ctx.send(EXCEPTION)
    elif len(args) == 2:
        GroupName = args[0]
        Index = args[1]
        if GroupName in DATA:
            if Index.isdigit() and (1 <= int(Index) <= len(DATA[GroupName])):
                STATES[(GroupName, int(Index))] = 0
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)
    WriteAll()

@bot.command()
async def check(ctx, *args):
    ReadAll()
    is_login = ctx.message.author.id in LOGINS
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if is_login or is_admin:
        if len(args) > 0:
            GroupName = args[0]
            if GroupName in DATA:
                if len(args) == len(DATA[GroupName]) + 1:
                    Answers = []
                    for i in range(1, len(args)):
                        if (DATA[GroupName][i - 1] == args[i]):
                            Answers.append(1)
                        else:
                            Answers.append(0)
                    Result = ''
                    if is_admin:
                        for elem in Answers:
                            if elem:
                                Result += STR_YES
                            else:
                                Result += STR_NO
                    else:
                        ReadLogins()
                        RowNumber = ''
                        for i in range(len(ALL_LOGINS)):
                            if ALL_LOGINS[i] == LOGINS[ctx.message.author.id]:
                                RowNumber = str(i + 2)
                        Results = ReadResults(GroupName, RowNumber)
                        sum_answers, blocked = 0, 0
                        for i in range(len(Answers)):
                            if not STATES[(GroupName, i + 1)] and Answers[i]:
                                sum_answers += 1
                            elif STATES[(GroupName, i + 1)]:
                                Answers[i] = Results[i]
                                blocked += 1
                            Answers[i] = str(Answers[i])
                        Result = str(sum_answers) + '/' + str(len(Answers) - blocked) + ' ' + STR_YES
                        results = service.spreadsheets().values().batchUpdate(spreadsheetId = spreadsheetId, body = {
                            "valueInputOption": "USER_ENTERED",
                            "data": [
                                {"range": GroupName + "!G" + RowNumber + ":" + chr(ord('G') + len(DATA[GroupName]) - 1) + RowNumber,
                                 "majorDimension": "ROWS",
                                 "values": [Answers]}
                            ]
                        }).execute()
                    await ctx.send(Result)
                else:
                    await ctx.send(EXCEPTION)
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send('Please, log in ' + ctx.message.author.mention)
    WriteAll()

@bot.command()
async def get(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if len(args) == 0:
        if len(DATA) > 0:
            emb = discord.Embed(colour = discord.Color.magenta())
            emb.set_footer(text = ctx.author.name, icon_url = ctx.author.avatar_url)
            for GroupName in DATA:
                Answers = ''
                if len(DATA[GroupName]) == 0:
                    Answers = 'No data'
                for i in range(len(DATA[GroupName])):
                    Answers += str(i + 1) + '. '
                    if is_admin:
                        Answers += DATA[GroupName][i] + ' '
                    if STATES[(GroupName, i + 1)]:
                        Answers += STR_NO
                    else:
                        Answers += STR_YES
                    Answers += '\n'
                emb.add_field(name = GroupName, value = Answers, inline = False)
            await ctx.send(embed = emb)
        else:
            emb = discord.Embed(title = 'No data', colour = discord.Color.dark_purple())
            emb.set_footer(text = ctx.author.name, icon_url = ctx.author.avatar_url)
            emb.set_image(url = 'https://c4.wallpaperflare.com/wallpaper/500/442/354/outrun-vaporwave-hd-wallpaper-thumb.jpg')
            await ctx.send(embed = emb)
    elif len(args) == 1:
        GroupName = args[0]
        if GroupName in DATA:
            Answers = ''
            if len(DATA[GroupName]) == 0:
                Answers = 'No data'
            for i in range(len(DATA[GroupName])):
                    Answers += str(i + 1) + '. '
                    if is_admin:
                        Answers += DATA[GroupName][i] + ' '
                    if (STATES[(GroupName, i + 1)]):
                        Answers += STR_NO
                    else:
                        Answers += STR_YES
                    Answers += '\n'
            emb = discord.Embed(title = GroupName, description = Answers, colour = discord.Color.magenta())
            emb.set_footer(text = ctx.author.name, icon_url = ctx.author.avatar_url)
            await ctx.send(embed = emb)
        else:
            await ctx.send(EXCEPTION)
    elif len(args) == 2:
        GroupName = args[0]
        Index = args[1]
        if GroupName in DATA:
            if Index.isdigit() and (1 <= int(Index) <= len(DATA[GroupName])):
                Answer = Index + '. '
                if is_admin:
                    Answer += DATA[GroupName][int(Index) - 1] + ' '
                if (STATES[(GroupName, int(Index))]):
                    Answer += STR_NO
                else:
                    Answer += STR_YES
                emb = discord.Embed(title = GroupName, description = Answer, colour = discord.Color.magenta())
                emb.set_footer(text = ctx.author.name, icon_url = ctx.author.avatar_url)
                await ctx.send(embed = emb)
            else:
                await ctx.send(EXCEPTION)
        else:
            await ctx.send(EXCEPTION)
    else:
        await ctx.send(EXCEPTION)

@bot.command()
async def help(ctx, *args):
    ReadAll()
    is_admin = bot.guilds[0].get_role(ADMIN_ROLE_ID) in bot.guilds[0].get_member(ctx.message.author.id).roles
    if is_admin:
        emb = discord.Embed(title = 'Все доступные команды бота для принимающих', colour = discord.Color.magenta())
        emb.add_field(name = ':small_blue_diamond: !add <Название группы заданий>', value = 'Добавляет новую группу заданий', inline = False)
        emb.add_field(name = ':small_blue_diamond: !add <Название группы заданий> <Ответ_1> ... <Ответ_n>', value = 'Добавляет ответы к группе', inline = False)
        emb.add_field(name = ':small_blue_diamond: !delete', value = 'Удаляет все группы заданий (требует подтверждения)', inline = False)
        emb.add_field(name = ':small_blue_diamond: !delete <Название группы заданий>', value = 'Удаляет группу заданий', inline = False)
        emb.add_field(name = ':small_blue_diamond: !delete <Название группы заданий> <Номер задания>', value = 'Удаляет задание из группы', inline = False)
        emb.add_field(name = ':small_blue_diamond: !edit <Название группы заданий> <Новое название>', value = 'Изменяет название группы', inline = False)
        emb.add_field(name = ':small_blue_diamond: !edit <Название группы заданий> <Номер задания> <Новый ответ>', value = 'Изменяет ответ на задание', inline = False)
        emb.add_field(name = ':small_blue_diamond: !lock', value = 'Блокирует все задания для сдачи (требует подтверждения)', inline = False)
        emb.add_field(name = ':small_blue_diamond: !lock <Название группы заданий>', value = 'Блокирует все задания в группе', inline = False)
        emb.add_field(name = ':small_blue_diamond: !lock <Название группы заданий> <Номер задания>', value = 'Блокирует задание', inline = False)
        emb.add_field(name = ':small_blue_diamond: !unlock', value = 'Разблокирует все задания для сдачи (требует подтверждения)', inline = False)
        emb.add_field(name = ':small_blue_diamond: !unlock <Название группы заданий>', value = 'Разблокирует все задания в группе', inline = False)
        emb.add_field(name = ':small_blue_diamond: !unlock <Название группы заданий> <Номер задания>', value = 'Разблокирует задание', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get', value = 'Выводит список всех групп и заданий в них с ответами и статусом блокировки', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get <Название группы>', value = 'Выводит список всех заданий группы с ответами и статусом блокировки', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get <Название группы> <Номер задания>', value = 'Выводит ответ и статус блокировки задания', inline = False)
        emb.add_field(name = ':small_blue_diamond: !check <Название группы> <Ответ_1> ... <Ответ_n>', value = 'Выводит результат проверки (отдельно для каждого задания)', inline = False)
    else:
        emb = discord.Embed(title = 'Все доступные команды бота для учеников', colour = discord.Color.magenta())
        emb.add_field(name = ':small_blue_diamond: !login <Ваш пароль>', value = 'Связывает ваш дискорд-аккаунт с гугл-таблицами и позволяет сдавать задания', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get', value = 'Выводит список всех групп и заданий в них со статусом блокировки', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get <Название группы>', value = 'Выводит список всех заданий группы со статусом блокировки', inline = False)
        emb.add_field(name = ':small_blue_diamond: !get <Название группы> <Номер задания>', value = 'Выводит статус блокировки задания', inline = False)
        emb.add_field(name = ':small_blue_diamond: !check <Название группы> <Ответ_1> ... <Ответ_n>', value = 'Выводит результат проверки (количество верных ответов)', inline = False)
    emb.set_footer(text = ctx.author.name, icon_url = ctx.author.avatar_url)
    emb.set_thumbnail(url = 'https://csfrager.ru/files/forums_imgs/1577119190.png')
    await ctx.send(embed = emb)

bot.run(settings['token'])


# -*- coding: utf-8 -*-

import ClassifierTemplate as ct
import pandas as pd

data = pd.read_csv("../../data/processed/train_set.csv", index_col=0)

# DataFrame containing label (!)
df = pd.DataFrame(data)

# Build Classifier object with DataFrame and column name of truth values
c = ct.Classifier(df,"productivity_binned_binary")

# drop columns not needed for Classification
c.dropColumns([
         "original_title"
        ,"adult"
        #,"belongs_to_collection"
        #,"budget"
        #,"runtime"
        #,"year"
        ,"quarter"
        ,"productivity_binned_multi"
        #,"productivity_binned_binary"
])

### scale something if needed
#c.scale([
#        "budget"
#])

### drop columns by prefix if needed
c.dropColumnByPrefix("actor_")
c.dropColumnByPrefix("director_")
#c.dropColumnByPrefix("company")
#c.dropColumnByPrefix("country")
#c.dropColumnByPrefix("genre")
c.dropColumnByPrefix("quarter_")

# lets print all non-zero columns of a movie to doublecheck
df = c.data.loc[19898]
df = df.iloc[df.nonzero()[0]]
print(df)
print(c.data.columns)

c.splitData()
#c.upsampleTrainData()
#c.downsampleTrainData()

# get parameters for GridSearch
scorer = c.f1(average="macro") # use F1 score with micro averaging
estimator = c.svc() # get svc estimator
cv = c.fold(
        k=10
        ,random_state=42
) # KStratifiedFold with random_state = 42
# parameters to iterate in GridSearch
parameters = {
    'multi_class':['ovr'],
    'class_weight':[None, 'balanced']
    #'class_weight': [{'yes':1, 'no':5}, {'yes':1, 'no':3}, {'yes':1, 'no':1}, None]
    # parameter can be used to tweak parallel computation / n = # of jobs
    #,"n_jobs":[1]
}

"""
# compute GridSearch
gs = c.gridSearch(
        estimator
        ,scorer
        ,parameters
        ,print_results=False # let verbose print the results
        ,verbose=2
        ,cv=cv
        ,onTrainSet=False
)

# print best result
c.gridSearchBestScore(gs)

# save all results in csv
c.gridSearchResults2CSV(gs,parameters,"svc_results.csv")
"""

# calculate cross validation: try samplings
estimator.set_params(
    multi_class='ovr',
    class_weight='balanced'
)
print(c.cross_validate(cv,estimator,sample=""))

"""
--------------------------- GRID SEARCH BEST SCORE ---------------------------
 Best score is 0.5771136567507072 with params {'class_weight': 'balanced', 'multi_class': 'ovr'}.
 ------------------------------------------------------------------------------
 No extra features dropped

 --------------------------- GRID SEARCH BEST SCORE ---------------------------
 Best score is 0.5786578739249424 with params {'class_weight': 'balanced', 'multi_class': 'ovr'}.
 ------------------------------------------------------------------------------
 DROPPED: ['quarter_', 'adult', 'actor_', 'director_']
"""


import json
import os
import torch
import torchvision
from PIL import Image
from torch.utils.data import Dataset, DataLoader

CUDA = torch.cuda.is_available()
DEVICE = torch.device('cuda' if CUDA else 'cpu')

HOME_PATH = os.path.abspath(os.pardir)
JSON_PATH = os.path.join(HOME_PATH, 'dataset')
TRAIN_JSON_PATH = os.path.join(JSON_PATH, 'train.json')
VAL_JSON_PATH = os.path.join(JSON_PATH, 'val.json')
TEST_JSON_PATH = os.path.join(JSON_PATH, 'test.json')
ORIGINAL_SEQ_PATH = os.path.join(HOME_PATH,
                                 r'data/faceforensics/c40/original_sequences/youtube/c40/processed_images')
MANIPULATED_SEQ_PATH = os.path.join(HOME_PATH,
                                    r'data/faceforensics/c40/manipulated_sequences/Deepfakes/c40/processed_images')
AUDIO_PATH = os.path.join(HOME_PATH, r'data/faceforensics/audio/padded_audio.npy')
AUDIO_LEN_PATH = os.path.join(HOME_PATH, r'data/faceforensics/audio/padded_audio.npy')

WHOLE_FRAME = -1
SINGLE_FRAME = 0  # used for dataloader for CNN
DEFAULT_FRAME_SIZE = 10  # num of frames per sequence in dataloader
DEFAULT_SKIP_TOLERANCE = 1  # max num of missed frames per sequence in dataloader
MIN_SEQ_LEN = 20

# class AudioDataset(Dataset):
#
#     def __init__(self, audio, audio_len):
#         self.audio = audio
#         self.audio_len = audio_len
#
#     def __len__(self):
#         return self.audio_len.shape[1]
#
#     def __getitem__(self, item):
#         raise NotImplementedError

class ImageDataset(Dataset):
    """Fake seq is labeled as 1, original seq as 0"""

    def __init__(self, folder_list, target_list, frame_size, skip_tolerance):
        """
            args:
                frame_size:     >=1: [frame_size] of frames as an utterance
                                            =0: flattened frames for CNN
                                            <0: whole video (not implemented)
        """
        self.folder_list = folder_list
        self.target_list = target_list
        self.frame_size = frame_size
        self.img_list = []
        self.label_list = []
        for i, folder in enumerate(self.folder_list):
            imgs = self.get_imgs(folder, frame_size, skip_tolerance)
            self.img_list.extend(imgs)
            self.label_list.extend([self.target_list[i]] * len(imgs))
        if self.frame_size > 0:
            print('loaded %d sets of %d-frame images ' % (
            len(self.img_list), frame_size))
        elif self.frame_size == 0:
            print('loaded %d images ' % len(self.img_list))
        else:
            print('loaded %d video sequences' % len(self.img_list))

    def __len__(self):
        return len(self.img_list)

    def __getitem__(self, index):
        imgs = self.img_list[index]
        if self.frame_size != 0:
            out = []
            for i in range(len(imgs)):
                try:
                    img = Image.open(imgs[i])
                    out.append(torchvision.transforms.ToTensor()(img))
                except:
                    print('error occurred while reading %s'%imgs[i])
            if self.frame_size > 0:
                while len(out) < self.frame_size:
                    out.append(out[-1])
            out = torch.stack(out)
        else:
            assert len(imgs) == 1
            img = Image.open(imgs[0])
            out = torchvision.transforms.ToTensor()(img)
        label = self.label_list[index]
        return out, label

    def get_imgs(self, folder, n, t):
        if n == 0:
            n = 1
        s = dict()
        for (root, dirs, files) in os.walk(folder):
            for f in files:
                try:
                    d = f.split('.')[0]
                    if d.isdigit():
                        s[int(d)] = f
                except:
                    print('can not handle',s)
                    pass
            break

        out = []
        ptr = 0
        while ptr <= min(len(s) - n, len(s) - 1):
            frames = []
            tolerance = 0
            while tolerance <= t and ptr < len(s):
                if ptr in s:
                    frames.append(os.path.join(folder, s[ptr]))
                else:
                    tolerance += 1
                ptr += 1
                if len(frames) == n or (self.frame_size < 0 and (tolerance > t or ptr == len(s))):
                    if self.frame_size>=0 or len(frames)>MIN_SEQ_LEN:
                        out.append(frames)
                    break
        return out

def collate(seq_list):
    inputs, targets = zip(*seq_list)
    len_input = torch.LongTensor(list(map(len, inputs)))
    inputs = torch.nn.utils.rnn.pad_sequence(inputs,batch_first=True)
    return inputs, targets, len_input

def get_dataset(json_dir, frame_size, skip_tolerance=DEFAULT_SKIP_TOLERANCE,
                original_dir=ORIGINAL_SEQ_PATH,
                manipulated_dir=MANIPULATED_SEQ_PATH):
    with open(json_dir, 'r') as f:
        l = json.load(f)
    folder_list = []
    target_list = []
    for f1, f2 in l:
        p0 = os.path.join(original_dir, f1)
        p1 = os.path.join(original_dir, f2)
        p2 = os.path.join(manipulated_dir, '%s_%s' % (f1, f2))
        p3 = os.path.join(manipulated_dir, '%s_%s' % (f2, f1))
        if os.path.isdir(p0):
            folder_list.append(p0)
            target_list.append(0)
        if os.path.isdir(p1):
            folder_list.append(p1)
            target_list.append(0)
        if os.path.isdir(p2):
            folder_list.append(p2)
            target_list.append(1)
        if os.path.isdir(p3):
            folder_list.append(p3)
            target_list.append(1)
    return ImageDataset(folder_list, target_list, frame_size, skip_tolerance)

if __name__ == '__main__':
    print('\ndataloader for batched frames (for training whole model)')
    train_dataset = get_dataset(TRAIN_JSON_PATH, DEFAULT_FRAME_SIZE)
    train_loader = DataLoader(train_dataset, batch_size=5, num_workers=0,
                              shuffle=True)
    for i, (d, y) in enumerate(train_loader):
        print(d.shape)
        break

    print('\ndataloader for whole videos (for training whole model)')
    train_dataset = get_dataset(TRAIN_JSON_PATH, WHOLE_FRAME)
    train_loader = DataLoader(train_dataset, batch_size=5, num_workers=0,
                              shuffle=True, collate_fn=collate)
    for i, (d, y, l) in enumerate(train_loader):
        print(d.shape)
        break

    print('\ndataloader for single frames (for training CNN)')
    train_dataset = get_dataset(TRAIN_JSON_PATH, SINGLE_FRAME)
    train_loader = DataLoader(train_dataset, batch_size=5, num_workers=0,
                              shuffle=True)
    for i, (d, y) in enumerate(train_loader):
        print(d.shape)
        break


import numpy as np
import matplotlib.pyplot as plt
import random as rand
from ImageCreator import ImageCreator
class Dataset:
    def plot_image(self, img): #Lihtsalt plottimis funktsioon kontrolliks. Tavaliselt ei kasuta.
        plt.imshow(img, cmap="gray")
        plt.show()
    def sigmoid(self, x):
        return 1/(1+np.exp(-x))
    def image(self, dchart, slice_length):
        length = dchart.shape[0]
        img = np.zeros((length, 13), dtype = 'float32')
        for i in range(length):
            ypos = int(dchart[i])+6
            if ypos<0: ypos = 0
            elif ypos>12: ypos = 12
            img[i, ypos] = 1
        return img

    def visualizer(self, chart, pos):
        size = 60
        delta = self.sigmoid(chart.delta[(pos-size+1):(pos+1)]*7000)
        rsi = chart.rsi[(pos-size+1):(pos+1)]/100
        dev25 = self.sigmoid(chart.dev25[(pos - size + 1):(pos + 1)])

        out = np.empty((size, 3))

        for i in range(size):
            out[i, 0]=delta[i]
            out[i, 1] = rsi[i]
            out[i, 2] = dev25[i]
        return self.sigmoid(out)
    def __init__(self, chart, start, stop, amount):
        self.slice_length = 100
        if(start<self.slice_length):start = self.slice_length
        if(stop+2>chart.length):stop=chart.length-1  #Igaks juhuks et mingit out of bounds paska ei tuleks
        if(amount%2==1):amount-=1 #Peab olema paaris

        #Loome listi positsioonidest graafikul mille järgi loome dataseti. List on juba balanced ehk buysid ja selle on ühepalju
        half = amount//2
        buys = np.empty(half, dtype='int32')
        sells = np.empty(half, dtype='int32')
        sellcount = 0
        while(sellcount<half):
            pos = rand.randrange(start, stop) #Võtab suvalise koha graafikul
            if(chart.direction[pos]==0):
                sells[sellcount]=pos
                sellcount+=1
        buycount = 0
        while (buycount < half):
            pos = rand.randrange(start, stop)  # Võtab suvalise koha graafikul
            if (chart.direction[pos] == 1):
                buys[buycount]=pos
                buycount += 1
        self.trades = np.concatenate([buys, sells])
        np.random.shuffle(self.trades)
        print("Created indexes")

        imgC = ImageCreator(chart)

        self.x = np.empty((amount, imgC.WIDTH, imgC.HEIGHT, 1), dtype='float32')
        self.y = np.empty((amount, 1), dtype='int32')

        for i in range(amount):
            pos = self.trades[i]
            # arr = chart.deltabid[(pos-self.slice_length+1):(pos+1)]
            self.x[i] = imgC.printImage(pos)
            self.y[i] = chart.direction[pos]


#! /usr/bin python3
import math

def isPrime(num):
    """
    Runs a quick test to verify if a number is prime, returns 1 if num is
    prime. 0 otherwise
    """
    if num <= 1:
        return 0
    if (num == 2) or (num == 3):
	    return 1
    if ((num +1) % 6 != 0) and ((num -1) % 6 != 0):
	    return 0
    lim = math.ceil(math.sqrt(num)) + 1
    for i in range(2,lim):
        if num % i == 0:
            return 0
    return 1

def isPalindrome(num):
    num = str(num)
    if num == num[::-1]:
        return 1
    return 0

def funct(a,b):
	n = 0
	counter = 0
	while (isPrime(n**2 + a*n + b)):
		n += 1
		counter += 1
	return counter

def day27():
	# define num of primes to test against
	primes = []
	for b in range(1001):
		if isPrime(b):
			primes.append(b)

	maxP = 0
	prod = 0
	for a in range(-1000,1001):
	    for b in primes:
	        c = funct(a,b)
	        if c > maxP:
	            maxP = c
	            prod = a*b
	            print(a,b,prod)

def translate(num):
	# takes a number and returns a rotation. i.e. 123 -> 231, 231 -> 312
	num = str(num)
	k = num[0]
	num.pop(0)
	num.append(k)
	return int(num)

def problem30():
	a = 0
	circular = {}
	while a < 1000000:
		if isPrime(a):
			print(a)
			if len(str(a)) == 1:
				circular[a] = 1
				#single digit numbers are circular

			#Convert number into string and test rotations

		a += 1

def problem31():
    #check if a number in dec is palindrome
    #convert to binary and repeat check
    s=0
    for i in range(1000000):
        if isPalindrome(i):
            j = str(bin(i))[2:] #binary repr starts as 0bxxyy
            if isPalindrome(j):
                s+= i
    print(s)

# Problem 37
# Truncate left & right functions

def trunc(num, dir=0):
    # takes a number, converts to string
    # if dir = 0: returns num[1:]
    # if dir = 1: returns num[:-1]
    k = str(num)
    if len(k) == 1:
        return k
    if dir == 0:
        return k[1:]
    return k[:-1]

def testP(num):
    k = num
    print(k)
    if (isPrime(k) == False):
        return 0
    if (k > 9):
        j = int(trunc(k))
        i = int(trunc(k, 1))
        return testP(i) and testP(j)
    return isPrime(num)

print(testP(3797))

def problem37():
    primes = []
    for i in range(10,1000):
        # check if the number is prime, then trunctate until false
        if isPrime(i):
            print(testP(i))

#problem37()


'''
This module contains some basic helper functions.
2019-12-20
With Yun Zheng
'''

import numpy as np
import healpy as hp
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = (10.0, 8.0)

##    Only for 1-field map
def plot_ps(maps):
    _nside = int(np.sqrt(len(maps)/12))
    cls = hp.anafast(maps, lmax = 2*_nside);
    _ell = np.arange(len(cls)); _ell2 = _ell[2:]
    plt.loglog(_ell2, _ell2*(_ell2+1)/2/np.pi*cls[2:])

def deconv(maps, beam_in, beam_out, lmax):
    '''
    Beam in unit of arc-miniute.
    This function changes the value of the input map itself.

    '''

    _maps = np.copy(maps)
    for j in range(1,3): ### only for Q\U;
        _maps[j] = hp.sphtfunc.decovlving(_maps[j], fwhm = beam_in/60/180*np.pi, lmax = lmax, verbose = False)
        _maps[j] = hp.smoothing(_maps[j], fwhm = beam_out/60/180*np.pi, lmax = lmax, verbose = False)
    return _maps

def smooth(maps, beam_out, lmax):
    _maps = np.copy(maps);
    for j in range(1,3):                        ###for Q and U. Exclude I.
        _maps[j] = hp.smoothing(_maps[j], fwhm = beam_out/60/180*np.pi, lmax = lmax, verbose = False)
    return _maps

def bin_l(cl, L, Q, l2 = False, bin_scheme = None):
    '''

    cl, L, Q(bin_number); no np.mean

    '''
    ### some problmes exit when Q = L !!! 2020-03-14
    if len(cl.shape) > 2 :
        bin_averages = np.zeros((Q, cl.shape[1], cl.shape[1]))
    else:
        bin_averages = np.zeros(Q)
    if l2:
        for l in range(L):
            cl[l] = l*(l+1)/2/np.pi*(cl[l])
    for q in range(Q):
        if q == 0:
            bin_averages[q] = sum(cl[2:((q+1)*L//Q)]/(L//Q))   ### l = 0,1 should be abandoned; SHOULDN'T use np.mean(), becaues it need to specify AXIS... . But for different shape, the axis is different .
        else:
            bin_averages[q] = sum(cl[q*L//Q:((q+1)*L//Q)]/(L//Q))
    return bin_averages

def get_ell(L, Q):
    ''' L, Q(bin_number) '''
    Ell = np.ones(Q)
    for i in range(0,int(Q)):
        Ell[i] = (2*i+1)*L//Q/2
    return Ell

def Power_spectrum(maps,R, lmax):
    global cl
    '''
    revised for upper-triangle of the matrix.
    Input:
    maps with multi-frequencies IQU sky maps.
    Galactic plane cut for calculating the power spectrum.
    lmax.

    Return: EE and BB cross power spectrum.
    Shape (2, lmax, nf, nf)
    '''
    n_f = len(maps)
    cl = np.ones((2, n_f*n_f, lmax +1)); Cl = np.zeros((2, lmax+1, n_f, n_f))
    k = 0
    for i in range(n_f):
        for j in range(n_f):

            if i >= j :
                cross_ps = hp.anafast(maps[i], maps[j], lmax = lmax, gal_cut=R, nspec=3) ## TT, EE, BB
            else:
                cross_ps = np.zeros((3, lmax+1))
            cl[0][k] = cross_ps[1]; cl[1][k] = cross_ps[2]  ## calculate the E and B_mode power spectrum
            k += 1

    for l in range(lmax+1):
        Cl[0, l, : , :] = cl[0, :,l].reshape(n_f, n_f); Cl[1, l, : , :] = cl[1, :,l].reshape(n_f, n_f)
        Cl[0, l] += Cl[0, l].T - np.diag(Cl[0, l].diagonal()) ; Cl[1, l] += Cl[1, l].T - np.diag(Cl[1, l].diagonal())
    return Cl

def m_l(lmax, l):
    '''
    Return the m of corresponding l.

    '''
    m_id = np.ones(l+1, dtype = np.int)
    for i in range(l + 1):
        m_id[i] = hp.sphtfunc.Alm.getidx(lmax, l , i)
    return (m_id)

ali_ma = hp.read_map('/smc/jianyao/Ali_maps/ali_mask_wo_edge.fits')#, verbose=False)
def Mask(maps):

    '''
    masked value = hp.unseen()
    '''
    maps_ma = hp.ma(maps)
    maps_ma.mask = np.logical_not(ali_ma)
    return maps_ma

def Mask_0(maps_raw):

    '''
    The masked values are equal to 0.

    '''
    maps = np.copy(maps_raw)

    index0 = np.arange(len(ali_ma));
    mask_index0 = index0[np.where(ali_ma<1)]

    _ndim = len(maps.shape)
    if _ndim > 2:  ### (Nf, 3, npix)
        for i in range(maps.shape[0]):
            for j in range(3):
                maps[i,j][mask_index0] = 0
    elif _ndim == 2: ### (3, npix)
        for j in range(maps.shape[0]):
            maps[j][mask_index0] = 0

    else: ### (npix)
        maps[mask_index0] = 0
    return maps

def l2(ell):
    '''
    get the l^2/np.pi
    '''

    return ell*(ell+1)/2/np.pi

def Select_fre(ps_in, sel):

        '''
        Take some part of the cross power spectrum matrix.
        ps_in : (Q, Nf, Nf)

        '''
        # sel = np.array((1,2,3))
        n_fre = len(sel); lbin = len(ps_in)
        ps_out = np.ones((lbin, n_fre, n_fre)); ### selected power spectra

        for q in range(lbin):
            x = 0;
            for i in (sel):
                y = 0;
                for j in (sel):
                    ps_out[q][x,y] = ps_in[q][i, j];
                    y += 1;
                x += 1;

        return ps_out


from bs4 import BeautifulSoup
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from last_angel.items import ScrapedChapter, ScrapedChapterLoader
import json
import logging
import os
import re
import requests
import scrapy

class LastAngelSpider(scrapy.Spider):
    """Parses the last angel forum to retrieve all messages pertaining to the story and saves the dict to a file"""

    name = "last_angel_spider"

    # Add additional threads (if more are created) here
    start_urls = [
        "https://forums.spacebattles.com/threads/the-last-angel.244209/reader/",
        "https://forums.spacebattles.com/threads/the-last-angel-ascension.346640/reader/",
        "https://forums.spacebattles.com/threads/the-last-angel-the-hungry-stars.868549/reader/",
    ]

    # a regular expression for finding the book name from the threadmarks
    book_name_re_dict = {
        "the-last-angel": None,
        "the-angels-fire": re.compile(r"The\s*Angel's\s*Fire\s*", re.IGNORECASE),
        "the-last-angel-ascension": None,
        "buried-in-the-past": re.compile(r"Buried\s*in\s*the\s*Past$", re.IGNORECASE),
        "entomology": re.compile(r"Entomology", re.IGNORECASE),
        "quiet": re.compile(r"Quiet\s*\(full\)", re.IGNORECASE),
        "stillness": re.compile(r"Stillness", re.IGNORECASE),
        "story-time": re.compile(r"Story\s*Time", re.IGNORECASE),
        "the-final-line": re.compile(r"The\s*Final\s*Line", re.IGNORECASE),
        "uneasy-lie-the-heads": re.compile(r"Uneasy\s*Lie\s*the\s*Heads", re.IGNORECASE),
        "predator-prey": re.compile(r"Predator,\s*(Prey|Predator)", re.IGNORECASE),
        "names-of-the-demons": re.compile(r"Names\s*of\s*the\s*Demon[:,]", re.IGNORECASE),
        "awakening": re.compile(r"Awakening,\s*Chapter", re.IGNORECASE),
        "hungry-stars": None,
        "sirens-song": re.compile(r"Siren's\s*Song,", re.IGNORECASE)
    }

    # a simple exclusion regular expression to capture anything that tends to fall through
    exclude_threadmarks = [
        re.compile(r"(map\s*of\s*galhemna\s*system)", re.IGNORECASE),
    ]

    # TODO: Change force_run for deploy
    def __init__(self, log_level: int = logging.INFO, force_run: bool = True, **kwargs):
        super().__init__(**kwargs)

        logging.getLogger("scrapy").setLevel(log_level)
        logging.getLogger("urllib3.connectionpool").setLevel(log_level)
        self.logger.setLevel(log_level)
        self.force_run = force_run

    def start_requests(self):
        # create the json file which saves line counts to check for changes
        toc_line_length_file = os.path.join(get_project_settings().get("DATA_SAVE_DIR"), "toc_line_length.json")
        os.makedirs(os.path.dirname(toc_line_length_file), exist_ok=True)
        if not os.path.isfile(toc_line_length_file):
            json.dump({}, open(toc_line_length_file, "w"), indent=4)

        # load from the json file a dict containing the line lengths of urls to check against their current line length
        toc_line_length_dict = json.load(open(toc_line_length_file))

        # check each link against the dict to see if we should run it or not
        for link in self.start_urls:
            self.logger.debug(f"Check Url: {link}")

            # finds how many urls are in the table of contents page
            link_length = len(BeautifulSoup(requests.get(link).content, "html.parser")
                              .find("article", {"class", "hasThreadmark"})
                              .find("div", {"class": "bbWrapper"})
                              .find_all("a"))
            self.logger.debug(f"Link Length ({link})={link_length}")

            # the url count is used to determine if a new chapter was added
            if toc_line_length_dict.get(link, 0) < link_length or self.force_run:
                self.logger.info(f"Start Url: {link}")
                yield scrapy.Request(link, callback=self.parse, errback=self.errback, dont_filter=True)
            toc_line_length_dict[link] = link_length

        # save dict to file for future reference
        json.dump(toc_line_length_dict, open(toc_line_length_file, "w"))

    def parse(self, response, **kwargs):
        """ The first parser takes a link and sends it off to be parsed by
        parse_contents_href and parse_table_contents_page"""
        self.logger.debug(f"Extracting links for starter url: {response.request.url}")
        toc_post_id = re.search(r"post-(?P<id>(\d*))", response.css("article.hasThreadmark")
                                .css("[data-content]").get(), re.IGNORECASE).group("id")
        base_thread = re.search(r"https://forums.spacebattles.com/threads/(?P<base_thread>[^.]*)", response.request.url,
                                re.IGNORECASE).group("base_thread")
        message_lines = response.css("article.hasThreadmark").css("div.bbWrapper").extract()[0].split("\n")
        link_re = re.compile(r"post[s]?[-/](?P<post_id>\d+)[/]?$", re.IGNORECASE)
        meta_dict = {"toc_post_id": toc_post_id,
                     "base_thread": base_thread,
                     "link_re": link_re,
                     "message_lines": message_lines}

        # double parses each url to ensure all posts are captured, even those listed in the same link
        yield response.follow(response.request.url, callback=self.parse_table_contents_page, errback=self.errback,
                              meta=meta_dict, dont_filter=True)
        yield response.follow(response.request.url, callback=self.parse_contents_href, errback=self.errback,
                              meta=meta_dict, dont_filter=True)

    def parse_contents_href(self, response):
        """ Uses the links found in the table of contents post to reveal more posts to follow"""
        for tag in BeautifulSoup("\n".join(response.meta["message_lines"]), "html.parser").find_all("a"):
            # looks for multiple chapters in one link (Chapter 13 and Chapter 14) but split by a newline and splits them
            for content_name in tag.get_text().strip().split("\n"):
                post_id_search = response.meta["link_re"].search(tag["href"])
                if post_id_search:
                    self.logger.debug(
                        f"Following href {post_id_search.group('post_id')} {content_name} {tag['href']}")
                    yield response.follow(tag["href"], callback=self.parse_post, errback=self.errback, dont_filter=True,
                                          meta={"post_id": post_id_search.group("post_id"),
                                                "content_name": content_name,
                                                "base_thread": response.meta["base_thread"],
                                                "source": "parse_contents_href"})

    def parse_table_contents_page(self, response):
        """
        Gets the content of the table of contents post (first posts of a thread)
        :return: dict of
            :content_name: the posts threadmark
            :post_id: the unique id of the post
            :book_name: the name of the thread it belongs to, extracted from the url
            :url: the url used to find the post
            :raw_html: the html data of the div.bbWrapper that held the post
        """
        # checks for any chapters embedded within the table of contents page
        for post in response.css("article.hasThreadmark"):
            post_id = re.search(r"post-(?P<id>(\d*))", post.css("[data-content]").get(), re.IGNORECASE).group("id")

            # ensures the following code only applies to the table of contents post
            if post_id == response.meta["toc_post_id"]:
                toc_threadmark = str(post.css("div.message-cell--threadmark-header").css("span::text").get()).strip()

                # checks for multiple chapters listed in table of contents post (table of contents and prologue)
                for chapter in toc_threadmark.split("and"):
                    # ensures the table of contents specifically isn't counted as a chapter
                    if not re.search(r"table\s*of\s*content(s?)\s*", chapter.strip(), re.IGNORECASE):
                        chapter_l = ScrapedChapterLoader(item=ScrapedChapter(), response=response)
                        chapter_l.add_value("content_name", chapter.strip())
                        chapter_l.add_value("post_id", post_id)
                        chapter_l.add_value("book_name", self.get_book_name(chapter, response.meta["base_thread"]))
                        chapter_l.add_value("url", response.request.url)
                        chapter_l.add_value("raw_html", post.css("div.bbWrapper").extract()[0].split("\n"))
                        chapter_item = chapter_l.load_item()
                        self.logger.debug(f"Extracted ChapterItem Source={response.meta.get('source')} {chapter_item}")
                        yield chapter_item

    def parse_post(self, response):
        """
        Gets the content of a specific post
        :return: dict of
            :content_name: the posts threadmark
            :post_id: the unique id of the post
            :book_name: the name of the thread it belongs to, extracted from the url
            :url: the url used to find the post
            :raw_html: the html data of the div.bbWrapper that held the post
        """
        for post in response.css("article.hasThreadmark"):
            # a specific post is listed in the meta of the response but more posts appear on the same page

            post_id = re.search(r"post-(?P<id>(\d*))", post.css("[data-content]").get(), re.IGNORECASE).group("id")
            if post_id == response.meta["post_id"]:
                threadmark = post.css("div.message-cell--threadmark-header").css("span.threadmarkLabel::text").get()
                book_name = self.get_book_name(threadmark, response.meta["base_thread"])

                # ensures a book name is present
                if book_name:
                    chapter_l = ScrapedChapterLoader(item=ScrapedChapter(), response=response)
                    chapter_l.add_value("content_name", response.meta["content_name"])
                    chapter_l.add_value("post_id", post_id)
                    chapter_l.add_value("book_name", book_name)
                    chapter_l.add_value("url", response.request.url)
                    chapter_l.add_value("raw_html", post.css("div.bbWrapper").extract()[0].split("\n"))
                    chapter_item = chapter_l.load_item()
                    self.logger.debug(f"Extracted ChapterItem Source={response.meta.get('source')} {chapter_item}")
                    return chapter_l.load_item()

    def get_book_name(self, threadmark: str, default: str = None):
        """ Checks the regex dict for a matching type of regex and uses it to find the book name from the link """
        for exclude_re in self.exclude_threadmarks:
            if exclude_re.search(threadmark):
                return None
        for key, val in self.book_name_re_dict.items():
            if val:
                search = val.search(threadmark)
                if search:
                    return key
        return default

    def errback(self, failure):
        """ Logs errors """
        self.logger.error(repr(failure))

if __name__ == "__main__":
    # TODO: Figure out how to do this from a different file
    process = CrawlerProcess(get_project_settings())
    process.crawl("last_angel_spider")
    process.start()


import os
import sys
import random

if __name__ == "__main__":
    git_username = sys.argv[1]
    repository = sys.argv[2]
    release = sys.argv[3]

    #sec = random.randrange(1, 10)
    #os.system('sleep {}'.format(sec))

    fetch_command = 'python3 fetch_release.py {} {} {}'.format(git_username, repository, release)
    res = os.system(fetch_command)

    if res == 0:
        fetch_command = 'python3 auto_test.py {} {} {}'.format(git_username, repository, release)
        os.system(fetch_command)


count = 0
def left_factoring(s):
    v = s.split('->')
    lhs= v[0].strip()
    rhs = [x.strip() for x in v[1].strip().split('|')]
    factor(lhs, rhs)
def factor(lhs, rhs):
    l=[]
    visited,variables,beta = [0 for i in rhs],{},[]
    for i in range(len(rhs)):
        if visited[i] == 0:
            flag,visited[i],rule = 0,1,rhs[i]
            for j in range(i+1, len(rhs)):
                if visited[j] == 0 and (rule[0] == rhs[j][0]):
                    visited[j] = flag=1
                    variables[rule[0]] = variables.get(rule[0], []) + [rhs[j][1:]]
            if flag == 0:
                beta.append(rule)
            else:
                variables[rule[0]] = variables.get(rule[0], []) + [rule[1:]]
    global count
    for key in variables.keys():
        count += 1
        lhs_new = lhs + '^'*(count - lhs.count('^'))
        print(lhs, '->', key + lhs_new)
        factor(lhs_new, variables[key])
    for rem in beta:
        print(lhs, '->', rem)
left_factoring('A -> aB | aY |aZ| id')
left_factoring('B->aB|aC|cD|cZ|e')
"""
aqid98@aqid98-VPCEB34EN:~/Desktop/Practical_Exam/SPCC/LEFT_FACTORING$ python3 lf.py
A -> aA^
A^ -> Y
A^ -> Z
A^ -> B
A -> id
B -> aB^^
B^^ -> C
B^^ -> B
B -> cB^^^
B^^^ -> Z
B^^^ -> D
B -> e
aqid98@aqid98-VPCEB34EN:~/Desktop/Practical_Exam/SPCC/LEFT_FACTORING$
"""

System Programming And Compiler Construction/Exp_4B/lf.py


# encoding: utf-8
import csv
import numbers
from six import text_type

def _stringify(s, encoding, errors):
    if s is None:
        return ''
    if isinstance(s, text_type):
        return s.encode(encoding, errors)
    elif isinstance(s, numbers.Number):
        pass  # let csv.QUOTE_NONNUMERIC do its thing.
    elif not isinstance(s, str):
        s = str(s)
    return s

def _stringify_list(l, encoding, errors='strict'):
    try:
        return [_stringify(s, encoding, errors) for s in iter(l)]
    except TypeError as e:
        raise csv.Error(str(e))

class UnicodeWriter(object):
    """
    >>> import cunicodecsv
    >>> from cStringIO import StringIO
    >>> f = StringIO()
    >>> w = cunicodecsv.writer(f, encoding='utf-8')
    >>> w.writerow((u'é', u'ñ'))
    >>> f.seek(0)
    >>> r = cunicodecsv.reader(f, encoding='utf-8')
    >>> row = r.next()
    >>> row[0] == u'é'
    True
    >>> row[1] == u'ñ'
    True
    """
    def __init__(self, f, dialect=csv.excel, encoding='utf-8', errors='strict', *args, **kwds):
        self.encoding = encoding
        self.writer = csv.writer(f, dialect, *args, **kwds)
        self.encoding_errors = errors

    def writerow(self, row):
        self.writer.writerow(_stringify_list(row, self.encoding, self.encoding_errors))

    def writerows(self, rows):
        for row in rows:
          self.writerow(row)

    @property
    def dialect(self):
        return self.writer.dialect

class UnicodeReader(object):
    def __init__(self, f, dialect=None, encoding='utf-8', errors='strict', **kwds):
        format_params = ['delimiter',
                         'doublequote',
                         'escapechar',
                         'lineterminator',
                         'quotechar',
                         'quoting',
                         'skipinitialspace']
        if dialect is None:
            if not any([kwd_name in format_params for kwd_name in kwds.keys()]):
                dialect = csv.excel
        self.reader = csv.reader(f, dialect, **kwds)
        self.encoding = encoding
        self.encoding_errors = errors
        self._parse_numerics = bool(
            self.dialect.quoting & csv.QUOTE_NONNUMERIC
        )

    def next(self):
        row = self.reader.next()
        encoding = self.encoding
        encoding_errors = self.encoding_errors
        unicode_ = unicode
        if self._parse_numerics:
            float_ = float
            return [value if isinstance(value, float_) else
                    unicode_(value, encoding, encoding_errors)
                    for value in row]
        else:
            return [unicode_(value, encoding, encoding_errors)
                    for value in row]

    def __iter__(self):
        return self

    @property
    def dialect(self):
        return self.reader.dialect

    @property
    def line_num(self):
        return self.reader.line_num


import sys
from dataclasses import dataclass
from getpass import getpass
from typing import Optional, Iterable, Dict

@dataclass
class Field:
    name: str
    label: Optional[str]
    value: str
    sensitive: bool
    required: bool
    default: str

    @staticmethod
    def build(name: str, label: Optional[str] = None, sensitive: Optional[bool] = False, required: Optional[bool] = True, default: Optional[str] = None):
        return Field(
            name=name,
            label=label or name,
            value=None,
            sensitive=sensitive,
            required=required,
            default=default,
        )

class Form:
    def __init__(self, fields: Iterable[Field], max_retry_count: int = 3):
        self.__max_retry_count = max_retry_count
        self.__fields: Iterable[Field] = fields

    def prompt(self):
        for field in self.__fields:
            prompt_user_for = getpass if field.sensitive else input

            remaining_retry_count = self.__max_retry_count
            value: Optional[str] = None

            while not value and field.required:
                if remaining_retry_count <= 0:
                    sys.stderr.write('<<< ERROR: You still have not given the valid input. Self-terminated for now '
                                     'but you may rerun this command later.\n')
                    raise IOError('Invalid input for ' + field.label)
                if remaining_retry_count < self.__max_retry_count:
                    sys.stderr.write('<<< WARNING: The input is invalid. Please try again.\n')
                remaining_retry_count -= 1
                value = prompt_user_for(f'>>> {field.label}: ')

            field.value = value or None

        return self

    def to_dict(self) -> Dict[str, str]:
        return {
            field.name: field.value
            for field in self.__fields
        }


from util.logging import Logging
import graph.directed_graph.kosaraju_helper as kh
from graph.directed_graph.viz_tracing import VizTracing

""" Helper module for DirectedGraph class
"""

def create_sccs_kosaraju_dfs(directed_graph, nontrivial):
    """ Function that creates a list of strongly connected components according to
    Kosaraju's algorithm (https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm) with a
    depth-first-search approach.

    Args:
        directed_graph (DirectedGraph): The directed graph for which the SCCS should be calculated
        nontrivial(bool): If True, only nontrivial sccs will be returned, otherwise all sccs

    Returns:
        list(set()) of SCCs: Each SCC is a set of vertices

    """

    Logging.log("\nStarting")
    stack = []
    sccs_trivial, visited = list(), dict()
    for vertex in directed_graph.get_vertices().keys():
        if visited.get(vertex) is None:
            Logging.log("Vertex {0} not visited, go deep", vertex)
            kh.fill_order_dfd_sccs(directed_graph, vertex, visited, stack)
        else:
            Logging.log("Vertex {0} already visited, skipping", vertex)

    reversed_graph = get_reversed_graph(directed_graph)

    visited = dict()
    for i in reversed(stack):
        if visited.get(i) is None:
            sccs_trivial.append(set())
            kh.visit_dfs_sccs(reversed_graph, i, visited, sccs_trivial[-1])

    if nontrivial:
        return filter_nontrivial(sccs_trivial, directed_graph)
    else:
        return sccs_trivial

def filter_nontrivial(sccs_trivial, directed_graph):
    """ This function filters out the trivial sccs

    A scc is nontrivial, iff there are at least two vertices in it,
    or there is only one vertex with a self-loop. A self-loop means
    that the indegree and the outdegree are both 1 and the tail is equal
    to the head

    Args:
        sccs_trivial(list): The list of trivial sccs
        directed_graph(DirectedGraph): The directed graph

    Returns:
        sccs_nontrivial(list): The list of nontrivial sccs

    """

    sccs_non_trivial = list()
    for scc in sccs_trivial:
        vertex = directed_graph.get_vertex(list(scc)[0])
        if (len(scc) >= 2) or \
            (len(scc) == 1 and vertex.get_indegree() == 1 and
            vertex.get_outdegree() == 1) and list(vertex.get_heads())[0] == list(scc)[0]:
            sccs_non_trivial.append(scc)

    return sccs_non_trivial

def get_reversed_graph(directed_graph):
    """ Function that returns the reverse of this graph

    Args:
        directed_graph (DirectedGraph): The directed graph

    Returns:
        DirectedGraph: The reversed graph

    """

    reversed = directed_graph.__class__()
    for i in directed_graph.get_vertices().keys():
        reversed.add_vertex(i)

    for i in directed_graph.get_vertices().keys():
        vertex = directed_graph.get_vertex(i)
        for j in vertex.get_heads():
            reversed.add_edge(j.get_label(), i)

    return reversed

def is_cyclic_dfs(directed_graph, vertex, traversed_already, in_cycle):
    """ Function that recursively searches the directed graph depth first and checks
    if a vertex was already in_cycle before.

    It checks all vertices that have not been traversed before. The heads of those
    vertices are followed. If in that traversal, a vertex is found that is present in
    the dict "in_cycle" with a value of true, then a cycle is present

    Args:
        directed_graph (DirectedGraph): The directed graph
        vertex(Vertex): The current vertex
        traversed_already (dict): A dictionary that maintains whether vertices have been
            traversed already. It's a performance measure put in place in order to shortcut
            processing if a vertex was already processed by another subtree
        in_cycle (list): A list that, if a vertex has been found to part be part of cycle,
            for that vertex, has a value of true. If that vertex is not part of a cycle, it's
            value is false

    Returns:
        bool: True if the vertex was in_cycle before, False otherwise

    """

    traversed_already[vertex.get_label()] = True
    in_cycle[vertex.get_label()] = True
    viztrace_visit_tail(directed_graph, vertex)

    for head in vertex.get_heads():
        if traversed_already.get(head.get_label()) is None:
            if is_cyclic_dfs(directed_graph, head, traversed_already, in_cycle):
                viztrace_cycle_reported(directed_graph, head)
                return True
            else:
                viztrace_vertex(directed_graph, vertex)
        elif in_cycle[head.get_label()]:
            viztrace_cycle_found(directed_graph, vertex, head)
            return True

    in_cycle[vertex.get_label()] = False
    return False

def is_cyclic(directed_graph):
    """ Function that checks whether a directed graph contains a cycle or not

    Args:
        directed_graph (DirectedGraph): The directed graph

    Returns:
        bool: True if the directed graph contains a cycle, otherwise False

    """

    Logging.log("\nStarting cycle check")
    traversed_already = dict()
    in_cycle = {i:False for i in directed_graph.get_vertices().keys()}
    for label, vertex in directed_graph.get_vertices().items():
        if traversed_already.get(label) is None:
            if is_cyclic_dfs(directed_graph, vertex, traversed_already, in_cycle):
                return True

    return False

def viztrace_cycle_reported(directed_graph, vertex):
    """ Function that is used along the way back from the origin
    of the cycle detection to the initial state. Along the way,
    all vertices are tagged with the state in_cycle

    Args:
        directed_graph (DirectedGraph): The directed graph
        vertex: the vertex that should get the status activated

    """
    VizTracing.set_status(directed_graph, vertex, VizTracing.IN_CYCLE)
    VizTracing.change_activated_vertex(directed_graph, vertex)
    VizTracing.snapshot()

def viztrace_visit_tail(directed_graph, vertex):
    """ Function that is used to tag vertices with the state "visisted",
    if these vertices have been visited once. So next time, when another predecessor
    of a tagged vertex is being considered, it is skipped

    Args:
        directed_graph (DirectedGraph): The directed graph
        vertex: the vertex that should get the status "visited"

    """
    VizTracing.change_activated_vertex(directed_graph, vertex)
    VizTracing.set_status(directed_graph, vertex, VizTracing.VISISTED)
    VizTracing.snapshot()

def viztrace_cycle_found(directed_graph, tail, head):
    """ Changes the state of a vertex when the vertex is part of a cycle

    Args:
        directed_graph (DirectedGraph): The directed graph
        tail: the tail vertex that should get the status activated
        head: the head vertex that should get the in_cycle status

    """

    VizTracing.set_status(directed_graph, head, VizTracing.IN_CYCLE)
    VizTracing.change_activated_vertex(directed_graph, head)
    VizTracing.snapshot()

def viztrace_vertex(directed_graph, vertex):
    """ Changes focus to the vertex and takes a snapshot

    Args:
        directed_graph (DirectedGraph): The directed graph
        vertex: the vertex that should get the status activated

    """

    VizTracing.change_activated_vertex(directed_graph, vertex)
    VizTracing.snapshot()