"""
Let's try to visualize what we predict...
"""

from tqdm import tqdm
import os
import h5py
import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
import cv2
import tifffile as tiff
from pylab import *

if __name__ == '__main__':
    data_path = '../data'
    model_name = 'resnet50'

    result_folder = os.path.join(data_path, 'predictions', model_name, 'visualize')

    try:
        os.mkdir(result_folder)
    except:
        pass

    num_folds = 10
    num_classes = 17

    r_val_prediction_aug = []

    r_val_labels = []

    for fold in tqdm(range(num_folds)):
        f = h5py.File(os.path.join(data_path, 'predictions', model_name, 'val_pred_{fold}.hdf5'.format(fold=fold)))

        val_prediction_aug = np.array(f['val_prediction_aug'])
        r_val_prediction_aug += [val_prediction_aug]

        r_val_labels += [pd.read_csv('../data/fold{fold}/val.csv'.format(fold=fold)).sort_values(by='path')]

        f.close()

    val_true = pd.concat(r_val_labels)
    y_true = val_true.drop('path', 1)

    val_true = val_true.set_index('path')

    val_pred_aug = np.vstack(r_val_prediction_aug)

    print('log_loss_aug = ', log_loss(y_true.values.ravel(), val_pred_aug.ravel(), eps=1e-7))

    for i, path in tqdm(enumerate(val_true.index)):
        file_name = path.split('/')[-1]

        jpg = cv2.imread(path)
        jpg = cv2.cvtColor(jpg, cv2.COLOR_BGR2RGB)

        tiff_path = path.replace('train-jpg', 'train-tif-v2').replace('.jpg', '.tif')
        tf = tiff.imread(tiff_path)

        objects = val_true.columns
        y_pos = np.arange(len(objects))
        performance = val_true.loc[path].values

        fig, ax = plt.subplots(2, 5, figsize=(30, 20))

        G = tf[:, :, 1].astype(np.float64)
        NIR = tf[:, :, 3].astype(np.float64)

        NDVI = 1.0 * (G - NIR) / (G + NIR)

        ax[0, 1].set_title('R', fontsize=30)
        ax[0, 1].imshow(jpg[:, :, 0])
        ax[0, 2].set_title('G', fontsize=30)
        ax[0, 2].imshow(jpg[:, :, 1])
        ax[0, 3].set_title('B', fontsize=30)
        ax[0, 3].imshow(jpg[:, :, 2])
        ax[0, 4].set_title('RGB', fontsize=30)
        ax[0, 4].imshow(jpg)

        ax[1, 1].imshow(tf[:, :, 2])
        ax[1, 1].set_title('R', fontsize=30)
        ax[1, 2].imshow(tf[:, :, 1])
        ax[1, 2].set_title('G', fontsize=30)
        ax[1, 3].imshow(tf[:, :, 0])
        ax[1, 3].set_title('B', fontsize=30)

        ax[1, 4].set_title('NIR', fontsize=30)
        ax[1, 4].imshow(tf[:, :, 3])

        ax[1, 0].set_title('NDVI', fontsize=30)
        ax[1, 0].imshow(NDVI)

        ax[0, 0].barh(y_pos, performance, align='center', alpha=0.5, color='blue')
        ax[0, 0].barh(y_pos, val_pred_aug[i], align='center', alpha=0.5, color='red')

        ax[0, 0].set_yticks(y_pos)
        ax[0, 0].set_yticklabels(objects, fontsize=30)

        plt.tight_layout()

        savefig(os.path.join(result_folder, file_name))
        close()


with open('input.txt') as f:
    lines = f.read().strip().splitlines()
lines.sort()

from collections import defaultdict
import re
import numpy
guard_log = defaultdict(numpy.zeros(60).copy)

current_guard = None
sleep_start = None

for line in lines:
    time, event = line.split('] ')
    minute = int(time[-2:])
    if '#' in event:
        gid = int(re.findall('\d+', event)[0])
        current_guard = guard_log[gid]
    elif event == "falls asleep":
        sleep_start = minute
    elif event == "wakes up":
        current_guard[sleep_start:minute] += 1

def max_on_vals(d, key=None):
    def new_key(x):
        if key is None:
            return d[x]
        else:
            return key(d[x])
    return max(d, key=new_key)

print("Part 1")
gid = max_on_vals(guard_log, key=numpy.sum)
minute = numpy.argmax(guard_log[gid])
print(gid*minute)

print("Part 2")
gid = max_on_vals(guard_log, key=numpy.max)
minute = numpy.argmax(guard_log[gid])
print(gid*minute)


'''
输入：a为给定实数，n为自然数
输出：a^n

       a^(n/2)*a^(n/2)                 n 为偶数
       a^((n-1)/2)*a^((n-1)/2)*a       n 为奇数

传统上，在正文中至少含有两个递归调用的例程叫做分治算法，
（最大子序列和的O(n*logn)解法、线性时间的树的遍历，归并排序，快速排序）
而正文中只含有一个递归调用例程不是分治算法

mypower()并不能算作分治策略，因为只进行了一次递归调用

'''

def powerWithUnsignedExponent(num, absn):
    if absn == 0:
        return 1
    if absn == 1:
        return num
    if absn % 2 == 0:
        return mypower(num * num, absn // 2)
    else:
        return mypower(num * num, (absn - 1) // 2) * num

def mypower(num, n):
    if num == 0 and n < 0:
        print("no....")
        return None
    absn = abs(n)
    if n < 0:
        return 1.0 / powerWithUnsignedExponent(num, absn)
    else:
        return powerWithUnsignedExponent(num, absn)

for i in range(11):
    print(mypower(2, i))
print(mypower(2, -3))
print(mypower(2, -2))
print(mypower(2, -1))


import warnings

import numpy as np
import pandas as pd

from scipy.stats import skew
from scipy.special import boxcox1p

from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC, Ridge
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
from sklearn.svm import SVR
from mlxtend.regressor import StackingRegressor
from sklearn.model_selection import GridSearchCV

warnings.filterwarnings('ignore')

def loadData():
    """
    产生数据集
    :return:
    """
    data_train = pd.read_csv('./data/train.csv')
    data_test = pd.read_csv('./data/test.csv')

    return data_train, data_test

def deleteOutlier(train):
    """
    删除训练集中的异常点
    :param train:
    :return:
    """
    train = train.drop(train[(train['GrLivArea'] > 4000) & (train['SalePrice'] < 300000)].index)

    return train

def imputingMissingData(dataSet):
    """
    对整个数据集进行缺失值的补全
    :param dataSet: 测试集和训练集的组合
    :return:
    """
    dataSet['PoolQC'] = dataSet['PoolQC'].fillna('None')
    dataSet['MiscFeature'] = dataSet['MiscFeature'].fillna('None')
    dataSet['Alley'] = dataSet['Alley'].fillna('None')
    dataSet['Fence'] = dataSet['Fence'].fillna('None')
    dataSet['FireplaceQu'] = dataSet['FireplaceQu'].fillna('None')

    # 根据Neighborhood特征将LotFrontage进行分类，然后针对每个类别进行去中位数去补全缺失值
    dataSet['LotFrontage'] = dataSet.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))

    for col in ('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
        dataSet[col] = dataSet[col].fillna('None')

    for col in ('GarageArea', 'GarageCars'):
        dataSet[col] = dataSet[col].fillna(0)

    dataSet['GarageYrBlt'] = dataSet['GarageYrBlt'].fillna(method='ffill')

    for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
        dataSet[col] = dataSet[col].fillna('None')

    for col in ('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
        dataSet[col] = dataSet[col].fillna(0)

    dataSet['MasVnrType'] = dataSet['MasVnrType'].fillna('None')
    dataSet['MasVnrArea'] = dataSet['MasVnrArea'].fillna(0)

    dataSet.drop(['Utilities'], axis=1, inplace=True)  # 该特征在测试集中的取值一样

    # 缺失值很少的值，而且也不是像前面的特征中那样是因为不存在而为Nan，类似这种的都可以用众数来补全
    dataSet['MSZoning'] = dataSet['MSZoning'].fillna(dataSet['MSZoning'].mode()[0])

    dataSet['Functional'] = dataSet['Functional'].fillna(dataSet['Functional'].mode()[0])

    dataSet['Electrical'] = dataSet['Electrical'].fillna(dataSet['Electrical'].mode()[0])

    dataSet['KitchenQual'] = dataSet['KitchenQual'].fillna(dataSet['KitchenQual'].mode()[0])

    dataSet['Exterior1st'] = dataSet['Exterior1st'].fillna(dataSet['Exterior1st'].mode()[0])
    dataSet['Exterior2nd'] = dataSet['Exterior2nd'].fillna(dataSet['Exterior2nd'].mode()[0])

    dataSet['SaleType'] = dataSet['SaleType'].fillna(dataSet['SaleType'].mode()[0])

    return dataSet

def transformStr(dataSet):
    """
    将一些类别型的特征中的数字值转换成字符串
    :param dataSet:
    :return:
    """
    dataSet['MSSubClass'] = dataSet['MSSubClass'].apply(str)
    dataSet['OverallCond'] = dataSet['OverallCond'].apply(str)
    dataSet['YrSold'] = dataSet['YrSold'].apply(str)
    dataSet['MoSold'] = dataSet['MoSold'].apply(str)

    return dataSet

def transformSortNum(dataSet):
    """
    将类别型特征中的值转换成序列型数字, 例如["jiang", "zhang", "jiang"] -> [0, 1, 0]
    :param dataSet:
    :return:
    """
    cols = ('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
            'ExterQual', 'ExterCond', 'HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
            'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
            'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
            'YrSold', 'MoSold')
    for col in cols:
        lbl = LabelEncoder()
        lbl.fit(list(dataSet[col].values))
        dataSet[col] = lbl.transform(list(dataSet[col].values))

    return dataSet

def extractFeat(dataSet):
    """
    从其他特征中提取新特征，将地下室和第一层，第二层的面积加在一起，构成一个新的特征
    :param dataSet:
    :return:
    """
    dataSet['TotalSF'] = dataSet['TotalBsmtSF'] + dataSet['1stFlrSF'] + dataSet['2ndFlrSF']

    return dataSet

def boxCoxFeat(dataSet):
    """
    skew大于0.75的将数值型特征都进行box-cox变换成服从正态分布的特征
    :param dataSet:
    :return:
    """
    numericalFeat = dataSet.dtypes[dataSet.dtypes != 'object'].index

    skewed_feats = dataSet[numericalFeat].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)

    skewness = pd.DataFrame({'Skew': skewed_feats})

    skewness = skewness[abs(skewness) > 0.75]  # 将斜度的绝对值大于0.75的特征进行BoxCox转换

    skewedFeatures = skewness.index
    lam = 0.15
    for feature in skewedFeatures:
        dataSet[feature] = boxcox1p(dataSet[feature], lam)

    return dataSet

def dummy(dataSet):
    """
    将一些类别型的特征转换成稀疏的数值型特征
    :param dataSet:
    :return:
    """
    dataSet = pd.get_dummies(dataSet)

    return dataSet

def train(trainX, trainY, model):
    """
    训练模型
    :param trainX:
    :param trainY:
    :param model:
    :return:
    """
    # trainData, validateData, trainLabels, validateLabels = train_test_split(trainX, trainY, test_size=0.2, random_state=0)
    # model.fit(trainData, trainLabels)
    #
    # scores = model.score(validateData, validateLabels)
    model.fit(trainX, trainY)
    scores = model.score(trainX, trainY)

    return scores, model

def test(testX, testId, model):
    """
    测试模型
    :param testX:
    :param testId:
    :param model:
    :return:
    """
    predictions = model.predict(testX)

    result = pd.DataFrame({'Id': testId.as_matrix(), 'SalePrice': np.exp(predictions).astype(np.float32)})

    result.to_csv('./data/rfg_prediction.csv', index=False)

def main():

    data_train, data_test = loadData()

    data_train = deleteOutlier(data_train)
    trainX = data_train.drop(['Id', 'SalePrice'], axis=1)
    trainY = data_train['SalePrice']
    trainY = np.log1p(trainY)
    ntrain = len(trainX)

    testId = data_test['Id']
    testX = data_test.drop(['Id'], axis=1)

    dataSet = pd.concat([trainX, testX], axis=0)

    dataSet = imputingMissingData(dataSet)
    dataSet = transformStr(dataSet)
    dataSet = transformSortNum(dataSet)
    dataSet = extractFeat(dataSet)
    dataSet = boxCoxFeat(dataSet)
    dataSet = dummy(dataSet)

    trainX = dataSet[:ntrain]
    testX = dataSet[ntrain:]

    models = {
        # "lgd": lgb.LGBMRegressor(objective='regression', num_leaves=5,
        #                          learning_rate=0.05, n_estimators=720,
        #                          max_bin=55, bagging_fraction=0.8,
        #                          bagging_freq=5, feature_fraction=0.2319,
        #                          feature_fraction_seed=9, bagging_seed=9,
        #                          min_data_in_leaf=6, min_sum_hessian_in_leaf=11),

        'gbdt': GradientBoostingRegressor(n_estimators=2000, learning_rate=0.01,
                                          max_depth=4, max_features='sqrt',
                                          min_samples_leaf=15, min_samples_split=12,
                                          loss='huber', random_state=0),
        "ridge": Ridge(alpha=0.05, max_iter=100),

        'krr': KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5),

        'enet': ElasticNet(alpha=0.0005, l1_ratio=1, random_state=3),

        'lasso': Lasso(alpha=0.0008, max_iter=100, random_state=1),

    }
    stackReg = StackingRegressor(regressors=[models['gbdt'], models['krr'], models['enet']], meta_regressor=models['lasso'])
    modelStack1 = StackingAverageModels(base_models=(models['ridge'], models['krr'], models['enet']),
                                        meta_model=models['lasso'])

    modelStack2 = StackingAverageModels(base_models=(models['gbdt'], models['ridge'], models['krr'], models['lasso']),
                                        meta_model=models['enet'])

    modelStack3 = StackingAverageModels(base_models=(models['gbdt'], models['lasso'], models['enet']),
                                        meta_model=models['krr'])

    modelStack4 = StackingAverageModels(base_models=(models['lasso'], models['krr'], models['enet']),
                                        meta_model=models['gbdt'])

    averageModel = AveragingModels((modelStack1, modelStack2, modelStack3, modelStack4))
    # for key in models:
    #     scores, reg = train(trainX, trainY, models[key])
    #
    #     print('model: {}   scores: {}'.format(key, scores))

    # parameters = {"n_estimators": [500], "min_samples_split": [12, 15], "min_samples_leaf": [12, 15],
    #                 "max_depth": [4, 6], "random_state": [0]}
    # clf = GridSearchCV(models['gbdt'], parameters)

    scores, reg = train(trainX.values, trainY.values, modelStack2)
    test(testX.values, testId, reg)
    print(scores)

# stacking models
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
    """
    简单的对多个模型进行融合，对每个模型的预测结果去平均值，从而达到更好的预测结果
    """

    def __init__(self, models):
        self.models = models

    def fit(self, X, y):
        # clone方法能对模型对象进行深拷贝， 因此下面实际上是对我们的模型集合进行了深度拷贝
        self.models_ = [clone(x) for x in self.models]

        for model in self.models_:
            """
            对每个模型进行训练，训练模型没有返回值
            """
            model.fit(X, y)
        return self

    def predict(self, X):
        """
        用fit函数训练号的模型来调用该方法，对测试集进行预测
        :param X:
        :return:
        """
        # 获得的predictions应该是每行为一个模型的预测数据
        predictions = np.column_stack([
            model.predict(X) for model in self.models_
        ])
        # 将所有模型预测的结果取一个平均值，对于回归的处理方式
        return np.mean(predictions, axis=1)

class StackingAverageModels(BaseEstimator, RegressorMixin, TransformerMixin):
    """
    自定义stacking模型类，用于堆砌其他的模型成为一个新的模型，目的在于集中每个模型的有点
    """
    def __init__(self, base_models: tuple, meta_model, n_folds=10):
        self.base_models = base_models  # 用于对训练集和测试集输出新的特征的模型
        self.meta_model = meta_model  # 用于对提取出的特征做预测过
        self.n_folds = n_folds

    def fit(self, X, y):
        """
        在此类中总共有两层的训练
        :param X:
        :param y:
        :return:
        """
        self.base_models_ = [[] for x in self.base_models]  # 用来存储所有训练过的模型，并将同类模型存入到一个列表中
        self.meta_model_ = clone(self.meta_model)
        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=0)

        # 创建一个数组用于存储每个模型的对训练集进行交叉预测的结果
        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
        # 依次取出每一个模型对数据进行预测
        for i, model in enumerate(self.base_models):
            # 将训练集进行进行分割
            for train_index, holdout_index in kfold.split(X, y):
                # 因为每个模型都会进行5次训练，而且每次训练的训练集都不一样，因此每次都对未fit的模型克隆一份来进行操作
                instance = clone(model)
                instance.fit(X[train_index], y[train_index])
                self.base_models_[i].append(instance)
                y_pred = instance.predict(X[holdout_index])
                out_of_fold_predictions[holdout_index, i] = y_pred

        # 将之前测试的结果作为新的特征用作训练集，训练集的输出值不变
        self.meta_model_.fit(out_of_fold_predictions, y)
        return self

    def predict(self, X):
        """
        对测试集进行测试，先用slef.base_models_中保存的模型对测试集进行预测，
        将同一类模型的预测值取平均值生成一个新的特征，然后用所有不同模型生成的特征作为新的输入值
        :param X:
        :return:
        """
        meta_feature = np.column_stack([
            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)
            for base_models in self.base_models_
        ])

        return self.meta_model_.predict(meta_feature)

main()


# Written by Nicholas Gibson on 2/1/18

import pygame
import numpy as np
import serial
import time

# Initialise motor Control message
motControl = "+000+000+000+000"

# Initialise serial port
ser = serial.Serial('/dev/ttyACM0', 9600) # For the boat
time.sleep(0.5)

# Define some colors
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)

# This is a simple class that will help us print to the screen
# It has nothing to do with the joysticks, just outputting the
# information.
class TextPrint:
    def __init__(self):
        self.reset()
        self.font = pygame.font.Font(None, 20)

    def Print(self, screen, textString):
        textBitmap = self.font.render(textString, True, BLACK)
        screen.blit(textBitmap, [self.x, self.y])
        self.y += self.line_height

    def reset(self):
        self.x = 10
        self.y = 10
        self.line_height = 15

    def indent(self):
        self.x += 10

    def unindent(self):
        self.x -= 10

    def blankline(self):
        self.y += self.line_height

pygame.init()

# Set the width and height of the screen [width,height]
size = [500, 300]
screen = pygame.display.set_mode(size)

pygame.display.set_caption("My Game")

# Loop until the user clicks the close button.
done = False

# Used to manage how fast the screen updates
clock = pygame.time.Clock()

# Initialize the joysticks
pygame.joystick.init()

# Get ready to print
textPrint = TextPrint()

# -------- Main Program Loop -----------
while done == False:
    # EVENT PROCESSING STEP
    for event in pygame.event.get():  # User did something
        if event.type == pygame.QUIT:  # If user clicked close
            done = True  # Flag that we are done so we exit this loop

        # Possible joystick actions: JOYAXISMOTION JOYBALLMOTION JOYBUTTONDOWN JOYBUTTONUP JOYHATMOTION
        if event.type == pygame.JOYBUTTONDOWN:
            print("Joystick button pressed.")
        if event.type == pygame.JOYBUTTONUP:
            print("Joystick button released.")

    # DRAWING STEP
    # First, clear the screen to white. Don't put other drawing commands
    # above this, or they will be erased with this command.
    screen.fill(WHITE)
    textPrint.reset()

    # Get count of joysticks
    joystick_count = pygame.joystick.get_count()

    #textPrint.Print(screen, "Number of joysticks: {}".format(joystick_count))
    #textPrint.indent()

    joystick = pygame.joystick.Joystick(0)
    joystick.init()

    textPrint.Print(screen, "RoboBoat Wireless Controller: Manual Mode".format(0))
    name = joystick.get_name()
    textPrint.Print(screen, "Joystick name: {}".format(name))

    textPrint.indent()

    axisLY = joystick.get_axis(1)
    axisRY = joystick.get_axis(4)

    textPrint.Print(screen, "Axis LY value: {:>6.3f}".format(axisLY))
    textPrint.Print(screen, "Axis RY value: {:>6.3f}".format(axisRY))
    textPrint.blankline()

    buttonB = joystick.get_button(1)
    textPrint.Print(screen, "Emergency Stop: {}".format(buttonB))

    textPrint.blankline()

    rightMot = int(axisLY*-255)
    leftMot = int(axisRY*-255)

    textPrint.Print(screen, "Motor 1: {}".format(leftMot))

    textPrint.Print(screen, "Motor 2: {}".format(leftMot))

    textPrint.Print(screen, "Motor 3: {}".format(rightMot))

    textPrint.Print(screen, "Motor 4: {}".format(rightMot))

    # Making motor message

    """
    # Left Motor sign
    if np.sign(leftMot) == -1:
        motControl = '-' + motControl[1:16]
        motControl = motControl[0:4] + '-' + motControl[5:16]
    else:
        motControl = '+' + motControl[1:16]
        motControl = motControl[0:4] + '+' + motControl[5:16]

    # Right Motor sign
    if np.sign(rightMot) == -1:
        motControl = motControl[0:8] + '-' + motControl[9:16]
        motControl = motControl[0:12] + '-' + motControl[13:16]
    else:
        motControl = motControl[0:8] + '+' + motControl[9:16]
        motControl = motControl[0:12] + '+' + motControl[13:16]

    # Left Motor value

    if leftMot > 255:
        leftMot = 255
    elif leftMot < -255:
        leftMot = -255
    """

    if leftMot > 99:
        leftMot = '+' + str(int(leftMot))
    elif leftMot > 9:
        leftMot = '+0' + str(int(leftMot))
    elif leftMot > -1:
        leftMot = '+00' + str(int(leftMot))
    elif leftMot > -10:
        leftMot = '-00' + str(int(abs(leftMot)))
    elif leftMot > -100:
        leftMot = '-0' + str(int(abs(leftMot)))
    elif leftMot > -256:
        leftMot = str(int(leftMot))

    # Right mot value
    if rightMot > 99:
        rightMot = '+' + str(int(rightMot))
    elif rightMot > 9:
        rightMot = '+0' + str(int(rightMot))
    elif rightMot > -1:
        rightMot = '+00' + str(int(rightMot))
    elif rightMot > -10:
        rightMot = '-00' + str(int(abs(rightMot)))
    elif rightMot > -100:
        rightMot = '-0' + str(int(abs(rightMot)))
    elif rightMot > -256:
        rightMot = str(int(rightMot))

    motControl = '<' + leftMot + leftMot + rightMot + rightMot + '>'

    textPrint.blankline()
    textPrint.Print(screen, "{}".format(motControl))
    ser.write(motControl)

    # Display Screen
    pygame.display.flip()

    # Limit to 20 frames per second
    clock.tick(20)

# Close the window and quit.
pygame.quit()


#!/usr/bin/env python3

from bottle import route
import bottle
import tempfile
import os
import shutil
import subprocess
import json
import hashlib

def is_known_format(fmt):
    return fmt in ['std', 'export', 'simple', 'inline']

def get_out_filename(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    f =  os.path.join(outdir, get_out_filename_rel(name, fmt))
    if not os.path.abspath(f).startswith(os.path.abspath(outdir)):
        raise Exception('Path out problem', os.path.abspath(f), os.path.abspath(outdir))
    return f

def get_out_filename_rel(name, fmt = 'std'):
    if not is_known_format(fmt): raise Exception('Unknown format')
    return "{}.{}.html".format(name, fmt)

def get_in_filename(name):
    f = os.path.join(indir, get_in_filename_rel(name))
    if not os.path.abspath(f).startswith(os.path.abspath(indir)):
        raise Exception('Path in problem', os.path.abspath(f), os.path.abspath(indir))
    for file_ending in [""] + file_endings:
        if os.path.exists(f + file_ending):
            return f + file_ending
    return f

def get_in_filename_rel(name):
    return name

def has_compile_file_ending(f):
    for file_ending in file_endings:
        if f.endswith(file_ending):
            return True
    return False

@route('/view/<name:path>')
def route_view(name):
    infile = get_in_filename(name)

    if os.path.exists(infile) and has_compile_file_ending(infile):
        fmt = bottle.request.query.fmt or "std"
        if not is_known_format(fmt): return 'Unknown format'

        compile_document(name, fmt)
        return bottle.static_file(get_out_filename_rel(name, fmt), outdir)

    else:
        return bottle.static_file(get_in_filename_rel(name), indir)

@route('/refresh/<name:path>')
def route_refresh(name):
    if needs_update(name):
        return "True"
    return "False"

@route('/generated/<name:path>')
def route_generated(name):
    return bottle.static_file(name, os.path.join(tempdir, "generated"))

@route('/')
def route_index():
    text = ""

    text += "<title>Panserver Index</title>"
    text += ''.format(style_basic + style_index)
    text += ""
    text += '
Panserver
'
    text += 'Serving Markdown documents rendered using pandoc. By Marcel Fischer'

    def dir_entry(dirname, toplevel = False):
        #collect markdown files recursively into a list
        #return '' if no markdown file is in the directory
        dirtext = ''
        print("dirname ", dirname)
        for name in sorted(os.listdir(os.path.join('.', dirname))):
            path = os.path.join(dirname, name)
            if os.path.isdir(path): continue
            if not has_compile_file_ending(path): continue
            d = {}
            d['name'] = os.path.basename(path)
            d['path'] = path
            dirtext += '
{name}
'.format(**d)

        for name in sorted(os.listdir(os.path.join('.', dirname))):
            path = os.path.join(dirname, name)
            if not os.path.isdir(path): continue
            subdirtext = dir_entry(path)
            #ignore directories with empty listings
            if subdirtext != '':
                dirtext += '
' + os.path.basename(path) + subdirtext + '
'

        if dirtext != '':
            dirtext = '
' + dirtext + '
'
        elif toplevel:
            dirtext = ' (no documents)'
        return dirtext

    text += "
Directory contents
"
    text += dir_entry('', toplevel=True)

    text += ''

    return text

def needs_update(name, fmt = 'std'):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if not os.path.exists(in_filename): return False
    if not os.path.exists(out_filename): return True
    return os.path.getmtime(out_filename) < os.path.getmtime(in_filename)

def compile_document(name, fmt):
    out_filename = get_out_filename(name, fmt)
    in_filename = get_in_filename(name)
    if needs_update(name, fmt):
        #file is not cached, recompile
        #create directory if needed
        out_filename_dir = os.path.dirname(out_filename)
        if not os.path.exists(out_filename_dir):
            os.makedirs(out_filename_dir)

        #combine action based on format
        action = ['pandoc']
        # Format to use GFM"
        action.append("--from=gfm")

        if fmt == 'std':
            action += ['-H', headerfile, '-B', topmenufile,]

        if fmt == 'export':
            action += ['-H', headerfile_export,]

        if fmt == 'std' or fmt == 'export':
            action += ['-B', beforefile, '-A', afterfile,]
            action += ['--toc']

        if fmt != 'inline':
            action += ['-s']

        # compile
        #print(action)
        # create json in intermediate step
        json_encoding_process = subprocess.Popen(action + ['-t', 'json', in_filename], stdout=subprocess.PIPE)
        json_text, _ = json_encoding_process.communicate()
        json_text = json_text.decode('utf-8')

        # process json to extract embedded dot diagrams and others
        json_text, math_option = process_document_json(json_text, alternative_title=os.path.basename(name))

        # render to HTML
        json_decoding_process = subprocess.Popen(action + math_option + ['-f', 'json', '-o', out_filename], stdin=subprocess.PIPE)
        json_decoding_process.communicate(json_text.encode('utf-8'))

def process_document_json(json_text, alternative_title):
    math_option = ['--mathjax']
    changed = False

    doc = json.loads(json_text)

    # TODO descent recursively into json doc

    for block in doc["blocks"]:
        if block["t"] == "CodeBlock":
            if "dot" in block["c"][0][1]:
                extract_embedding(block, "dot")
                changed = True
            elif "plantuml" in block["c"][0][1]:
                extract_embedding(block, "plantuml")
                changed = True

    # Add title if missing
    if "title" not in doc["meta"] and "pagetitle" not in doc["meta"]:
        doc["meta"]["pagetitle"] = {'t': 'MetaInlines', 'c': [{'t': 'Str', 'c': alternative_title}]}
        changed = True

    # Allow user to select math option using metadata
    if "panserver_math" in doc["meta"]:
        meta_math = doc["meta"]["panserver_math"]["c"][0]["c"]
        print("math option ", meta_math)
        if meta_math == "mathml":
            math_option = ['--mathml']
        elif meta_math == "mathjax":
            math_option = ['--mathjax']
        elif meta_math == "none":
            math_option = []

    if changed:
        json_text = json.dumps(doc)
    return json_text, math_option

def extract_embedding(block, embedding_type):
    code = block["c"][1]
    md5 = hashlib.md5(code.encode('utf-8')).hexdigest()
    generation_dir = os.path.join(tempdir, "generated")
    error = None
    if not os.path.exists(generation_dir):
        os.makedirs(generation_dir)
    target = os.path.join(generation_dir, "{}.{}.png".format(md5, embedding_type))
    if not os.path.exists(target):
        if embedding_type == "dot":
            process = subprocess.Popen(["dot", "-Tpng"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        elif embedding_type == "plantuml":
            process = subprocess.Popen(["plantuml", "-pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        process_out, process_err = process.communicate(code.encode('utf-8'))
        error_code = process.wait()
        if error_code != 0:
            error = process_err.decode('utf-8')
        else:
            with open(target, "wb") as f:
                f.write(process_out)

    if error is None:
        # Replace embedding in block with image
        block["t"] = "Para"
        block["c"] = [ { "t" : "Image", "c" : [
            ["", [], []],
            [],
            [ "/generated/{}.{}.png".format(md5, embedding_type), "fig:"]
            ]}]
    else:
        # Insert error
        block["c"][0][1] = []
        block["c"][1] = error

def create_header(autorefresh):
    headertext = ""
    headertext += """{}""".format(meta_no_mobilescale, style_basic + style_document_add)
    headertext += markdown_css_link

    if autorefresh:
        headertext += """

"""

    with open(headerfile, 'w') as f:
        f.write(headertext)

def create_header_export():
    text = """{}""".format(meta_no_mobilescale, style_basic)
    text += markdown_css_link

    with open(headerfile_export, 'w') as f:
        f.write(text)

def create_topmenufile():
    text = """
    Panserver: Index
    TOC
    Format:
    Export
    Simple
    Inline

    """
    with open(topmenufile, 'w') as f:
        f.write(text)

def create_beforefile():
    text = """

    """
    with open(beforefile, 'w') as f:
        f.write(text)

def create_afterfile():
    with open(afterfile, 'w') as f:
        f.write("")

#global vars
file_endings = [".md", ".markdown", ".rst"]
tempdir = tempfile.mkdtemp()
outdir = os.path.join(tempdir, "out")
indir = os.path.abspath('.')

headerfile = os.path.join(tempdir, "header.html")
headerfile_export = os.path.join(tempdir, "header.export.html")
beforefile = os.path.join(tempdir, "before.html")
topmenufile = os.path.join(tempdir, "topmenu.html")
afterfile = os.path.join(tempdir, "after.html")

if not os.path.exists(outdir):
    os.makedirs(outdir)

meta_no_mobilescale = """<meta name="viewport" content="width=device-width, initial-scale=1.0">"""

style_basic = """
        body {
            width: 92%; max-width: 40em;  margin: auto;
            font-size: 1.2rem; line-height: 1.5; padding-bottom: 3em;}
        code { font-family: "DejaVu Sans Mono", Consolas, monospace; }
        pre { overflow: auto; }
        math, .LaTeX { font-size: 1.1em; }
        div.figure .caption {  padding: 0em 1em 1em; font-size: 0.8em; }
        ul li { list-style-type: disc; }
        li p { margin: 0.5em 0; }
        #TOC { border: 1px solid lightgray; padding: 0.5em; margin-bottom: 1em;}
        #TOC li { list-style: circle; }
        @media (min-width: 102em) {
            body { position: relative; }
            #TOC { display: block; position: absolute; left: -50%; top: 5em; max-width: 46%; width: 45%; }
        }
"""

style_document_add = """
        @media (max-width: 101em) {
            #TOC { display: none; }
        }
        .topmenu { font-size: 1em; color: lightgrey; }
        .topmenu a { color: lightgrey; }
"""
style_index = """
        ul.file-listing a { text-decoration: none; }
        ul.file-listing ul.file-listing { padding-left: 1.5em; }
        ul li.dir-entry {
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAApElEQVQ4je3UsQqBYRTG8Z9SMpjsrsAdYMI1WNyDhdnKriiD1U3YXML3SWE3uAgDX+lF8iqlPPX01qnn/55OncMvqYMUSeA1KjHAFKUH9S7GMcDkSb2Jk/vOQ2/QhzKq2F/fd52/ftzCBKZYYRnhA2ohcIb6m+PJdJv9A//AT4GND4BZtp0BBxgiFwG8zY7QgyIWLsv96gCE3uKIHeYoRDT1ZZ0BsR1JIXdX8A8AAAAASUVORK5CYII=');
        }
        ul li.file-entry{
            list-style-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAA00lEQVQ4jcXUQUqCURSG4Qc0bKaUiKsIhEbazMBhOHILES3EUYIGtpBwlDODhhFCTqQdtIYGf5H+eOVeFP4P3ul7L4fzHbZziQme9jDEqYhU8IkervbwjecYaRPziIeXuIuRNvESKRQjjRV+4f6XN8xQPkQ4wO0Ga9QPEeazKER4gi6uA3T9zy1KeCZb8McAY1RThCkpRtjAh2yJd/EuG0txPzy68BxT4RM2RS1FWEZH+IS1UUoRpuQ1JPw7sDfCDcnTx0rWrJ1pYSTckDwPuNgU/ADJ7ku1B60bXwAAAABJRU5ErkJggg==');
        }
"""

markdown_css_link= """<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/3.0.1/github-markdown.css">"""

if __name__ == '__main__':
    import webbrowser
    import argparse

    parser = argparse.ArgumentParser(description='Run a local markdown compiling server')

    parser.add_argument('-a', '--no-refresh', action='store_false',
                        help="Don't refresh the page",
                        dest='autorefresh')
    parser.add_argument('-p', '--port', type=int, default=10543,
                        help="Port to be opened, default is 10543",
                       )
    parser.add_argument('-b', '--no-open-browser', action='store_false',
                        help="Don't open browser",
                        dest='open_browser')
    parser.add_argument('-r', action='store_true',
                        help="""Allow remote access to this server.
                        Warning! can be extremely dangerous!"""
                       )
    parser.add_argument('path', nargs='?', help="Path to directory")
    config = parser.parse_args()

    create_header(config.autorefresh)
    create_header_export()
    create_topmenufile()
    create_beforefile()
    create_afterfile()

    if config.path != None:
        if os.path.isdir(config.path):
            os.chdir(config.path)
            indir = os.path.abspath('.')
        else:
            raise Exception('Unknown path argument')

    if config.open_browser:
        webbrowser.get().open('http://localhost:{}/'.format(config.port))

    host = 'localhost'
    if config.r:
        host = ''

    bottle.run(host=host, port=config.port)
    shutil.rmtree(outdir)


from xai.brain.wordbase.nouns._yearbook import _YEARBOOK

#calss header
class _YEARBOOKS(_YEARBOOK, ):
	def __init__(self,):
		_YEARBOOK.__init__(self)
		self.name = "YEARBOOKS"
		self.specie = 'nouns'
		self.basic = "yearbook"
		self.jsondata = {}


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Pixel Starships Prestige Bot
#
# 1. Create Discord Bot at:
#    https://discordapp.com/developers/applications/me
# 2. Set Discord Bot permissions at:
#    https://discordapi.com/permissions.html

# ----- Packages ------------------------------------------------------
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from discord.ext import commands

import discord
import logging
import os
import pss_dropship as dropship
import pss_fleets as flt
import pss_market as mkt
import pss_prestige as p
import pss_research as rs
import pytz
import time, datetime, holidays

# ----- Setup ---------------------------------------------------------
command_prefix = os.getenv('COMMAND_PREFIX')
BOT_OWNER = os.getenv("DISCORD_BOT_OWNER")
BOT_VERSION = '0.05'
BOT_DESCRIPTION = 'This is a bot for pixel starships'
RATE = 3
COOLDOWN = 30.0

# tbl, rtbl = p.get_char_sheet()
ctbl, tbl_i2n, tbl_n2i, rarity = p.get_char_sheet()
df_research_designs = rs.get_research_designs()

raw_text = mkt.load_item_design_raw()
df_items = mkt.xmltext_to_df(raw_text)
item_rlookup = mkt.get_item_rlookup(df_items)

# welcome_txt = """**Welcome to the Pixel Starships Discord!**
# This is a place where we can interact with devs and players from other alliances/fleets
# 1. :zipper_mouth: you can mute any channel to avoid notifications from that channel
# 2. :microphone: #announcements-and-tips-of-the-day are for anything really important on PSS, as well as a means for the devs to post announcements
# 3. :bulb: if you have any suggestions for the game, you can post them in #suggestion-posts. previously discussed suggestions are in #summaries
# 4. :beetle: #game-support is for bugs, etc. \@@TheRealTiffany is usually there
# 5. :robot: We have a #dolores-bot-room. type “`/help`” to find out how to use this bot for character stats/prestige combos, and item prices/stats.
# 6. :art: Get a Discord color by posting a screenshot with your trophy level in #screenshots
# 7. :dove: The usual guidelines for any forum apply (try to be civil, don’t spam, etc)."""
welcome_txt = """**Welcome to the Pixel Starships Discord!**
This is a place where we can interact with devs and players from other alliances/fleets
1. 🤐 you can mute any channel to avoid notifications from that channel
2. 🎤 #announcements-and-tips-of-the-day are for anything really important on PSS, as well as a means for the devs to post announcements
3. 💡 if you have any suggestions for the game, you can post them in #suggestion-posts. previously discussed suggestions are in #summaries
4. 🐞 #game-support is for bugs, etc. \@@TheRealTiffany is usually there
5. 🤖 We have a #dolores-bot-room. type “`/help`” to find out how to use this bot for character stats/prestige combos, and item prices/stats.
6. 🎨 Get a Discord color by posting a screenshot with your trophy level in #screenshots
7. 🕊 The usual guidelines for any forum apply (try to be civil, don’t spam, etc)."""

# ----- Utilities -----------------------------------------------------
def write_log(log_text, author, server):
    if os.path.isfile("debug") is False:
        return
    time_text = datetime.datetime.now().strftime('%Y%m%d %H:%M:%S')
    ctx_text = '' if ctx is None else \
        '{}@{}'.format(author, server)

    final_text = '{} {}: {}{}'.format(
        time_text, ctx_text, command_prefix, log_text)
    with open("discord-bot.log", "a") as f:
        f.write(final_text + '\n')

# ----- Bot Setup -------------------------------------------------------------
logging.basicConfig(
    level=logging.INFO,
    style = '{',
    datefmt = "%Y%m%d %H:%M:%S",
    format = "{asctime} [{levelname:<8}] {name}: {message}")
    # level="INFO")
bot = commands.Bot(command_prefix=command_prefix,
                   description=BOT_DESCRIPTION)
setattr(bot, "logger", logging.getLogger("bot.py"))

# ----- Bot Events ------------------------------------------------------------
@bot.event
async def on_ready():
    print('Bot logged in as {} (id={})'.format(
        bot.user.name, bot.user.id))

@bot.event
async def on_command_error(ctx, err):
    if isinstance(err, commands.CommandOnCooldown):
        await ctx.send('Error: {}'.format(err))

# ----- Bot Commands ----------------------------------------------------------
@bot.command(brief='Ping the server')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def ping(ctx):
    """Ping the server to verify that it\'s listening for commands"""
    await ctx.send('Pong!')

@bot.command(brief='Get prestige combos of crew')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def prestige(ctx, *, name: str=None):
    """Get the prestige combinations of the character specified"""
    if name is None:
        help_txt = 'Enter: {}prestige [character name]'.format(command_prefix)
        await ctx.send(help_txt)
        return

    # write_log(command_prefix + 'prestige {}'.format(name), ctx.author, ctx.server)
    # print("Calling p.prestige({}, 'from', tbl_i2n, tbl_n2i)".format(name))
    prestige_txt, success = p.get_prestige(name, 'from', tbl_i2n, tbl_n2i)
    # print("prestige_txt = {}".format(prestige_txt))
    # print("success = {}".format(success))
    for txt in prestige_txt:
        await ctx.send(txt)

@bot.command(brief='Get character/item recipes')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def recipe(ctx, *, name=None):
    """Get the prestige recipes of a character or ingredients for an item"""
    if name is None:
        help_txt = 'Enter: {}recipe [name]'.format(command_prefix)
        await ctx.send(help_txt)
        return

    # write_log(command_prefix + 'recipe {}'.format(name), ctx.author, ctx.server)
    recipe_found = False

    # Character Recipe
    prestige_txt, success = p.get_prestige(name, 'to', tbl_i2n, tbl_n2i)
    if success is True:
        for txt in prestige_txt:
            await ctx.send(txt)
        recipe_found = True
        return

    # Item Recipe
    content, real_name = mkt.get_item_recipe(name, levels=5)
    if real_name is not None:
        content = '**Recipe for {}**\n'.format(real_name) + content
        content = content + '\n\nNote: bux prices listed here may not always be accurate due to transfers between alts/friends or other reasons'
        await ctx.send(content)
        recipe_found = True

    if recipe_found is False:
        await ctx.send("Could not find character or item named '{}'".format(name))

# @bot.command(brief='Get recent market postings')
# @commands.cooldown(rate=1, per=COOLDOWN, type=commands.BucketType.channel)
# async def market(ctx):
#     """Get a list of items recently posted on the market"""
#     market_txt = mkt.get_market_data()
#     await ctx.send(market_txt)

@bot.command(brief='Get item prices from the PSS API')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def price(ctx, *, item_name=None):
    """Get the average price in bux of the item(s) specified, as returned by the PSS API. Note that prices returned by the API may not reflect the real market value, due to transfers between alts/friends"""
    if item_name is None:
        await ctx.send('Enter: {}price [item name]'.format(command_prefix))
        return

    # write_log(command_prefix + '{} {}'.format(ctx.invoked_with, item_name), ctx.author, ctx.server)
    raw_text = mkt.load_item_design_raw()
    item_lookup = mkt.parse_item_designs(raw_text)

    real_name = mkt.get_real_name(item_name, item_lookup)
    if len(item_name) < 2:
        await ctx.send("Please enter at least two characters for item name")
    elif real_name is not None:
        market_txt = mkt.filter_item_designs(item_name, item_lookup, filter='price')
        market_txt = "**Prices matching '{}'**\n".format(item_name) + market_txt
        market_txt += '\n\nNote: bux prices listed here may not always be accurate due to transfers between alts/friends or other reasons'
        await ctx.send(market_txt)
    else:
        await ctx.send("Could not find item name '{}'".format(item_name))

@bot.command(name='list', brief='List items/characters')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def list(ctx, *, action=''):
    """action=chars: shows all characters
    action=newchars: shows the newest 10 characters that have been added to the game
    action=items: shows all items"""
    # write_log(command_prefix + ctx.invoked_with + ' ' + action, ctx.author, ctx.server)
    txt_list = []
    if action in ['chars', 'newchars']:
        txt_list = p.get_char_list(action)
    elif action == 'items':
        txt_list = mkt.get_item_list()
    elif action == 'research':
        txt_list = rs.get_research_names()
    elif action == 'rooms':
        txt_list = rs.get_room_names()

    for txt in txt_list:
        await ctx.send(txt)

@bot.command(name='stats', aliases=['item'],
    brief='Get item/character stats')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def stats(ctx, *, name=None):
    """Get the stats of a character/crew or item"""
    if name is None:
        await bot.say('Enter: {} [name]'.format(command_prefix + ctx.invoked_with))
        return

    # First try to find a character match
    # (skip this section if command was invoked with 'item'
    if ctx.invoked_with != 'item':
        # write_log('stats {}'.format(name), str(ctx.author), str(ctx.server))
        result = p.get_stats(name, embed=False)
        if result is not None:
            await ctx.send(result)
            found_match = True

    # Next try to find an item match
    # raw_text = mkt.load_item_design_raw()
    # item_lookup = mkt.parse_item_designs(raw_text)
    # market_txt = mkt.filter_item_designs(name, item_lookup, filter='stats')
    market_txt = mkt.get_item_stats(name)
    if market_txt is not None:
        await ctx.send(market_txt)
        found_match = True

    if found_match is False:
        await ctx.send('Could not find {}'.format(name))

# @bot.command()
# async def stats2(ctx, *, name=None):
#     if name is None:
#         await ctx.send('Enter: {}stats [character name]'.format(command_prefix))
#     else:
#         write_log(command_prefix + "stats {}".format(name), ctx)
#         result = p.get_stats(name, embed=True)
#         if result is None:
#             await ctx.send('Could not find {}'.format(name))
#         # else:
#         #     await bot.send_message(ctx.message.channel, embed=result)

@bot.command(brief='Get best items for a slot')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def best(ctx, slot=None, enhancement=None):
    """Get the best enhancement item for a given slot. If multiple matches are found, matches will be shown in descending order."""
    if slot is None:
        txt = 'Enter: {}best [slot] [enhancement]'.format(command_prefix)
        await ctx.send(txt)
        return
    txt = command_prefix + 'best {} {}'.format(slot, enhancement)
    # write_log(txt, ctx.author, ctx.server)

    raw_text = mkt.load_item_design_raw()
    item_lookup = mkt.parse_item_designs(raw_text)
    df_items = mkt.rtbl2items(item_lookup)
    df_filter = mkt.filter_item(
        df_items, slot, enhancement,
        cols=['ItemDesignName', 'EnhancementValue', 'MarketPrice'])

    txt = mkt.itemfilter2txt(df_filter)
    if txt is None:
        await ctx.send('No entries found for {} slot, {} enhancement'.format(
            slot, enhancement))

        str_slots = ', '.join(df_items['ItemSubType'].value_counts().index)
        str_enhancements = ', '.join(df_items['EnhancementType'].value_counts().index)
        txt  = 'Slots: {}\n'.format(str_slots)
        txt += 'Enhancements: {}'.format(str_enhancements)
        await ctx.send(txt)
    else:
        await ctx.send(txt)

@bot.command(brief='Get research data')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def research(ctx, *, research=None):
    """Get the research details on a specific research. If multiple matches are found, only a brief summary will be provided"""
    if research is None:
        txt = 'Enter: {}research [research]'.format(command_prefix)
        await ctx.send(txt)
        return
    txt = command_prefix + 'research {}'.format(research)
    # write_log(txt, ctx.author, ctx.server)

    df_selected = rs.filter_researchdf(df_research_designs, research)
    txt = rs.research_to_txt(df_selected)
    if txt is None:
        await ctx.send("No entries found for '{}'".format(research))
    else:
        await ctx.send(txt)

@bot.command(brief='Get collections')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def collection(ctx, *, collection=None):
    """Get the details on a specific collection."""
    if collection is None:
        txt = 'Enter: {}collection [collection]'.format(command_prefix)
        await ctx.send(txt)
        return
    txt = command_prefix + 'collection {}'.format(collection)
    # write_log(txt, ctx.author, ctx.server)

    print(collection)
    txt = p.show_collection(collection)
    if txt is None:
        await ctx.send("No entries found for '{}'".format(collection))
    else:
        await ctx.send(txt)

@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
@bot.command(hidden=True, brief='Division stars')
async def stars(ctx, *, division=None):
    """Get stars earned by each fleet during final tournament week"""
    if division is None:
        txt = 'Enter: {}stars [division]'.format(command_prefix)
        await ctx.send(txt)
        return
    txt = command_prefix + 'stars {}'.format(division)
    # write_log(txt, ctx.author, ctx.server)
    txt = flt.get_division_stars(division)
    await ctx.send(txt)

# @commands.cooldown(rate=1, per=COOLDOWN, type=commands.BucketType.channel)
# @bot.command(hidden=True, brief='Armor damage')
# async def armor(ctx, *, enhancement=None):
#     """Get armor damage"""
#     txt = command_prefix + 'armor {}'.format(enhancement)
#     # write_log(txt, ctx.author, ctx.server)
#     damage_fraction = 100.0 / (1.0 + float(enhancement)/100.0)
#     txt = "A room enhanced %.1f%% by armor takes a damage of %.1f%% of the base value" % (enhancement, damage_fraction)
#     await ctx.send(txt)

@commands.cooldown(rate=2, per=600.0, type=commands.BucketType.channel)
@bot.command(hidden=True, brief='Show the welcome message')
async def welcome(ctx):
    """Show the welcome message"""
    # write_log('welcome ' + channel, ctx.author, ctx.server)
    await ctx.send(welcome_txt)

@commands.cooldown(rate=2, per=60.0, type=commands.BucketType.channel)
@bot.command(hidden=True, brief='Show the dailies')
async def daily(ctx):
    """Show the dailies"""
    txt = dropship.get_dropship_text()
    await ctx.send(txt)

@bot.command(brief='Get PSS stardate & Melbourne time')
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def time(ctx):
    """Get PSS stardate, as well as the day and time in Melbourne, Australia. Gives the name of the Australian holiday, if it is a holiday in Australia."""
    now = datetime.datetime.now()
    today = datetime.date(now.year, now.month, now.day)
    pss_start = datetime.date(year=2016, month=1, day=6)
    pss_stardate = (today - pss_start).days
    str_time = 'Today is Stardate {}\n'.format(pss_stardate)

    mel_tz = pytz.timezone('Australia/Melbourne')
    mel_time = now.replace(tzinfo=pytz.utc).astimezone(mel_tz)
    str_time += mel_time.strftime('It is %A, %H:%M in Melbourne')

    aus_holidays = holidays.Australia(years=now.year, prov='ACT')
    mel_time = datetime.date(mel_time.year, mel_time.month, mel_time.day)
    if mel_time in aus_holidays:
        str_time += '\nIt is also a holiday ({}) in Australia'.format(aus_holidays[mel_time])
    await ctx.send(str_time)

@bot.command(hidden=True,
    brief='These are testing commands, usually for debugging purposes')
@commands.is_owner()
@commands.cooldown(rate=RATE, per=COOLDOWN, type=commands.BucketType.channel)
async def testing(ctx, *, action=None):
    """These are testing commands, usually for debugging purposes"""
    if action == 'refresh':
        tbl, rtbl = p.get_char_sheet(refresh=True)
        raw_text = mkt.load_item_design_raw(refresh=True)
        await ctx.send('Refreshed')
    elif action == 'restart':
        await ctx.send('Bot will restart')
    elif action == 'info':
        server = ctx.server
        author = ctx.author
        channel = ctx.channel
        txt = 'Server: {} (id={})\n'.format(server, server.id)
        txt += 'Message Author: {} (id={})\n'.format(author, author.id)
        txt += 'This Channel: {} (id={})\n'.format(channel, channel.id)
        txt += 'Discord.py version: {}'.format(discord.__version__)
        await ctx.send(txt)
    elif action == 'say':
        if isinstance(text, str):
            await ctx.send(txt)
    elif action == 'actions':
        print(dir(bot))
        bot_actions = list(dir(bot))
        bot_actions = 'Bot actions:' + ', '.join(bot_actions)
        await ctx.send(bot_actions)

    await bot.delete_message(ctx.message)
    if action == 'restart':
        print('Attempting to restart')
        bot.close()
        quit()

@bot.command(hidden=True, brief='Gives the version number of the bot')
@commands.cooldown(rate=1, per=COOLDOWN, type=commands.BucketType.channel)
async def version(ctx):
    """Gives the version number of the bot"""
    txt = 'Bot version is {}'.format(BOT_VERSION)
    await ctx.send(txt)

if __name__ == '__main__':
    # Running the Discord Bot
    token = os.getenv("DISCORD_BOT_TOKEN")
    bot.run(token)


from django.shortcuts import render
from user.models import UserSensorDetail
from sensor_owner.models import SensorDetail

def is_sensor_present(request, sensor_data):
    if UserSensorDetail.objects.filter(user_name=request.user, sensor_id=sensor_data).exists():
        return True
    else:
        return False

def manage_sensors(request):
    user_sensor_data = UserSensorDetail.objects.filter(user_name=request.user)
    available_sensor_data = SensorDetail.objects.all()
    return render(request, 'manage_sensors.html', {'user_sensors': user_sensor_data, 'available_sensors': available_sensor_data})

def add_sensor(request, pk):
    available_sensor_data = SensorDetail.objects.all()
    sensor_data = available_sensor_data.get(id=pk)
    current_user = request.user
    error_message = ''
    success_message = ''
    if is_sensor_present(request, sensor_data):
        error_message = 'Sensor already subscribed'
    else:
        new_sensor = UserSensorDetail()
        new_sensor.user_name = current_user
        new_sensor.sensor_id = sensor_data
        new_sensor.save()
        success_message = 'Sensor with ID '+ str(sensor_data.sensor_id) + ' subscribed'

    user_sensor_data = UserSensorDetail.objects.filter(user_name=request.user)
    return render(request, 'manage_sensors.html', {'user_sensors': user_sensor_data, 'available_sensors': available_sensor_data,
                                                   'error_message': 'Sensor already subscribed', 'error_message': error_message,
                                                   'success_message': success_message})

def delete_sensor(request, pk):
    sensor = UserSensorDetail.objects.get(id=pk)
    sensor.delete()

    user_sensor_data = UserSensorDetail.objects.filter(user_name=request.user)
    available_sensor_data = SensorDetail.objects.all()
    success_message = 'Sensor deleted'

    return render(request, 'manage_sensors.html', {'user_sensors': user_sensor_data, 'available_sensors': available_sensor_data,
                                                   'success_message': success_message})


import io
import pathlib
import shutil
import tarfile
import traceback

import gitlab
import rstgen
import urllib3
from analysis import classifiers, logs, tests

cfrs_sorted = sorted(classifiers.classifiers, key=lambda cfr: cfr.skb)

def _strip_match(match):
    """Strip detailed test log from a match to safe space"""

    if not "test" in match:
        return match

    test = dict(match["test"])
    if "msgs" in test:
        del test["msgs"]
    if "teardown" in test:
        del test["teardown"]
    if "detail" in test:
        del test["detail"]
    if "teardown_detail" in test:
        del test["teardown_detail"]

    match_stripped = dict(match)
    match_stripped["test"] = test
    return match_stripped

def _is_tarball(fname):
    return (
        fname.endswith(".tar.gz")
        or fname.endswith(".tar.xz")
        or fname.endswith(".tar.bz2")
    )

class Report:
    def __init__(self, matches_per_clfr_count=10, context_lines=10):

        # Parameters
        self.matches_per_clfr_count = matches_per_clfr_count
        self.context_lines = context_lines

        # Work structures
        self.http = urllib3.PoolManager()
        self.matches_per_file = {}
        self.matches_per_clfr = {cfr.skb: [] for cfr in cfrs_sorted}
        self.all_matches_per_clfr = {cfr.skb: [] for cfr in cfrs_sorted}
        self.total_lines = 0
        self.log_id = 0
        self.pod_timings = {}
        self.revisions = {}
        self.revision_files = {}
        self.file_date = {}

        # See build_maps()
        self.files_per_cfr = None

    def add_log(self, fname, log, source=None, revision=None):
        """Adds a log to the report

        :param fname: (File) name of the log
        :param log: List of log lines as dictionaries
        :param source: Original source of the log, say if extracted from an URL
        :param sha: The Git revision associated witht the log
        """

        # Extract test data, classify
        self.total_lines += len(log)
        test_data = tests.collect_tests(log, 1)
        matches = classifiers.classify_test_results(test_data)

        # Note all matches, but remove detailed test logs to save space
        self.matches_per_file[fname] = [_strip_match(match) for match in matches]

        # Add to revision-file map
        if revision is not None:
            if revision not in self.revision_files:
                self.revision_files[revision] = set()
            self.revision_files[revision].add(fname)

        # Extract date from first log message
        log_date = None
        for l in log:
            if "time" in l:
                log_date = l["time"]
                break
        if log_date is not None:
            self.file_date[fname] = log_date

            # Add revision to dictionary, noting the latest date we have seen it
            # (for sorting later)
            if revision is not None and (
                revision not in self.revisions or self.revisions[revision] < log_date
            ):
                self.revisions[revision] = log_date

            for cfr in classifiers.classifiers:
                cfr_matches = [match for match in matches if match["cfr"] is cfr]
                if not cfr_matches:
                    continue
                self.add_cfr_matches(
                    fname, source, log, log_date, cfr, cfr_matches, revision
                )

            # Collect timings
            timings = logs.collect_pod_timings(log)
            for pod, pod_ts in timings.items():
                current_pod_ts = self.pod_timings.get(pod, {})
                # Note that theoretically a container can appear multiple times.
                # We ignore that here.
                for cont, cont_t in pod_ts:
                    if cont not in current_pod_ts:
                        current_pod_ts[cont] = []
                    current_pod_ts[cont].append((cont_t - log_date).total_seconds())
                self.pod_timings[pod] = current_pod_ts

        self.log_id += 1

    def add_cfr_matches(self, fname, source, log, log_date, cfr, cfr_matches, revision):

        # Strip test information
        cfr_matches_stripped = [_strip_match(cfr_match) for cfr_match in cfr_matches]

        # Add to list, retaining only a number of matches (as
        # we keep basically the entire log for them in
        # memory!)
        match_list = self.matches_per_clfr.get(cfr.skb, [])
        match_log = dict(
            date=log_date,
            log=log,
            log_id=self.log_id,
            cfr=cfr,
            name=fname,
            source=source,
            matches=cfr_matches,
            revision=revision,
        )
        match_list.append(match_log)
        match_list = sorted(match_list, key=lambda match: match["date"], reverse=True)
        self.matches_per_clfr[cfr.skb] = match_list[: self.matches_per_clfr_count]

        # Retain stripped matches for all
        all_matches_list = self.all_matches_per_clfr.get(cfr.skb, [])
        match_log_stripped = dict(match_log)
        match_log_stripped["matches"] = cfr_matches_stripped
        del match_log_stripped["log"]
        all_matches_list.append(match_log_stripped)
        self.all_matches_per_clfr[cfr.skb] = list(all_matches_list)

    def add_tarball(self, tar, source, revision=None):
        while True:
            # Read through tar file sequentially to minimise seeking
            info = tar.next()
            if info is None:
                return
            # Extract
            with tar.extractfile(info) as f:
                try:
                    self.add_log(
                        info.name, logs.collect_file(info.name, 1, f), source, revision
                    )
                except Exception:
                    traceback.print_exc()

    def add_file_or_uri(self, fname, revision=None):

        # Is a URI?
        if fname.startswith("http://") or fname.startswith("https://"):

            # Read it to a memory buffer
            bio = io.BytesIO()
            print(f"Retrieving {fname}...", flush=True)
            _fname = urllib3.util.parse_url(fname).path
            with self.http.request("GET", fname, preload_content=False) as r:
                # Get suggested file name from header
                if (
                    r.getheaders()
                    .get("Content-Disposition", "")
                    .startswith('attachment; filename="')
                ):
                    _fname = r.getheaders().get("Content-Disposition")[22:-1]
                # Copy data
                shutil.copyfileobj(r, bio)
            print(f" ... got {len(bio.getvalue())} bytes.", flush=True)
            bio.seek(0)

            # Tarball?
            if _is_tarball(_fname):
                with tarfile.open(mode="r:*", fileobj=bio) as tar:
                    self.add_tarball(tar, fname, revision)
            else:
                # Otherwise expect it to be a flat file
                self.add_log(_fname, logs.collect_file(fname, 1, bio), fname, revision)
            return

        # Assume it's a file. Tarball?
        if _is_tarball(fname):
            with tarfile.open(fname, mode="r:*") as tar:
                self.add_tarball(tar, fname, revision)
        else:
            self.add_log(fname, logs.collect_file(fname, 1), revision)

    def add_from_gitlab(self, uri, header, project, search, job_names, artifact):

        # Get project, search for pipelines
        gl = gitlab.Gitlab(uri, **header)
        proj = gl.projects.get(project)
        pips = proj.pipelines.list(**search)
        print(f"Found {len(pips)} GitLab pipelines", flush=True)

        # Go through pipelines, look for matching jobs
        for pip in pips:

            # Find a (successful) job matching the desired name
            jobs = pip.jobs.list(scope="success", include_retried="yes")
            for job in (job for job in jobs if job.name in job_names):

                # (Attempt to) download the artefact
                try:
                    self.add_file_or_uri(
                        f"{uri}/{project}/-/jobs/{job.id}/artifacts/raw/{artifact}?inline=false",
                        pip.sha,
                    )
                except Exception:
                    traceback.print_exc()

    def make_matches_table(self, f):

        # Compose header from classifier names
        header = ["name"]
        for cfr in cfrs_sorted:
            header.append(cfr.skb)

        # Now go through files
        rows = []
        match_count = {cfr.skb: 0 for cfr in classifiers.classifiers}
        for fname in sorted(self.matches_per_file.keys()):
            matches = self.matches_per_file[fname]
            row = [fname]
            for cfr in cfrs_sorted:
                cfr_matches = [match for match in matches if match["cfr"] is cfr]
                if cfr_matches:
                    row.append("X")
                    match_count[cfr.skb] += 1
                else:
                    row.append("")
            rows.append(row)

        # Add a table with the count at the end
        rows.append(["Sum:"] + [str(match_count[cfr.skb]) for cfr in cfrs_sorted])
        print(rstgen.table(header, rows), file=f)

    def build_maps(self):

        # Files that match a given classifier
        self.files_per_cfr = {
            cfr.skb: {
                fname
                for fname, matches in self.matches_per_file.items()
                if any(match["cfr"] is cfr for match in matches)
            }
            for cfr in classifiers.classifiers
        }

    def make_overview_table(self, f, revision=None):

        # Print a header
        if revision is None:
            files = self.matches_per_file.keys()
        else:
            files = self.revision_files[revision]
        file_count = len(files)
        print(f"Covering {file_count} logs.", file=f)
        if file_count > 0:
            files_sorted = sorted(files, key=lambda fname: self.file_date.get(fname))
            print(
                f"Date range: {self.file_date[files_sorted[0]]} to {self.file_date[files_sorted[-1]]}\n",
                file=f,
            )

        # Start composing table
        header = [
            "**SKB**",
            "**Message**",
            "**Affected Runs**",
            "**Last**",
            "**Cross-Check**",
        ]
        rows = []
        for cfr in sorted(
            cfrs_sorted, key=lambda cfr: len(self.files_per_cfr[cfr.skb]), reverse=True
        ):

            matches = self.all_matches_per_clfr[cfr.skb]
            files_with_matches = self.files_per_cfr[cfr.skb]

            # Specialise to revision, if requested.
            if revision is not None:

                # Skip rows that have no matches globally (so we don't
                # repeat them for every revision).
                if not matches:
                    continue

                matches = [
                    cfr_match
                    for cfr_match in matches
                    if cfr_match["revision"] == revision
                ]
                files_with_matches = files_with_matches & self.revision_files[revision]
            files_matched_rel = len(files_with_matches) / max(1, file_count)

            # Include date + link to latest match
            if matches:
                last_match = sorted(matches, key=lambda match: match["date"])[-1]
                # Generate link - if it is one of the matches we are keeping
                if any(
                    cfr_match["log_id"] == last_match["log_id"]
                    for cfr_match in self.matches_per_clfr[cfr.skb]
                ):
                    last_ref = f":ref:`{last_match['date']} <{cfr.skb}-{last_match['log_id']}>`"
                else:
                    last_ref = last_match["date"]
            else:
                last_ref = ""

            # Determine whether another kind of error appears to happen more often
            other_skb_changes = ""
            if files_with_matches:

                # Collect number of runs that fail for other classifiers as well
                cfr_intersection = {
                    cfr2.skb: len(self.files_per_cfr[cfr2.skb] & files_with_matches)
                    for cfr2 in classifiers.classifiers
                }

                # Determine ratio of those other failures in relation
                # to our failures with the ratio to all runs.  Filter
                # out classifiers that did not happen, this SKB - as
                # well as taints if this classifier likely caused the
                # taint.
                cfr_intersection_changes = sorted(
                    [
                        (
                            cfr2,
                            cfr_intersection[cfr2.skb] / len(files_with_matches)
                            - len(self.files_per_cfr[cfr2.skb]) / file_count,
                        )
                        for cfr2 in classifiers.classifiers
                        if len(self.files_per_cfr[cfr2.skb]) > 0
                        and cfr2.skb != cfr.skb
                        and (not cfr.taints or not cfr2.skb.startswith("TAINT"))
                    ],
                    key=lambda skb_diff: skb_diff[1],
                    reverse=True,
                )

                other_skb_changes = ", ".join(
                    f":ref:`{cfr2.skb} <{cfr2.skb}>`: {change*100:+.1f}%"
                    for cfr2, change in cfr_intersection_changes[:3]
                    if change > 0
                )

            rows.append(
                [
                    f":ref:`{cfr.skb} <{cfr.skb}>`",
                    cfr.message,
                    f"{len(files_with_matches)} ({files_matched_rel*100:.1f}%)",
                    last_ref,
                    other_skb_changes,
                ]
            )

        print(rstgen.table(header, rows), file=f)

    def make_log_block(self, f, log, highlight_lines):

        # Find log lines to highlight
        to_highlight = sorted(
            [i for i, l in enumerate(log) if l.get("time") in highlight_lines]
        )
        if not to_highlight:
            to_highlight = [0, len(log) - 1]
        # print(to_highlight)

        # Go through log lines, decide what to do
        skipping = False
        lines_out = []
        highlight_out = []
        for i, l in enumerate(log):

            while to_highlight and i > to_highlight[0] + self.context_lines:
                to_highlight.pop(0)

            # Done?
            if not to_highlight:
                if not skipping:
                    lines_out.append(f"  ... {len(log)-i} lines skipped ...")
                break

            # Skip?
            if i < to_highlight[0] - self.context_lines:
                if not skipping:
                    lines_out.append(
                        f"  ... {to_highlight[0]-i-self.context_lines} lines skipped ..."
                    )
                    skipping = True
                continue

            # Show line
            lines_out.append("  " + logs.pp_line(l))
            if l.get("time") in highlight_lines:
                highlight_out.append(len(lines_out))
            skipping = False

        print(".. code-block:: none", file=f)
        if highlight_out:
            print(
                f"  :emphasize-lines: {','.join(str(i) for i in highlight_out)}", file=f
            )
        print(file=f)
        for s in lines_out:
            print(s, file=f)
        print(file=f)

    def make_test_report(self, f, test, matches):

        # Get the timestamps of all lines that matched. We assume those to
        # be unique.
        match_lines = set()
        for match in matches:
            match_lines |= {line["time"] for line in match["matched"]}

        # Print test data
        self.make_log_block(f, test["msgs"], match_lines)
        for section_name, section_lines in test.get("detail", {}).items():
            print(f"{section_name}:\n", file=f)
            self.make_log_block(f, section_lines, match_lines)

        if "teardown" not in test:
            return
        print(rstgen.header(4, f"Teardown"), file=f)
        self.make_log_block(f, test["teardown"], match_lines)
        for section_name, section_lines in test.get("teardown_detail", {}).items():
            print(f"{section_name}:\n", file=f)
            self.make_log_block(f, section_lines, match_lines)

    def make_cfr_report(self, f, cfr):

        print(f".. _{cfr.skb}:\n", file=f)
        print(rstgen.header(1, cfr.skb + " " + cfr.message), file=f)

        total_matches = len(self.all_matches_per_clfr[cfr.skb])
        files_with_matches = self.files_per_cfr[cfr.skb]
        files_matched_rel = len(files_with_matches) / max(1, len(self.matches_per_file))

        print(
            f"Found a total of {len(self.all_matches_per_clfr[cfr.skb])} matches in "
            f"{len(files_with_matches)} ({files_matched_rel*100:.1f}%) runs "
            f'{", ".join(sorted(files_with_matches))}\n',
            file=f,
        )

        # Show example matches
        for cfr_match in self.matches_per_clfr[cfr.skb]:

            first = True

            def test_id(match):
                return (
                    match["test"]["file"],
                    match["test"]["name"],
                    match["test"].get("param"),
                )

            for tid in {test_id(match) for match in cfr_match["matches"]}:

                # Get all data
                matches = [
                    match for match in cfr_match["matches"] if test_id(match) == tid
                ]
                test = matches[0]["test"]

                # Show header for test
                if "param" in test:
                    test_name = f"{test.get('status')} test {test['file']}::{test['name']}[{test['param']}]"
                else:
                    test_name = (
                        f"{test.get('status')} test {test['file']}::{test['name']}"
                    )

                date = cfr_match["date"].strftime("%Y-%m-%d")
                if first:
                    print(f".. _{cfr.skb}-{cfr_match['log_id']}:\n", file=f)
                    first = False
                print(rstgen.header(2, f" {date} {test_name}"), file=f)

                print(
                    f"See also :download:`{cfr_match['name']} <log-{cfr_match['log_id']}.txt>`"
                    f"{'' if cfr_match['source'] is None else ' (source ' + cfr_match['source'] + ')'}"
                    f", recorded {cfr_match['date']}\n",
                    file=f,
                )

                self.make_test_report(f, test, matches)

        return files_matched_rel

    def write_log(self, f, cfr_match):

        # Indicate where log came from
        source = cfr_match["source"]
        if source:
            print(f"Log {cfr_match['name']} retrieved from {source}, matched:", file=f)
        else:
            print(f"Log {cfr_match['name']}, matched:", file=f)

        # List matched classifiers, collecting lines to highlight
        to_highlight = {}
        other_matches = self.matches_per_file[cfr_match["name"]]
        for other in other_matches:
            print(f" * {other['cfr'].skb} {other['cfr'].message}", file=f)
            for line in other["matched"]:
                if line["time"] not in to_highlight:
                    to_highlight[line["time"]] = [other]
                else:
                    to_highlight[line["time"]].append(other)

        # Now pretty-print lines
        for l in cfr_match["log"]:
            print(logs.pp_line(l), file=f)
            for other in to_highlight.get(l["time"], []):
                print(f" ^^ {other['cfr'].skb} {other['cfr'].message}", file=f)

    def write_logs(self, directory):

        # Collect all logs to generate
        logs_to_generate = {}
        for cfr in classifiers.classifiers:
            for cfr_match in self.matches_per_clfr[cfr.skb]:
                logs_to_generate[cfr_match["log_id"]] = cfr_match

        # Generate them!
        for log_id, cfr_match in logs_to_generate.items():
            with open(
                pathlib.Path(directory, f"log-{log_id}.txt"), "w", encoding="utf-8"
            ) as f:
                self.write_log(f, cfr_match)

        return len(logs_to_generate)

    def make_pod_timing_table(self, f):

        # Determine average start
        pod_timings_avg = {
            pod: {cont: sum(ts) / len(ts) for cont, ts in pod_conts.items()}
            for pod, pod_conts in self.pod_timings.items()
            if all(len(ts) > 1 for ts in pod_conts.values())
        }

        if not pod_timings_avg:
            return
        max_stages = max(len(container) for container in pod_timings_avg.values())
        header = ["**Pod**"] + [""] * max_stages

        # Go through pods sorted by last average finish
        pods_sorted = sorted(
            pod_timings_avg.items(), key=lambda pod_map: max(pod_map[1].values())
        )
        rows = []
        for pod, cont_avgs in pods_sorted:

            # Right-justify the row
            row = [pod] + (max_stages - len(cont_avgs)) * [""]
            for cont, avg in sorted(
                cont_avgs.items(), key=lambda cont_avg: cont_avg[1]
            ):
                row.append(f"{cont}: {avg:.2f} s")
            rows.append(row)

        print(rstgen.table(header, rows), file=f)


import csv
import pickle
import random

import numpy as np
import ujson as json
import torch
import pandas as pd
from sklearn.model_selection import train_test_split
from typing import List
import spacy as spacy

'''
Read data from file and return a list of list
every item in the list is a list of original text, response text and label
label = 0 -> not sarcasm
label = 1 -> sarcasm
'''

def read_discussion_forum(file="./data/dicussion-forum-data.csv"):
    '''Read data from discussion forum data'''
    with open(file) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        data = []
        for row in csv_reader:
            # if line_count > 100:
            #     break
            if line_count == 0:
                print('Column names are {}'.format(", ".join(row)))
                line_count += 1
            else:
                '''Remember to trim leading and trailing spaces'''
                data.append([row[3].strip(), row[4].strip(), 1 if row[1] == 'sarc' else 0])
                line_count += 1
        print('Processed {} lines.'.format(line_count))
        return data

'''
Stores indices to file in the format of [contexts, responses, labels]'''
def save_discussion_forum_data():
    data = read_discussion_forum()
    contexts, responses, labels = generate_indices(data, "discussion")
    with open('./data/discussion/discussion_forum_indices', 'wb') as f:
        pickle.dump([contexts, responses, labels], f)

def read_discussion_forum_from_file():
    with open('./data/discussion/discussion_forum_indices', 'rb') as f:
        my_list = pickle.load(f)
    d = list(zip(my_list[0], my_list[1], my_list[2]))
    random.shuffle(d)
    contexts, responses, labels = zip(*d)
    return contexts, responses, labels

def save_reddit_data():
    data = read_reddit_data()
    random.shuffle(data)
    contexts, responses, labels = generate_indices(data, "reddit")
    with open('./data/reddit/reddit_indices', 'wb') as f:
        pickle.dump([contexts, responses, labels], f)

def read_reddit_data_from_file():
    with open('./data/reddit/reddit_indices', 'rb') as f:
        my_list = pickle.load(f)
    d = list(zip(my_list[0], my_list[1], my_list[2]))
    random.shuffle(d)
    contexts, responses, labels = zip(*d)
    return contexts, responses, labels

def torch_from_json(path, dtype=torch.float32):
    """Load a PyTorch Tensor from a JSON file.

    Args:
        path (str): Path to the JSON file to load.
        dtype (torch.dtype): Data type of loaded array.

    Returns:
        tensor (torch.Tensor): Tensor loaded from JSON file.
    """

    with open(path, 'r') as fh:
        array = np.array(json.load(fh))

    tensor = torch.from_numpy(array).type(dtype)
    if torch.cuda.is_available():
        tensor = tensor.cuda()

    return tensor

'''
Separate every content to sentences, and generate index for each word. Pad the sentence and contents.
sents: a list of contents
Every content is consisted of a list: [original sentences, response sentences, label]
'''
def generate_indices(sents: List, tag):
    '''Generate index for words. If the word doesn't exist, return 1 for that position'''

    nlp = spacy.load("en_core_web_sm")
    if tag == "discussion":
        word2idxPath = './data/discussion/word2idx.json'
    else:
        word2idxPath = './data/reddit/word2idx.json'

    with open('./data/discussion/word2idx.json') as handle:
            word2idx = json.loads(handle.read())

    originals = []
    responses = []
    labels = []

    count = 0

    for content in sents:
        # separate every content into a list of sentences
        original = [sentence.text for sentence in nlp(content[0]).sents]
        response = [sentence.text for sentence in nlp(content[1]).sents]
        label = content[2]
        originals.append([[word2idx[word.text] if word.text in word2idx else 1 for word in nlp(sentence)] for sentence in original])
        responses.append([[word2idx[word.text] if word.text in word2idx else 1 for word in nlp(sentence)] for sentence in response])
        labels.append(label)
        count += 1
        if count % 1000 is 0:
            print("At count {}".format(count))
            originals = pad_sents(originals)
            responses = pad_sents(responses)
            with open('./data/reddit/reddit_indices', 'wb') as f:
                pickle.dump([originals, responses, labels], f)
            # if count > 8000:
            #     break
    originals = pad_sents(originals)
    responses = pad_sents(responses)
    # print(originals)
    # print(responses)
    print("Generated indices for contexts and responses.")
    return originals, responses, labels

'''
@return: A list of content padded
Every content has max_content_length of sentences
Every sentence has max_sentence_length of words
'''
def pad_sents(sents: List):
    # max_content_length = np.max([len(content) for content in sents])
    # max_sentence_length = [len(sentence) for sentence in content for content in sents]
    max_content_length = 10
    max_sentence_length = 50
    for content in sents:
        if len(content) > max_content_length:
            del content[max_content_length:]
        else:
            content += (max_content_length - len(content)) * [max_sentence_length * [0]]

        for sentence in content:
            if len(sentence) > max_sentence_length:
                del sentence[max_sentence_length:]
            else:
                sentence += (max_sentence_length - len(sentence)) * [0]
    return sents

def read_reddit_data(file="./data/reddit/train-balanced-sarcasm.csv"):
    with open(file) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        data = []
        for row in csv_reader:
            # if line_count > 1000:
            #     break
            if line_count == 0:
                print('Column names are {}'.format(", ".join(row)))
                line_count += 1
            else:
                '''Remember to trim leading and trailing spaces'''
                data.append([row[9].strip(), row[1].strip(), 1 if row[0] == '1' else 0])
                line_count += 1
        print('Processed {} lines.'.format(line_count))
        return data

if __name__ == '__main__':
    # read_discussion_forum()
    # save_discussion_forum_data()
    save_reddit_data()
    # with open('./data/discussion/discussion_forum_indices', 'rb') as f:
    #     my_list = pickle.load(f)
    #     print(my_list)


"""
You are given an array of arrays a. Your task is to group the arrays a[i] by their mean values,
 so that arrays with equal mean values are in the same group, and arrays with different mean
 values are in different groups.

Each group should contain a set of indices (i, j, etc), such that the corresponding
arrays (a[i], a[j], etc) all have the same mean. Return the set of groups as an array
of arrays, where the indices within each group are sorted in ascending order, and the
groups are sorted in ascending order of their minimum element.

"""
from collections import defaultdict

def meanGroups(a):
    grouped = defaultdict(list)

    for i in range(len(a)):
        grouped[sum(a[i]) / max(1, len(a[i]))].append(i)

    res = list(grouped.values())
    [x.sort() for x in res]
    return sorted(res, key=min)


# from django.contrib.gis.geoip2 import GeoIP2
from collections import namedtuple
from .models import Destination, Tourist, DestinationCityDetails, DestinationMetaDetails
import folium
from geopy.geocoders import Nominatim
import socket
import geocoder
from geopy.distance import geodesic
import requests
import json
from ipregistry import IpregistryClient

#creating my project helper functions here

def get_center_coordinates(latA, longA, latB=None, longB=None):
    cord = (latA, longA)
    if latB:
        cord = [(latA+latB)/2, (longA+longB)/2]
    return cord

def get_zoom(distance):

    if distance <=20:
        return 12
    elif distance<=50:
        return 10
    elif distance <= 100:
        return 9
    elif distance > 100 and distance <= 1500:
        return 5
    elif distance > 1501 and distance <= 3000:
        return 4
    else:
        return 2

def location_module(destination_):

    city=destination_
    geolocator = Nominatim(user_agent='syedarfa')

    client = IpregistryClient("escffgkb0orli5")
    ipInfo = client.lookup()
    # print(ipInfo)
    # print(ipInfo.ip)
    # print(ipInfo.location.get('latitude'))
    # print(ipInfo.location.get('longitude'))
    mylatitude = ipInfo.location.get('latitude')
    mylongitude = ipInfo.location.get('longitude')
    mycity = ipInfo.location.get('city')

    tourist_lat, tourist_lon = mylatitude, mylongitude

    print("yahan", tourist_lat)
    touristLocation = (tourist_lat, tourist_lon)
    # to get the location details of the tourist
    touristlocdetails = geolocator.reverse(str(tourist_lat) + ',' + str(tourist_lon))
    touristaddress = touristlocdetails.raw['address']
    # print(touristaddress)
    tourist_state = touristaddress.get('state', '')
    tourist_country = touristaddress.get('country', '')
    tourist_locdetails = tourist_state + tourist_country
    # print('Details are ' + tourist_locdetails)

    # Map using folium library
    # the map will point to the location of tourist
    mapobj = folium.Map(width=380, height=600, location=touristLocation, zoom_start=15)
    # adding marker at tourist location
    folium.Marker([tourist_lat, tourist_lon], tooltip='Click to get the name of city', popup=mycity,
                  icon=folium.Icon(color='green')).add_to(mapobj)

    if geolocator.geocode(destination_) is None:
        return "not found"
    # print(destination.address)
    # print(destination.latitude)
    # if not destination.latitude:
    else:
        destination = geolocator.geocode(destination_)
        dest_lat = destination.latitude
        # print(destination.longitude)
        dest_long = destination.longitude

        destinationLocation = (dest_lat, dest_long)

        # Now calculating the distance between tourist location and distance location
        distance = geodesic(touristLocation, destinationLocation).km

        # Now to plot the route between tourist location and the destination
        mapobj = folium.Map(width=550, height=380,
                            location=get_center_coordinates(tourist_lat, tourist_lon, dest_lat, dest_long),
                            zoom_start=get_zoom(distance))
        # TouristLocation
        folium.Marker([tourist_lat, tourist_lon], tooltip='Click to get the name of city', popup=mycity,
                    icon=folium.Icon(color='green')).add_to(mapobj)
        # TouristDestination
        folium.Marker([dest_lat, dest_long], tooltip='Click to get the name of city', popup=destination,
                    icon=folium.Icon(color='blue')).add_to(mapobj)
        # Now we will draw line between touristlocation and tourist destination
        line = folium.PolyLine(locations=[touristLocation, destinationLocation], weight=2.5, color='red')
        # adding the line to our map now
        mapobj.add_child(line)
        mapobj = mapobj._repr_html_()
        dest_meta_details = DestinationMetaDetails.objects.filter(meta_destination__in=DestinationCityDetails.objects.filter(destination_name__icontains=city))
        # print("--------------------------------------,",dest_meta_details)
        # print("destttttttttttttttttttt===", dest_meta_details)
        return [distance,mapobj,dest_meta_details,tourist_lat,tourist_lon,tourist_locdetails,dest_lat,dest_long]


from selenium import webdriver
from selenium.webdriver.common import desired_capabilities as DC
from Modules import reserve, sendEmail
import winsound, time

# 预约档案信息
recordNumber = '201708010291318'
questionText = '您父亲的姓名？'
questionAnswer = '张全'
months_after_default_month = 1
max_trails = 5
time_to_pause = 3

# 模拟浏览器操作
browser = reserve.initiate_browser()
browser = reserve.enter_reservation_page(browser,recordNumber,questionText,questionAnswer)
browser = reserve.load_reservation_page(browser, months_after_default_month, max_trails)
fc_events = browser.find_elements_by_class_name('fc-event')
print(len(fc_events))
for i in range(len(fc_events)):
    print(i, fc_events[i-1].text, fc_events[i-1].get_attribute("style"))


from re import I
import numpy as np
import cv2
from sklearn.cluster import KMeans
from collections import Counter
import imutils
import pprint
from matplotlib import pyplot as plt
import math
import pandas as pd
from functools import wraps
import sqlite3
import dlib

from flask import *
import os
app = Flask(__name__)
@app.route("/")
def home():
    return render_template("mainpage.html")
@app.route("/first")
def first():
    return render_template("mainpage.html")
@app.route("/second")
def second():
    return render_template("module1.html")
@app.route("/lipstick")
def lipstick():
    return render_template("lipstick.html")
@app.route("/blush")
def blush():
    return render_template("blush.html")
@app.route("/eyeshadow")
def eyeshadow():
    return render_template("eyeshadow.html")
@app.route("/virtual")
def virtual():
    return render_template("virtual.html")

uploads_dir = os.path.join(app.instance_path, 'C:/Users/madhuri/Desktop/flaskapp/static')
app.config['UPLOAD_FOLDER'] = uploads_dir

@app.route("/success",methods = ['POST',"GET"])

def success():

    output = request.form.to_dict()
    event = output["name"]
    f = request.files["file"]
    f.filename = "temp.jpg"  #some custom file name that you want
    f.save(os.path.join(uploads_dir, f.filename))
    excel_file = 'data.xlsx'
    df = pd.read_excel(excel_file,engine='openpyxl')
    def extractSkin(image):
        # Taking a copy of the image
        img = image.copy()
        # Converting from BGR Colours Space to HSV
        img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

        # Defining HSV Threadholds
        lower_threshold = np.array([0, 48, 80], dtype=np.uint8)         #dark
        upper_threshold = np.array([187, 207, 255], dtype=np.uint8)      #fair

        # Single Channel mask,denoting presence of colours in the about threshold
        skinMask = cv2.inRange(img, lower_threshold, upper_threshold)

        # Cleaning up mask using Gaussian Filter
        skinMask = cv2.GaussianBlur(skinMask, (3, 3), 0)

        # Extracting skin from the threshold mask
        skin = cv2.bitwise_and(img, img, mask=skinMask)

        # Return the Skin image
        return cv2.cvtColor(skin, cv2.COLOR_HSV2BGR)

    def removeBlack(estimator_labels, estimator_cluster):

        # Check for black
        hasBlack = False

        # Get the total number of occurance for each color
        occurance_counter = Counter(estimator_labels)

        # Quick lambda function to compare to lists
        def compare(x, y): return Counter(x) == Counter(y)

        # Loop through the most common occuring color
        for x in occurance_counter.most_common(len(estimator_cluster)):

            # Quick List comprehension to convert each of RBG Numbers to int
            color = [int(i) for i in estimator_cluster[x[0]].tolist()]

            # Check if the color is [0,0,0] that if it is black
            if compare(color, [0, 0, 0]) == True:
                # delete the occurance
                del occurance_counter[x[0]]
                # remove the cluster
                hasBlack = True
                estimator_cluster = np.delete(estimator_cluster, x[0], 0)
                break

        return (occurance_counter, estimator_cluster, hasBlack)

    def getColorInformation(estimator_labels, estimator_cluster, hasThresholding=False):

        # Variable to keep count of the occurance of each color predicted
        occurance_counter = None

        # Output list variable to return
        colorInformation = []

        # Check for Black
        hasBlack = False

        # If a mask has be applied, remove th black
        if hasThresholding == True:

            (occurance, cluster, black) = removeBlack(
                estimator_labels, estimator_cluster)
            occurance_counter = occurance
            estimator_cluster = cluster
            hasBlack = black

        else:
            occurance_counter = Counter(estimator_labels)

        # Get the total sum of all the predicted occurances
        totalOccurance = sum(occurance_counter.values())

        # Loop through all the predicted colors
        for x in occurance_counter.most_common(len(estimator_cluster)):

            index = (int(x[0]))

            # Quick fix for index out of bound when there is no threshold
            index = (index-1) if ((hasThresholding & hasBlack)
                                & (int(index) != 0)) else index

            # Get the color number into a list
            color = estimator_cluster[index].tolist()

            # Get the percentage of each color
            color_percentage = (x[1]/totalOccurance)

            # make the dictionay of the information
            colorInfo = {"cluster_index": index, "color": color,
                        "color_percentage": color_percentage}

            # Add the dictionary to the list
            colorInformation.append(colorInfo)

        return colorInformation

    def extractDominantColor(image, number_of_colors=5, hasThresholding=False):

        # Quick Fix Increase cluster counter to neglect the black(Read Article)
        if hasThresholding == True:
            number_of_colors += 1

        # Taking Copy of the image
        img = image.copy()

        # Convert Image into RGB Colours Space
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        # Reshape Image
        img = img.reshape((img.shape[0]*img.shape[1]), 3)

        # Initiate KMeans Object
        estimator = KMeans(n_clusters=number_of_colors, random_state=0)

        # Fit the image
        estimator.fit(img)

        # Get Colour Information
        colorInformation = getColorInformation(
            estimator.labels_, estimator.cluster_centers_, hasThresholding)
        return colorInformation

    """## Section Two.4.2 : Putting it All together: Pretty Print

    The function makes print out the color information in a readable manner
    """

    def prety_print_data(color_info):
        global link
        r = color_info[0]['color'][0]
        tone =""
        if(r>217 and r<=255):
            tone = "fair"
        elif(r>190 and r<=217):
            tone="medium"
        elif(r>120 and r<=190):
            tone="brown"
        elif(r>=80 and r<=120):
            tone="dark"
        print(tone)
        makeup = df['suggestions'].where(df['Tone']==tone).where(df['occasion']==event)
        makeup = makeup.dropna()
        link = makeup.to_string(index=False)
        print(link)

        #mainpath = r'{}'.format(link)

    # Get Image from URL. If you want to upload an image file and use that comment the below code and replace with  image=cv2.imread("FILE_NAME")

    path = r'C:/Users/madhuri/Desktop/flaskapp/static/temp.jpg'
    img_src = cv2.imread(path)
    skin = extractSkin(img_src)

    # Find the dominant color. Default is 1 , pass the parameter 'number_of_colors=N' where N is the specified number of colors
    dominantColors = extractDominantColor(skin, hasThresholding=True)
    # Show in the dominant color information
    print("Color Information")
    prety_print_data(dominantColors)
    print("link=",link)
    return render_template("success.html",name = event,image=f.filename, link=link)

@app.route("/virtualtryon",methods = ['POST',"GET"])
def virtualtryon():
    output = request.form.to_dict()
    color = output["name1"]
    f = request.files["file1"]
    f.filename = "temp1.jpg"  #some custom file name that you want
    f.save(os.path.join(uploads_dir, f.filename))
    img = cv2.imread('C:/Users/madhuri/Desktop/flaskapp/static/temp1.jpg')
    img = cv2.resize(img, (0, 0), None, 0.5, 0.5)
    imgOriginal = img.copy()
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

    def createBox(img, points, scale = 5, masked = False, cropped = False):
        if masked:
            mask = np.zeros_like(img)
            mask = cv2.fillPoly(mask, [points], (255, 255, 255))
            img = cv2.bitwise_and(img, mask)
            #cv2.imshow("mask", img)
        if cropped:
            bbox = cv2.boundingRect(points)
            x, y, w, h = bbox
            imgCrop = img[y:y+h, x:x+w]
            imgCrop = cv2.resize(imgCrop, (0, 0), None, scale, scale)
            return imgCrop
        else:
            return mask

    imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    faces = detector(imgGray)

    excel_file = 'myntracolours_dataset.xls'
    df = pd.read_excel(excel_file)
    r = (df['R'].where(df['color'] == color)).dropna()
    g = (df['G'].where(df['color'] == color)).dropna()
    b = (df['B'].where(df['color'] == color)).dropna()
    r = r.to_string(index=False)
    b = b.to_string(index=False)
    g = g.to_string(index=False)
    red = float(r)
    green = float(g)
    blue = float(b)
    print(red,green,blue)

    for face in faces:
        x1, y1 = face.left(), face.top()
        x2, y2 = face.right(), face.bottom()
        #imgOriginal = cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)
        landmarks = predictor(imgGray, face)
        myPoints = []
        for n in range(68):
            x = landmarks.part(n).x
            y = landmarks.part(n).y
            myPoints.append([x, y])

        myPoints = np.array(myPoints)
        imglips = createBox(img, myPoints[48:61], 3, masked = True, cropped = False)

        imgColorlips = np.zeros_like(imglips)
        imgColorlips[:] = blue, green, red #BGR 255,192,203
        imgColorlips = cv2.bitwise_and(imgColorlips, imglips)
        imgColorlips = cv2.GaussianBlur(imgColorlips, (7, 7), 10)
        imgColorlips = cv2.addWeighted(imgOriginal, 1, imgColorlips, 0.4, 1)
        cv2.imshow('Colored', imgColorlips)
        path1 = 'C:/Users/madhuri/Desktop/flaskapp/static'
        cv2.imwrite(os.path.join(path1,'colored.jpg'), imgColorlips)
        return render_template("virtualtryon.html",color = color,link="C:/Users/madhuri/Desktop/flaskapp/static/colored.jpg")

if __name__=='__main__':
    app.run(debug= True,port=5001)


#considere a empresa de telefonia TCHAU. abaixo de 200 minutos, a empresa cobra R$ 0,20 por minuto.
#entre 200 e 400 minutos, o preço é 0,18 R$. Acima de 400 minutoso preço é de R$ 0,15.
#e com promoção de acima de 800 minutos preço de R$ 0,08.
#Calcule sua conta de telefone

minuto = float(input("Digite os minutos utilizados: "))
valor1 = (minuto * 0.20)
valor2 = (minuto * 0.18)
valor3 = (minuto * 0.15)
valor4 = (minuto * 0.08)
if minuto < 200:
    print ('Você terá que pagar R$: %.2f' %valor1)
elif minuto <= 400:
    print("Você terá que pagar R$: %.2f" %valor2)
elif minuto >= 800:
    print("Você terá que pagar R$: %.2f" %valor4)
else:
    print("Sua conta é de R$: %.2f" %valor3)